From d3a1d9b5093c1c9a357b9d470abc7f979e3b2168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Mon, 13 Jan 2025 09:26:55 +0100 Subject: [PATCH 01/44] Add "always denied" network access checks (#119867) --- .../bridge/EntitlementChecker.java | 22 ++++ .../common/RestEntitlementsCheckAction.java | 100 ++++++++++++++++-- .../api/ElasticsearchEntitlementChecker.java | 39 +++++++ .../runtime/policy/PolicyManager.java | 14 +++ 4 files changed, 166 insertions(+), 9 deletions(-) diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java index 8b03aeb178587..3a359eb921fc8 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -15,14 +15,18 @@ import java.net.ContentHandlerFactory; import java.net.DatagramSocketImplFactory; import java.net.FileNameMap; +import java.net.ProxySelector; +import java.net.ResponseCache; import java.net.SocketImplFactory; import java.net.URL; +import java.net.URLStreamHandler; import java.net.URLStreamHandlerFactory; import java.util.List; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; import javax.net.ssl.SSLSocketFactory; @SuppressWarnings("unused") // Called from instrumentation code inserted by the Entitlements agent @@ -167,4 +171,22 @@ public interface EntitlementChecker { void check$java_net_URLConnection$$setContentHandlerFactory(Class callerClass, ContentHandlerFactory fac); + //////////////////// + // + // Network access + // + void check$java_net_ProxySelector$$setDefault(Class callerClass, ProxySelector ps); + + void check$java_net_ResponseCache$$setDefault(Class callerClass, ResponseCache rc); + + void check$java_net_spi_InetAddressResolverProvider$(Class callerClass); + + void check$java_net_spi_URLStreamHandlerProvider$(Class callerClass); + + void check$java_net_URL$(Class callerClass, String protocol, String host, int port, String file, URLStreamHandler handler); + + void check$java_net_URL$(Class callerClass, URL context, String spec, URLStreamHandler handler); + + // The only implementation of SSLSession#getSessionContext(); unfortunately it's an interface, so we need to check the implementation + void check$sun_security_ssl_SSLSessionImpl$getSessionContext(Class callerClass, SSLSession sslSession); } diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java index 9869af4d85251..1dd8daf556226 100644 --- a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java +++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java @@ -34,14 +34,19 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.net.DatagramSocket; -import java.net.DatagramSocketImpl; -import java.net.DatagramSocketImplFactory; import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.ProxySelector; +import java.net.ResponseCache; import java.net.ServerSocket; import java.net.Socket; import java.net.URL; import java.net.URLClassLoader; import java.net.URLConnection; +import java.net.URLStreamHandler; +import java.net.spi.InetAddressResolver; +import java.net.spi.InetAddressResolverProvider; +import java.net.spi.URLStreamHandlerProvider; import java.security.NoSuchAlgorithmException; import java.util.List; import java.util.Map; @@ -50,6 +55,9 @@ import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; import static java.util.Map.entry; import static org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction.CheckAction.alwaysDenied; @@ -57,6 +65,7 @@ import static org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction.CheckAction.forPlugins; import static org.elasticsearch.rest.RestRequest.Method.GET; +@SuppressWarnings("unused") public class RestEntitlementsCheckAction extends BaseRestHandler { private static final Logger logger = LogManager.getLogger(RestEntitlementsCheckAction.class); public static final Thread NO_OP_SHUTDOWN_HOOK = new Thread(() -> {}, "Shutdown hook for testing"); @@ -125,9 +134,87 @@ static CheckAction alwaysDenied(Runnable action) { entry("socket_setSocketImplFactory", alwaysDenied(RestEntitlementsCheckAction::socket$$setSocketImplFactory)), entry("url_setURLStreamHandlerFactory", alwaysDenied(RestEntitlementsCheckAction::url$$setURLStreamHandlerFactory)), entry("urlConnection_setFileNameMap", alwaysDenied(RestEntitlementsCheckAction::urlConnection$$setFileNameMap)), - entry("urlConnection_setContentHandlerFactory", alwaysDenied(RestEntitlementsCheckAction::urlConnection$$setContentHandlerFactory)) + entry("urlConnection_setContentHandlerFactory", alwaysDenied(RestEntitlementsCheckAction::urlConnection$$setContentHandlerFactory)), + + entry("proxySelector_setDefault", alwaysDenied(RestEntitlementsCheckAction::setDefaultProxySelector)), + entry("responseCache_setDefault", alwaysDenied(RestEntitlementsCheckAction::setDefaultResponseCache)), + entry("createInetAddressResolverProvider", alwaysDenied(RestEntitlementsCheckAction::createInetAddressResolverProvider)), + entry("createURLStreamHandlerProvider", alwaysDenied(RestEntitlementsCheckAction::createURLStreamHandlerProvider)), + entry("createURLWithURLStreamHandler", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler)), + entry("createURLWithURLStreamHandler2", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler2)), + entry("sslSessionImpl_getSessionContext", alwaysDenied(RestEntitlementsCheckAction::sslSessionImplGetSessionContext)) ); + private static void createURLStreamHandlerProvider() { + var x = new URLStreamHandlerProvider() { + @Override + public URLStreamHandler createURLStreamHandler(String protocol) { + return null; + } + }; + } + + private static void sslSessionImplGetSessionContext() { + SSLSocketFactory factory = HttpsURLConnection.getDefaultSSLSocketFactory(); + try (SSLSocket socket = (SSLSocket) factory.createSocket()) { + SSLSession session = socket.getSession(); + + session.getSessionContext(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("deprecation") + private static void createURLWithURLStreamHandler() { + try { + var x = new URL("http", "host", 1234, "file", new URLStreamHandler() { + @Override + protected URLConnection openConnection(URL u) { + return null; + } + }); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("deprecation") + private static void createURLWithURLStreamHandler2() { + try { + var x = new URL(null, "spec", new URLStreamHandler() { + @Override + protected URLConnection openConnection(URL u) { + return null; + } + }); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + private static void createInetAddressResolverProvider() { + var x = new InetAddressResolverProvider() { + @Override + public InetAddressResolver get(Configuration configuration) { + return null; + } + + @Override + public String name() { + return "TEST"; + } + }; + } + + private static void setDefaultResponseCache() { + ResponseCache.setDefault(null); + } + + private static void setDefaultProxySelector() { + ProxySelector.setDefault(null); + } + private static void setDefaultSSLContext() { try { SSLContext.setDefault(SSLContext.getDefault()); @@ -270,12 +357,7 @@ private static void setHttpsConnectionProperties() { @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API") private static void datagramSocket$$setDatagramSocketImplFactory() { try { - DatagramSocket.setDatagramSocketImplFactory(new DatagramSocketImplFactory() { - @Override - public DatagramSocketImpl createDatagramSocketImpl() { - throw new IllegalStateException(); - } - }); + DatagramSocket.setDatagramSocketImplFactory(() -> { throw new IllegalStateException(); }); } catch (IOException e) { throw new IllegalStateException(e); } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index 686fb73e10bc2..ca4aaceabcebf 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -18,14 +18,18 @@ import java.net.ContentHandlerFactory; import java.net.DatagramSocketImplFactory; import java.net.FileNameMap; +import java.net.ProxySelector; +import java.net.ResponseCache; import java.net.SocketImplFactory; import java.net.URL; +import java.net.URLStreamHandler; import java.net.URLStreamHandlerFactory; import java.util.List; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; import javax.net.ssl.SSLSocketFactory; /** @@ -310,4 +314,39 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { public void check$javax_net_ssl_SSLContext$$setDefault(Class callerClass, SSLContext context) { policyManager.checkChangeJVMGlobalState(callerClass); } + + @Override + public void check$java_net_ProxySelector$$setDefault(Class callerClass, ProxySelector ps) { + policyManager.checkChangeNetworkHandling(callerClass); + } + + @Override + public void check$java_net_ResponseCache$$setDefault(Class callerClass, ResponseCache rc) { + policyManager.checkChangeNetworkHandling(callerClass); + } + + @Override + public void check$java_net_spi_InetAddressResolverProvider$(Class callerClass) { + policyManager.checkChangeNetworkHandling(callerClass); + } + + @Override + public void check$java_net_spi_URLStreamHandlerProvider$(Class callerClass) { + policyManager.checkChangeNetworkHandling(callerClass); + } + + @Override + public void check$java_net_URL$(Class callerClass, String protocol, String host, int port, String file, URLStreamHandler handler) { + policyManager.checkChangeNetworkHandling(callerClass); + } + + @Override + public void check$java_net_URL$(Class callerClass, URL context, String spec, URLStreamHandler handler) { + policyManager.checkChangeNetworkHandling(callerClass); + } + + @Override + public void check$sun_security_ssl_SSLSessionImpl$getSessionContext(Class callerClass, SSLSession sslSession) { + policyManager.checkReadSensitiveNetworkInformation(callerClass); + } } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index 9c45f2d42f03a..57449a23a8215 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -171,6 +171,20 @@ public void checkChangeJVMGlobalState(Class callerClass) { }); } + /** + * Check for operations that can modify the way network operations are handled + */ + public void checkChangeNetworkHandling(Class callerClass) { + checkChangeJVMGlobalState(callerClass); + } + + /** + * Check for operations that can access sensitive network information, e.g. secrets, tokens or SSL sessions + */ + public void checkReadSensitiveNetworkInformation(Class callerClass) { + neverEntitled(callerClass, "access sensitive network information"); + } + private String operationDescription(String methodName) { // TODO: Use a more human-readable description. Perhaps share code with InstrumentationServiceImpl.parseCheckerMethodName return methodName.substring(methodName.indexOf('$')); From 8ac3c2d3718cc208f08ee791108e1dcb4a4a8f04 Mon Sep 17 00:00:00 2001 From: George Wallace Date: Mon, 13 Jan 2025 04:03:59 -0700 Subject: [PATCH 02/44] Adjustments to match Opster, readability (#119812) (#120031) * Adjustments to match Opster, readability https://opster.com/guides/elasticsearch/search-apis/elasticsearch-bool-queries/ * Edits * Spacing --------- Co-authored-by: Kofi B <23384471+thekofimensah@users.noreply.github.com> --- docs/reference/query-dsl/bool-query.asciidoc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc index c24135a370914..27220f0d85149 100644 --- a/docs/reference/query-dsl/bool-query.asciidoc +++ b/docs/reference/query-dsl/bool-query.asciidoc @@ -13,21 +13,24 @@ occurrence types are: |======================================================================= |Occur |Description |`must` |The clause (query) must appear in matching documents and will -contribute to the score. +contribute to the score. Each query defined under a `must` acts as a logical "AND", returning only documents that match _all_ the specified queries. + +|`should` |The clause (query) should appear in the matching document. Each query defined under a `should` acts as a logical "OR", returning documents that match _any_ of the specified queries. |`filter` |The clause (query) must appear in matching documents. However unlike `must` the score of the query will be ignored. Filter clauses are executed in <>, meaning that scoring is ignored -and clauses are considered for caching. - -|`should` |The clause (query) should appear in the matching document. +and clauses are considered for caching. Each query defined under a `filter` acts as a logical "AND", returning only documents that match _all_ the specified queries. |`must_not` |The clause (query) must not appear in the matching documents. Clauses are executed in <> meaning that scoring is ignored and clauses are considered for caching. Because scoring is -ignored, a score of `0` for all documents is returned. +ignored, a score of `0` for all documents is returned. Each query defined under a `must_not` acts as a logical "NOT", returning only documents that do not match any of the specified queries. + |======================================================================= +The `must` and `should` clauses function as logical AND, OR operators, contributing to the scoring of results. However, these results will not be cached for faster retrieval. In contrast, the `filter` and `must_not` clauses are used to include or exclude results without impacting the score, unless used within a `constant_score` query. + The `bool` query takes a _more-matches-is-better_ approach, so the score from each matching `must` or `should` clause will be added together to provide the final `_score` for each document. From 161b7f727342f52472a83d790424440c23decbdf Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Mon, 13 Jan 2025 12:01:42 +0000 Subject: [PATCH 03/44] Remove deprecated ThreadPools (ES-9378) (#119941) DIRECT and FIXED_AUTO_QUEUE_SIZE thread pools were deprecated not used in Elasticsearch 8.18 --- .../main/java/org/elasticsearch/threadpool/ThreadPool.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index f55e3740aaa8f..37a3ec586d104 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.node.Node; import org.elasticsearch.node.ReportingService; import org.elasticsearch.telemetry.metric.Instrument; @@ -120,13 +119,7 @@ public static class Names { public static final String THREAD_POOL_METRIC_NAME_REJECTED = ".threads.rejected.total"; public enum ThreadPoolType { - @Deprecated(forRemoval = true) - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // no longer used, remove in v9 - DIRECT("direct"), FIXED("fixed"), - @Deprecated(forRemoval = true) - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // no longer used, remove in v9 - FIXED_AUTO_QUEUE_SIZE("fixed_auto_queue_size"), SCALING("scaling"); private final String type; From 873dc52360a9265824a70b3113c3dd350ff9249a Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Mon, 13 Jan 2025 12:03:17 +0000 Subject: [PATCH 04/44] Remove occurrences of the Java annotation @UpdateForV9 (ES-9378) (#119105) Restore contentTypeVersion check in RestCompatibleVersionHelper which was commented out when RestApiVersion.V_9 was introduced, update tests to work correctly with v9 version and remove @UpdateForV9 annotation that is no longer applicable --- .../rest/RestCompatibleVersionHelper.java | 7 ++----- ...ansportGetFeatureUpgradeStatusActionTests.java | 7 +++---- .../CompatibleNamedXContentRegistryTests.java | 3 --- .../rest/RestCompatibleVersionHelperTests.java | 15 +++++++++------ .../deprecation/NodesDeprecationCheckAction.java | 7 ------- 5 files changed, 14 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java b/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java index 07cc73f4da2b1..31758be719a66 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java +++ b/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java @@ -11,7 +11,6 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.xcontent.MediaType; import org.elasticsearch.xcontent.ParsedMediaType; @@ -27,7 +26,6 @@ class RestCompatibleVersionHelper { /** * @return The requested API version, or {@link Optional#empty()} if there was no explicit version in the request. */ - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) static Optional getCompatibleVersion( @Nullable ParsedMediaType acceptHeader, @Nullable ParsedMediaType contentTypeHeader, @@ -52,8 +50,7 @@ static Optional getCompatibleVersion( if (hasContent) { // content-type version must be current or prior - // This can be uncommented once all references to RestApiVersion.V_7 are removed - /*if (contentTypeVersion > RestApiVersion.current().major || contentTypeVersion < RestApiVersion.minimumSupported().major) { + if (contentTypeVersion > RestApiVersion.current().major || contentTypeVersion < RestApiVersion.minimumSupported().major) { throw new ElasticsearchStatusException( "Content-Type version must be either version {} or {}, but found {}. Content-Type={}", RestStatus.BAD_REQUEST, @@ -62,7 +59,7 @@ static Optional getCompatibleVersion( contentTypeVersion, contentTypeHeader ); - }*/ + } // if both accept and content-type are sent, the version must match if (contentTypeVersion != acceptVersion) { throw new ElasticsearchStatusException( diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java index 8a51963097dae..9fef4c4ed328f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java @@ -13,8 +13,8 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndexDescriptorUtils; import org.elasticsearch.indices.SystemIndices; @@ -31,7 +31,8 @@ public class TransportGetFeatureUpgradeStatusActionTests extends ESTestCase { public static String TEST_SYSTEM_INDEX_PATTERN = ".test*"; - private static final IndexVersion TEST_OLD_VERSION = IndexVersion.fromId(6000099); + // Version just before MINIMUM_COMPATIBLE in order to check that UpgradeStatus.MIGRATION_NEEDED is set correctly + private static final IndexVersion TEST_OLD_VERSION = IndexVersion.fromId(IndexVersions.MINIMUM_COMPATIBLE.id() - 1); private static final ClusterState CLUSTER_STATE = getClusterState(); private static final SystemIndices.Feature FEATURE = getFeature(); @@ -85,8 +86,6 @@ private static ClusterState getClusterState() { .numberOfReplicas(0) .build(); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - // Once we start testing 9.x, we should update this test to use a 7.x "version created" IndexMetadata indexMetadata2 = IndexMetadata.builder(".test-index-2") .settings(Settings.builder().put("index.version.created", TEST_OLD_VERSION).build()) .numberOfShards(1) diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java index de1990361e766..8f62f8d33eacd 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; @@ -116,8 +115,6 @@ public static NewSubObject parse(XContentParser parser) { } } - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - @AwaitsFix(bugUrl = "this can be re-enabled once our rest api version is bumped to V_9") public void testNotCompatibleRequest() throws IOException { NamedXContentRegistry registry = new NamedXContentRegistry( List.of( diff --git a/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java b/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java index 040ab9fd5c2e9..028438b5e9267 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java @@ -210,12 +210,15 @@ public void testObsoleteVersion() { assertThat( e.getMessage(), equalTo( - "A compatible version is required on both Content-Type and Accept headers if either one has requested a " - + "compatible version and the compatible versions must match. " - + "Accept=" - + acceptHeader(PREVIOUS_VERSION) - + ", Content-Type=" - + contentTypeHeader(OBSOLETE_VERSION) + "Content-Type version must be either version " + + CURRENT_VERSION + + " or " + + PREVIOUS_VERSION + + ", but found " + + OBSOLETE_VERSION + + ". " + + "Content-Type=" + + acceptHeader(OBSOLETE_VERSION) ) ); } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java index 2e40481f7c561..48009f8105429 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java @@ -12,7 +12,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; @@ -32,7 +31,6 @@ private NodesDeprecationCheckAction() { super(NAME); } - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // this can be replaced with TransportRequest.Empty in v9 public static class NodeRequest extends TransportRequest { public NodeRequest() {} @@ -40,11 +38,6 @@ public NodeRequest() {} public NodeRequest(StreamInput in) throws IOException { super(in); } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } } public static class NodeResponse extends BaseNodeResponse { From 0e85d408a01ab9f011cb57b6f36f250b9a2970ea Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Mon, 13 Jan 2025 14:32:42 +0100 Subject: [PATCH 05/44] ESQL: Rename Validatable iface to PostLogicalOptimizationVerificationAware (#119985) Rename interface to align it with the similar post-analysis interfaces. Related #119798. --- .../PostOptimizationVerificationAware.java | 42 +++++++++++++++++++ .../xpack/esql/capabilities/Validatable.java | 23 ---------- .../function/aggregate/WeightedAvg.java | 3 +- .../expression/function/fulltext/Match.java | 6 +-- .../expression/function/fulltext/Term.java | 6 +-- .../expression/function/grouping/Bucket.java | 6 +-- .../function/grouping/Categorize.java | 3 +- .../convert/FoldablesConvertFunction.java | 6 +-- .../function/scalar/multivalue/MvSort.java | 6 +-- .../xpack/esql/optimizer/LogicalVerifier.java | 6 +-- 10 files changed, 62 insertions(+), 45 deletions(-) create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostOptimizationVerificationAware.java delete mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostOptimizationVerificationAware.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostOptimizationVerificationAware.java new file mode 100644 index 0000000000000..e3628d2554a78 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostOptimizationVerificationAware.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.capabilities; + +import org.elasticsearch.xpack.esql.common.Failures; +import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; + +/** + * Interface implemented by expressions that require validation post logical optimization, + * when the plan and references have been not just resolved but also replaced. + */ +public interface PostOptimizationVerificationAware { + + /** + * Validates the implementing expression - discovered failures are reported to the given + * {@link Failures} class. + * + *

+ * Example: the {@link Bucket} function, which produces buckets over a numerical or date field, based on a number of literal + * arguments needs to check if its arguments are all indeed literals. This is how this verification is performed: + * 

+     *     {@code
+     *
+     *      @Override
+     *      public void postLogicalOptimizationVerification(Failures failures) {
+     *          String operation = sourceText();
+     *
+     *          failures.add(isFoldable(buckets, operation, SECOND))
+     *              .add(from != null ? isFoldable(from, operation, THIRD) : null)
+     *              .add(to != null ? isFoldable(to, operation, FOURTH) : null);
+     *      }
+     *     }
+     *
+ * + */ + void postLogicalOptimizationVerification(Failures failures); +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java deleted file mode 100644 index f6733fa3f175c..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.capabilities; - -import org.elasticsearch.xpack.esql.common.Failures; - -/** - * Interface implemented by expressions that require validation post logical optimization, - * when the plan and references have been not just resolved but also replaced. - */ -public interface Validatable { - - /** - * Validates the implementing expression - discovered failures are reported to the given - * {@link Failures} class. - */ - default void validate(Failures failures) {} -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java index 49c68d002440f..56d034a2eae1d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.capabilities.Validatable; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -34,7 +33,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; -public class WeightedAvg extends AggregateFunction implements SurrogateExpression, Validatable { +public class WeightedAvg extends AggregateFunction implements SurrogateExpression { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "WeightedAvg", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java index fb9f8cff1b8b0..ce23860dbdba7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java @@ -13,7 +13,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -60,7 +60,7 @@ /** * Full text function that performs a {@link QueryStringQuery} . */ -public class Match extends FullTextFunction implements Validatable { +public class Match extends FullTextFunction implements PostOptimizationVerificationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Match", Match::readFrom); @@ -201,7 +201,7 @@ protected TypeResolution checkParamCompatibility() { } @Override - public void validate(Failures failures) { + public void postLogicalOptimizationVerification(Failures failures) { Expression fieldExpression = field(); // Field may be converted to other data type (field_name :: data_type), so we need to check the original field if (fieldExpression instanceof AbstractConvertFunction convertFunction) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java index ff8085cd1b44b..e77f95073050f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java @@ -12,7 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -39,7 +39,7 @@ /** * Full text function that performs a {@link TermQuery} . */ -public class Term extends FullTextFunction implements Validatable { +public class Term extends FullTextFunction implements PostOptimizationVerificationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Term", Term::readFrom); @@ -100,7 +100,7 @@ protected TypeResolution resolveNonQueryParamTypes() { } @Override - public void validate(Failures failures) { + public void postLogicalOptimizationVerification(Failures failures) { if (field instanceof FieldAttribute == false) { failures.add( Failure.fail( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java index 12932ba8d6e11..113989323eff2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java @@ -15,7 +15,7 @@ import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; -import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Foldables; @@ -56,7 +56,7 @@ * from a number of desired buckets (as a hint) and a range (auto mode). * In the former case, two parameters will be provided, in the latter four. */ -public class Bucket extends GroupingFunction implements Validatable, TwoOptionalArguments { +public class Bucket extends GroupingFunction implements PostOptimizationVerificationAware, TwoOptionalArguments { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Bucket", Bucket::new); // TODO maybe we should just cover the whole of representable dates here - like ten years, 100 years, 1000 years, all the way up. @@ -408,7 +408,7 @@ private static TypeResolution isStringOrDate(Expression e, String operationName, } @Override - public void validate(Failures failures) { + public void postLogicalOptimizationVerification(Failures failures) { String operation = sourceText(); failures.add(isFoldable(buckets, operation, SECOND)) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java index a100dd64915f1..570ce7a96dd68 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; -import org.elasticsearch.xpack.esql.capabilities.Validatable; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -37,7 +36,7 @@ * For the implementation, see {@link org.elasticsearch.compute.aggregation.blockhash.CategorizeBlockHash} *

*/ -public class Categorize extends GroupingFunction implements Validatable { +public class Categorize extends GroupingFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "Categorize", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java index 8f43a6481db07..842e899ebdac6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.convert; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -26,7 +26,7 @@ * Base class for functions that converts a constant into an interval type - DATE_PERIOD or TIME_DURATION. * The functions will be folded at the end of LogicalPlanOptimizer by the coordinator, it does not reach data node. */ -public abstract class FoldablesConvertFunction extends AbstractConvertFunction implements Validatable { +public abstract class FoldablesConvertFunction extends AbstractConvertFunction implements PostOptimizationVerificationAware { protected FoldablesConvertFunction(Source source, Expression field) { super(source, field); @@ -70,7 +70,7 @@ public final Object fold() { } @Override - public final void validate(Failures failures) { + public final void postLogicalOptimizationVerification(Failures failures) { failures.add(isFoldable(field(), sourceText(), null)); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java index 2286a1357ced8..86538c828ece7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java @@ -29,7 +29,7 @@ import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeDouble; import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeInt; import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeLong; -import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -58,7 +58,7 @@ /** * Sorts a multivalued field in lexicographical order. */ -public class MvSort extends EsqlScalarFunction implements OptionalArgument, Validatable { +public class MvSort extends EsqlScalarFunction implements OptionalArgument, PostOptimizationVerificationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvSort", MvSort::new); private final Expression field, order; @@ -230,7 +230,7 @@ public DataType dataType() { } @Override - public void validate(Failures failures) { + public void postLogicalOptimizationVerification(Failures failures) { if (order == null) { return; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java index dce828dbf192d..7e6eddd3ef041 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.esql.optimizer; -import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.optimizer.rules.PlanConsistencyChecker; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -28,8 +28,8 @@ public Failures verify(LogicalPlan plan) { if (failures.hasFailures() == false) { p.forEachExpression(ex -> { - if (ex instanceof Validatable v) { - v.validate(failures); + if (ex instanceof PostOptimizationVerificationAware va) { + va.postLogicalOptimizationVerification(failures); } }); } From 8b9a8740ae236b9b01f656f4d77e8de455784422 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 14 Jan 2025 00:50:29 +1100 Subject: [PATCH 06/44] Mute org.elasticsearch.entitlement.qa.EntitlementsDeniedIT testCheckThrows {pathPrefix=denied actionName=sslSessionImpl_getSessionContext} #120053 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 2f652f87ab283..1ff2a88e570cf 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -271,6 +271,9 @@ tests: - class: org.elasticsearch.index.mapper.IntervalThrottlerTests method: testThrottling issue: https://github.com/elastic/elasticsearch/issues/120023 +- class: org.elasticsearch.entitlement.qa.EntitlementsDeniedIT + method: testCheckThrows {pathPrefix=denied actionName=sslSessionImpl_getSessionContext} + issue: https://github.com/elastic/elasticsearch/issues/120053 # Examples: # From 0ec55e57c44459cb313c209bfadb26a65993b39c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 14 Jan 2025 00:50:40 +1100 Subject: [PATCH 07/44] Mute org.elasticsearch.entitlement.qa.EntitlementsDeniedIT testCheckThrows {pathPrefix=denied_nonmodular actionName=sslSessionImpl_getSessionContext} #120054 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 1ff2a88e570cf..d9fcebb7f2916 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -274,6 +274,9 @@ tests: - class: org.elasticsearch.entitlement.qa.EntitlementsDeniedIT method: testCheckThrows {pathPrefix=denied actionName=sslSessionImpl_getSessionContext} issue: https://github.com/elastic/elasticsearch/issues/120053 +- class: org.elasticsearch.entitlement.qa.EntitlementsDeniedIT + method: testCheckThrows {pathPrefix=denied_nonmodular actionName=sslSessionImpl_getSessionContext} + issue: https://github.com/elastic/elasticsearch/issues/120054 # Examples: # From e9f2d7892393e1d339d2a10b1298a5cab490afb1 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Mon, 13 Jan 2025 09:11:52 -0500 Subject: [PATCH 08/44] Esql additional date format testing (#120000) This wires up the randomized testing for DateFormat. Prior to this PR, none of the randomized testing was hitting the one parameter version of the function, so I wired that up as well. This required some compromises on the type signatures, see comments in line.less --------- Co-authored-by: elasticsearchmachine --- .../kibana/definition/date_format.json | 12 ++++ .../esql/functions/types/date_format.asciidoc | 1 + .../function/scalar/date/DateFormat.java | 2 +- .../expression/function/TestCaseSupplier.java | 24 ++++++++ .../scalar/date/DateFormatErrorTests.java | 13 ++++- .../function/scalar/date/DateFormatTests.java | 57 +++++++++---------- 6 files changed, 78 insertions(+), 31 deletions(-) diff --git a/docs/reference/esql/functions/kibana/definition/date_format.json b/docs/reference/esql/functions/kibana/definition/date_format.json index 6e2738fafb964..629415da30fa2 100644 --- a/docs/reference/esql/functions/kibana/definition/date_format.json +++ b/docs/reference/esql/functions/kibana/definition/date_format.json @@ -4,6 +4,18 @@ "name" : "date_format", "description" : "Returns a string representation of a date, in the provided format.", "signatures" : [ + { + "params" : [ + { + "name" : "dateFormat", + "type" : "date", + "optional" : true, + "description" : "Date format (optional). If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/types/date_format.asciidoc b/docs/reference/esql/functions/types/date_format.asciidoc index b2e97dfa8835a..580094e9be906 100644 --- a/docs/reference/esql/functions/types/date_format.asciidoc +++ b/docs/reference/esql/functions/types/date_format.asciidoc @@ -5,6 +5,7 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== dateFormat | date | result +date | | keyword keyword | date | keyword text | date | keyword |=== diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java index 60bc014ccbeec..920a3bb1f4a13 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java @@ -55,7 +55,7 @@ public class DateFormat extends EsqlConfigurationFunction implements OptionalArg ) public DateFormat( Source source, - @Param(optional = true, name = "dateFormat", type = { "keyword", "text" }, description = """ + @Param(optional = true, name = "dateFormat", type = { "keyword", "text", "date" }, description = """ Date format (optional). If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used. If `null`, the function returns `null`.""") Expression format, @Param(name = "date", type = { "date" }, description = "Date expression. If `null`, the function returns `null`.") Expression date, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index 72d816a65e632..f2bae0c5a4979 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -1007,6 +1007,17 @@ public static List dateCases() { return dateCases(Long.MIN_VALUE, Long.MAX_VALUE); } + /** + * Generate cases for {@link DataType#DATETIME}. + *

+ * For multi-row parameters, see {@link MultiRowTestCaseSupplier#dateCases}. + *

+ * Helper function for if you want to specify your min and max range as dates instead of longs. + */ + public static List dateCases(Instant min, Instant max) { + return dateCases(min.toEpochMilli(), max.toEpochMilli()); + } + /** * Generate cases for {@link DataType#DATETIME}. *

@@ -1045,6 +1056,19 @@ public static List dateCases(long min, long max) { return cases; } + /** + * + * @return randomized valid date formats + */ + public static List dateFormatCases() { + return List.of( + new TypedDataSupplier("", () -> new BytesRef(ESTestCase.randomDateFormatterPattern()), DataType.KEYWORD), + new TypedDataSupplier("", () -> new BytesRef(ESTestCase.randomDateFormatterPattern()), DataType.TEXT), + new TypedDataSupplier("", () -> new BytesRef("yyyy"), DataType.KEYWORD), + new TypedDataSupplier("", () -> new BytesRef("yyyy"), DataType.TEXT) + ); + } + /** * Generate cases for {@link DataType#DATE_NANOS}. * diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java index 985f1144fbcf2..a5e6514b3e02c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java @@ -28,11 +28,22 @@ protected List cases() { @Override protected Expression build(Source source, List args) { - return new DateFormat(source, args.get(0), args.get(1), EsqlTestUtils.TEST_CFG); + return new DateFormat(source, args.get(0), args.size() == 2 ? args.get(1) : null, EsqlTestUtils.TEST_CFG); } @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + // Single argument version + String source = sourceForSignature(signature); + String name = signature.get(0).typeName(); + if (signature.size() == 1) { + return equalTo("first argument of [" + source + "] must be [datetime], found value [] type [" + name + "]"); + } + // Two argument version + // Handle the weird case where we're calling the two argument version with the date first instead of the format. + if (signature.get(0).isDate()) { + return equalTo("first argument of [" + source + "] must be [string], found value [] type [" + name + "]"); + } return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { case 0 -> "string"; case 1 -> "datetime"; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java index 8dfdd1ba486c7..3dd1f3e629da4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java @@ -11,18 +11,21 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase; import org.elasticsearch.xpack.esql.session.Configuration; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; +import java.time.Instant; +import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.matchesPattern; public class DateFormatTests extends AbstractConfigurationFunctionTestCase { public DateFormatTests(@Name("TestCase") Supplier testCaseSupplier) { @@ -31,39 +34,35 @@ public DateFormatTests(@Name("TestCase") Supplier tes @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( - true, - List.of( - new TestCaseSupplier( - List.of(DataType.KEYWORD, DataType.DATETIME), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("yyyy"), DataType.KEYWORD, "formatter"), - new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "val") - ), - "DateFormatEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], locale=en_US]", - DataType.KEYWORD, - equalTo(BytesRefs.toBytesRef("2023")) - ) + List suppliers = new ArrayList<>(); + // Formatter supplied cases + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + (format, value) -> new BytesRef( + DateFormatter.forPattern(((BytesRef) format).utf8ToString()).formatMillis(((Instant) value).toEpochMilli()) ), - new TestCaseSupplier( - List.of(DataType.TEXT, DataType.DATETIME), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("yyyy"), DataType.TEXT, "formatter"), - new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "val") - ), - "DateFormatEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], locale=en_US]", - DataType.KEYWORD, - equalTo(BytesRefs.toBytesRef("2023")) - ) - ) + DataType.KEYWORD, + TestCaseSupplier.dateFormatCases(), + TestCaseSupplier.dateCases(Instant.parse("1900-01-01T00:00:00.00Z"), Instant.parse("9999-12-31T00:00:00.00Z")), + matchesPattern("DateFormatEvaluator\\[val=Attribute\\[channel=1], formatter=Attribute\\[(channel=0|\\w+)], locale=en_US]"), + (lhs, rhs) -> List.of(), + false ) ); + // Default formatter cases + TestCaseSupplier.unary( + suppliers, + "DateFormatConstantEvaluator[val=Attribute[channel=0], formatter=format[strict_date_optional_time] locale[]]", + TestCaseSupplier.dateCases(Instant.parse("1900-01-01T00:00:00.00Z"), Instant.parse("9999-12-31T00:00:00.00Z")), + DataType.KEYWORD, + (value) -> new BytesRef(EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER.formatMillis(((Instant) value).toEpochMilli())), + List.of() + ); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override protected Expression buildWithConfiguration(Source source, List args, Configuration configuration) { - return new DateFormat(source, args.get(0), args.get(1), configuration); + return new DateFormat(source, args.get(0), args.size() == 2 ? args.get(1) : null, configuration); } } From 838a41a8391f1d579a1c8e77c8630a42cddcd087 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Mon, 13 Jan 2025 09:48:23 -0500 Subject: [PATCH 09/44] [ML] Adding docs for the unified inference API (#118696) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Including examples * Using js instead of json * Adding unified docs to main page * Adding missing description text * Refactoring to remove unified route * Addign back references to the _unified route * Update docs/reference/inference/chat-completion-inference.asciidoc Co-authored-by: István Zoltán Szabó * Address feedback --------- Co-authored-by: István Zoltán Szabó --- .../chat-completion-inference.asciidoc | 417 ++++++++++++++++++ .../inference/inference-apis.asciidoc | 10 +- .../inference/inference-shared.asciidoc | 46 +- .../inference/put-inference.asciidoc | 8 +- .../inference/service-openai.asciidoc | 12 +- .../inference/stream-inference.asciidoc | 6 +- 6 files changed, 487 insertions(+), 12 deletions(-) create mode 100644 docs/reference/inference/chat-completion-inference.asciidoc diff --git a/docs/reference/inference/chat-completion-inference.asciidoc b/docs/reference/inference/chat-completion-inference.asciidoc new file mode 100644 index 0000000000000..83a8f94634f2f --- /dev/null +++ b/docs/reference/inference/chat-completion-inference.asciidoc @@ -0,0 +1,417 @@ +[role="xpack"] +[[chat-completion-inference-api]] +=== Chat completion inference API + +Streams a chat completion response. + +IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. +For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. +However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. + + +[discrete] +[[chat-completion-inference-api-request]] +==== {api-request-title} + +`POST /_inference//_unified` + +`POST /_inference/chat_completion//_unified` + + +[discrete] +[[chat-completion-inference-api-prereqs]] +==== {api-prereq-title} + +* Requires the `monitor_inference` <> +(the built-in `inference_admin` and `inference_user` roles grant this privilege) +* You must use a client that supports streaming. + + +[discrete] +[[chat-completion-inference-api-desc]] +==== {api-description-title} + +The chat completion {infer} API enables real-time responses for chat completion tasks by delivering answers incrementally, reducing response times during computation. +It only works with the `chat_completion` task type for `openai` and `elastic` {infer} services. + +[NOTE] +==== +The `chat_completion` task type is only available within the _unified API and only supports streaming. +==== + +[discrete] +[[chat-completion-inference-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +The unique identifier of the {infer} endpoint. + + +``:: +(Optional, string) +The type of {infer} task that the model performs. If included, this must be set to the value `chat_completion`. + + +[discrete] +[[chat-completion-inference-api-request-body]] +==== {api-request-body-title} + +`messages`:: +(Required, array of objects) A list of objects representing the conversation. +Requests should generally only add new messages from the user (role `user`). The other message roles (`assistant`, `system`, or `tool`) should generally only be copied from the response to a previous completion request, such that the messages array is built up throughout a conversation. ++ +.Assistant message +[%collapsible%closed] +===== +`content`:: +(Required unless `tool_calls` is specified, string or array of objects) +The contents of the message. ++ +include::inference-shared.asciidoc[tag=chat-completion-schema-content-with-examples] ++ +`role`:: +(Required, string) +The role of the message author. This should be set to `assistant` for this type of message. ++ +`tool_calls`:: +(Optional, array of objects) +The tool calls generated by the model. ++ +.Examples +[%collapsible%closed] +====== +[source,js] +------------------------------------------------------------ +{ + "tool_calls": [ + { + "id": "call_KcAjWtAww20AihPHphUh46Gd", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\"location\":\"Boston, MA\"}" + } + } + ] +} +------------------------------------------------------------ +// NOTCONSOLE +====== ++ +`id`::: +(Required, string) +The identifier of the tool call. ++ +`type`::: +(Required, string) +The type of tool call. This must be set to the value `function`. ++ +`function`::: +(Required, object) +The function that the model called. ++ +`name`:::: +(Required, string) +The name of the function to call. ++ +`arguments`:::: +(Required, string) +The arguments to call the function with in JSON format. +===== ++ +.System message +[%collapsible%closed] +===== +`content`::: +(Required, string or array of objects) +The contents of the message. ++ +include::inference-shared.asciidoc[tag=chat-completion-schema-content-with-examples] ++ +`role`::: +(Required, string) +The role of the message author. This should be set to `system` for this type of message. +===== ++ +.Tool message +[%collapsible%closed] +===== +`content`:: +(Required, string or array of objects) +The contents of the message. ++ +include::inference-shared.asciidoc[tag=chat-completion-schema-content-with-examples] ++ +`role`:: +(Required, string) +The role of the message author. This should be set to `tool` for this type of message. ++ +`tool_call_id`:: +(Required, string) +The tool call that this message is responding to. +===== ++ +.User message +[%collapsible%closed] +===== +`content`:: +(Required, string or array of objects) +The contents of the message. ++ +include::inference-shared.asciidoc[tag=chat-completion-schema-content-with-examples] ++ +`role`:: +(Required, string) +The role of the message author. This should be set to `user` for this type of message. +===== + +`model`:: +(Optional, string) +The ID of the model to use. By default, the model ID is set to the value included when creating the inference endpoint. + +`max_completion_tokens`:: +(Optional, integer) +The upper bound limit for the number of tokens that can be generated for a completion request. + +`stop`:: +(Optional, array of strings) +A sequence of strings to control when the model should stop generating additional tokens. + +`temperature`:: +(Optional, float) +The sampling temperature to use. + +`tools`:: +(Optional, array of objects) +A list of tools that the model can call. ++ +.Structure +[%collapsible%closed] +===== +`type`:: +(Required, string) +The type of tool, must be set to the value `function`. ++ +`function`:: +(Required, object) +The function definition. ++ +`description`::: +(Optional, string) +A description of what the function does. This is used by the model to choose when and how to call the function. ++ +`name`::: +(Required, string) +The name of the function. ++ +`parameters`::: +(Optional, object) +The parameters the functional accepts. This should be formatted as a JSON object. ++ +`strict`::: +(Optional, boolean) +Whether to enable schema adherence when generating the function call. +===== ++ +.Examples +[%collapsible%closed] +====== +[source,js] +------------------------------------------------------------ +{ + "tools": [ + { + "type": "function", + "function": { + "name": "get_price_of_item", + "description": "Get the current price of an item", + "parameters": { + "type": "object", + "properties": { + "item": { + "id": "12345" + }, + "unit": { + "type": "currency" + } + } + } + } + } + ] +} +------------------------------------------------------------ +// NOTCONSOLE +====== + +`tool_choice`:: +(Optional, string or object) +Controls which tool is called by the model. ++ +String representation::: +One of `auto`, `none`, or `requrired`. `auto` allows the model to choose between calling tools and generating a message. `none` causes the model to not call any tools. `required` forces the model to call one or more tools. ++ +Object representation::: ++ +.Structure +[%collapsible%closed] +===== +`type`:: +(Required, string) +The type of the tool. This must be set to the value `function`. ++ +`function`:: +(Required, object) ++ +`name`::: +(Required, string) +The name of the function to call. +===== ++ +.Examples +[%collapsible%closed] +===== +[source,js] +------------------------------------------------------------ +{ + "tool_choice": { + "type": "function", + "function": { + "name": "get_current_weather" + } + } +} +------------------------------------------------------------ +// NOTCONSOLE +===== + +`top_p`:: +(Optional, float) +Nucleus sampling, an alternative to sampling with temperature. + +[discrete] +[[chat-completion-inference-api-example]] +==== {api-examples-title} + +The following example performs a chat completion on the example question with streaming. + + +[source,console] +------------------------------------------------------------ +POST _inference/chat_completion/openai-completion/_stream +{ + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": "What is Elastic?" + } + ] +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The following example performs a chat completion using an Assistant message with `tool_calls`. + +[source,console] +------------------------------------------------------------ +POST _inference/chat_completion/openai-completion/_stream +{ + "messages": [ + { + "role": "assistant", + "content": "Let's find out what the weather is", + "tool_calls": [ <1> + { + "id": "call_KcAjWtAww20AihPHphUh46Gd", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\"location\":\"Boston, MA\"}" + } + } + ] + }, + { <2> + "role": "tool", + "content": "The weather is cold", + "tool_call_id": "call_KcAjWtAww20AihPHphUh46Gd" + } + ] +} +------------------------------------------------------------ +// TEST[skip:TBD] + +<1> Each tool call needs a corresponding Tool message. +<2> The corresponding Tool message. + +The following example performs a chat completion using a User message with `tools` and `tool_choice`. + +[source,console] +------------------------------------------------------------ +POST _inference/chat_completion/openai-completion/_stream +{ + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What's the price of a scarf?" + } + ] + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_price", + "description": "Get the current price of a item", + "parameters": { + "type": "object", + "properties": { + "item": { + "id": "123" + } + } + } + } + } + ], + "tool_choice": { + "type": "function", + "function": { + "name": "get_current_price" + } + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The API returns the following response when a request is made to the OpenAI service: + + +[source,txt] +------------------------------------------------------------ +event: message +data: {"chat_completion":{"id":"chatcmpl-Ae0TWsy2VPnSfBbv5UztnSdYUMFP3","choices":[{"delta":{"content":"","role":"assistant"},"index":0}],"model":"gpt-4o-2024-08-06","object":"chat.completion.chunk"}} + +event: message +data: {"chat_completion":{"id":"chatcmpl-Ae0TWsy2VPnSfBbv5UztnSdYUMFP3","choices":[{"delta":{"content":Elastic"},"index":0}],"model":"gpt-4o-2024-08-06","object":"chat.completion.chunk"}} + +event: message +data: {"chat_completion":{"id":"chatcmpl-Ae0TWsy2VPnSfBbv5UztnSdYUMFP3","choices":[{"delta":{"content":" is"},"index":0}],"model":"gpt-4o-2024-08-06","object":"chat.completion.chunk"}} + +(...) + +event: message +data: {"chat_completion":{"id":"chatcmpl-Ae0TWsy2VPnSfBbv5UztnSdYUMFP3","choices":[],"model":"gpt-4o-2024-08-06","object":"chat.completion.chunk","usage":{"completion_tokens":28,"prompt_tokens":16,"total_tokens":44}}} <1> + +event: message +data: [DONE] +------------------------------------------------------------ +// NOTCONSOLE + +<1> The last object message of the stream contains the token usage information. diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc index ca273afc478ea..4f27409973ca2 100644 --- a/docs/reference/inference/inference-apis.asciidoc +++ b/docs/reference/inference/inference-apis.asciidoc @@ -26,6 +26,7 @@ the following APIs to manage {infer} models and perform {infer}: * <> * <> * <> +* <> * <> [[inference-landscape]] @@ -34,9 +35,9 @@ image::images/inference-landscape.jpg[A representation of the Elastic inference An {infer} endpoint enables you to use the corresponding {ml} model without manual deployment and apply it to your data at ingestion time through -<>. +<>. -Choose a model from your provider or use ELSER – a retrieval model trained by +Choose a model from your provider or use ELSER – a retrieval model trained by Elastic –, then create an {infer} endpoint by the <>. Now use <> to perform <> on your data. @@ -67,7 +68,7 @@ The following list contains the default {infer} endpoints listed by `inference_i Use the `inference_id` of the endpoint in a <> field definition or when creating an <>. The API call will automatically download and deploy the model which might take a couple of minutes. Default {infer} enpoints have {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations] enabled. -For these models, the minimum number of allocations is `0`. +For these models, the minimum number of allocations is `0`. If there is no {infer} activity that uses the endpoint, the number of allocations will scale down to `0` automatically after 15 minutes. @@ -84,7 +85,7 @@ Returning a long document in search results is less useful than providing the mo Each chunk will include the text subpassage and the corresponding embedding generated from it. By default, documents are split into sentences and grouped in sections up to 250 words with 1 sentence overlap so that each chunk shares a sentence with the previous chunk. -Overlapping ensures continuity and prevents vital contextual information in the input text from being lost by a hard break. +Overlapping ensures continuity and prevents vital contextual information in the input text from being lost by a hard break. {es} uses the https://unicode-org.github.io/icu-docs/[ICU4J] library to detect word and sentence boundaries for chunking. https://unicode-org.github.io/icu/userguide/boundaryanalysis/#word-boundary[Word boundaries] are identified by following a series of rules, not just the presence of a whitespace character. @@ -135,6 +136,7 @@ PUT _inference/sparse_embedding/small_chunk_size include::delete-inference.asciidoc[] include::get-inference.asciidoc[] include::post-inference.asciidoc[] +include::chat-completion-inference.asciidoc[] include::put-inference.asciidoc[] include::stream-inference.asciidoc[] include::update-inference.asciidoc[] diff --git a/docs/reference/inference/inference-shared.asciidoc b/docs/reference/inference/inference-shared.asciidoc index da497c6581e5d..b133c54082810 100644 --- a/docs/reference/inference/inference-shared.asciidoc +++ b/docs/reference/inference/inference-shared.asciidoc @@ -41,7 +41,7 @@ end::chunking-settings[] tag::chunking-settings-max-chunking-size[] Specifies the maximum size of a chunk in words. Defaults to `250`. -This value cannot be higher than `300` or lower than `20` (for `sentence` strategy) or `10` (for `word` strategy). +This value cannot be higher than `300` or lower than `20` (for `sentence` strategy) or `10` (for `word` strategy). end::chunking-settings-max-chunking-size[] tag::chunking-settings-overlap[] @@ -63,4 +63,48 @@ Specifies the chunking strategy. It could be either `sentence` or `word`. end::chunking-settings-strategy[] +tag::chat-completion-schema-content-with-examples[] +.Examples +[%collapsible%closed] +====== +String example +[source,js] +------------------------------------------------------------ +{ + "content": "Some string" +} +------------------------------------------------------------ +// NOTCONSOLE + +Object example +[source,js] +------------------------------------------------------------ +{ + "content": [ + { + "text": "Some text", + "type": "text" + } + ] +} +------------------------------------------------------------ +// NOTCONSOLE +====== + +String representation::: +(Required, string) +The text content. ++ +Object representation::: +`text`:::: +(Required, string) +The text content. ++ +`type`:::: +(Required, string) +This must be set to the value `text`. +end::chat-completion-schema-content-with-examples[] +tag::chat-completion-docs[] +For more information on how to use the `chat_completion` task type, please refer to the <>. +end::chat-completion-docs[] diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index f0c15323863d7..da07d1d3e7d84 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -42,7 +42,7 @@ include::inference-shared.asciidoc[tag=inference-id] include::inference-shared.asciidoc[tag=task-type] + -- -Refer to the service list in the <> for the available task types. +Refer to the service list in the <> for the available task types. -- @@ -61,7 +61,7 @@ The create {infer} API enables you to create an {infer} endpoint and configure a The following services are available through the {infer} API. -You can find the available task types next to the service name. +You can find the available task types next to the service name. Click the links to review the configuration details of the services: * <> (`completion`, `rerank`, `sparse_embedding`, `text_embedding`) @@ -73,10 +73,10 @@ Click the links to review the configuration details of the services: * <> (`rerank`, `sparse_embedding`, `text_embedding` - this service is for built-in models and models uploaded through Eland) * <> (`sparse_embedding`) * <> (`completion`, `text_embedding`) -* <> (`rerank`, `text_embedding`) +* <> (`rerank`, `text_embedding`) * <> (`text_embedding`) * <> (`text_embedding`) -* <> (`completion`, `text_embedding`) +* <> (`chat_completion`, `completion`, `text_embedding`) * <> (`text_embedding`) * <> (`text_embedding`, `rerank`) diff --git a/docs/reference/inference/service-openai.asciidoc b/docs/reference/inference/service-openai.asciidoc index e4be7f18e09dd..590f280b1c494 100644 --- a/docs/reference/inference/service-openai.asciidoc +++ b/docs/reference/inference/service-openai.asciidoc @@ -31,10 +31,18 @@ include::inference-shared.asciidoc[tag=task-type] -- Available task types: +* `chat_completion`, * `completion`, * `text_embedding`. -- +[NOTE] +==== +The `chat_completion` task type only supports streaming and only through the `_unified` API. + +include::inference-shared.asciidoc[tag=chat-completion-docs] +==== + [discrete] [[infer-service-openai-api-request-body]] ==== {api-request-body-title} @@ -61,7 +69,7 @@ include::inference-shared.asciidoc[tag=chunking-settings-strategy] `service`:: (Required, string) -The type of service supported for the specified task type. In this case, +The type of service supported for the specified task type. In this case, `openai`. `service_settings`:: @@ -176,4 +184,4 @@ PUT _inference/completion/openai-completion } } ------------------------------------------------------------ -// TEST[skip:TBD] \ No newline at end of file +// TEST[skip:TBD] diff --git a/docs/reference/inference/stream-inference.asciidoc b/docs/reference/inference/stream-inference.asciidoc index 42abb589f9afd..4a3ce31909712 100644 --- a/docs/reference/inference/stream-inference.asciidoc +++ b/docs/reference/inference/stream-inference.asciidoc @@ -38,8 +38,12 @@ However, if you do not plan to use the {infer} APIs to use these models or if yo ==== {api-description-title} The stream {infer} API enables real-time responses for completion tasks by delivering answers incrementally, reducing response times during computation. -It only works with the `completion` task type. +It only works with the `completion` and `chat_completion` task types. +[NOTE] +==== +include::inference-shared.asciidoc[tag=chat-completion-docs] +==== [discrete] [[stream-inference-api-path-params]] From 67e1bd46a6d45e0d1e2f7d6f22486413f42bf48e Mon Sep 17 00:00:00 2001 From: Jan Kuipers <148754765+jan-elastic@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:50:56 +0100 Subject: [PATCH 10/44] Clean up lingering tasks after DatafeedJobsIT. (#120040) --- .../elasticsearch/xpack/ml/integration/DatafeedJobsIT.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java index 5287d149fae3d..367c1cee8b0ee 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java @@ -11,6 +11,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.ReferenceDocs; @@ -80,6 +82,10 @@ public class DatafeedJobsIT extends MlNativeAutodetectIntegTestCase { public void cleanup() { updateClusterSettings(Settings.builder().putNull("logger.org.elasticsearch.xpack.ml.datafeed")); cleanUp(); + // Race conditions between closing and killing tasks in these tests, + // sometimes result in lingering persistent tasks (such as "_close"), + // which cause subsequent tests to fail. + client().execute(TransportCancelTasksAction.TYPE, new CancelTasksRequest()); } public void testLookbackOnly() throws Exception { From c990377c955ffeca7d08235411a0c468bba1ac36 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 13 Jan 2025 10:04:27 -0500 Subject: [PATCH 11/44] ESQL: Limit memory usage of `fold` (#118602) `fold` can be surprisingly heavy! The maximally efficient/paranoid thing would be to fold each expression one time, in the constant folding rule, and then store the result as a `Literal`. But this PR doesn't do that because it's a big change. Instead, it creates the infrastructure for tracking memory usage for folding as plugs it into as many places as possible. That's not perfect, but it's better. This infrastructure limit the allocations of fold similar to the `CircuitBreaker` infrastructure we use for values, but it's different in a critical way: you don't manually free any of the values. This is important because the plan itself isn't `Releasable`, which is required when using a real CircuitBreaker. We could have tried to make the plan releasable, but that'd be a huge change. Right now there's a single limit of 5% of heap per query. We create the limit at the start of query planning and use it throughout planning. There are about 40 places that don't yet use it. We should get them plugged in as quick as we can manage. After that, we should look to the maximally efficient/paranoid thing that I mentioned about waiting for constant folding. That's an even bigger change, one I'm not equipped to make on my own. --- .../compute/operator/EvalBenchmark.java | 25 ++- docs/changelog/118602.yaml | 5 + .../functions/kibana/definition/bucket.json | 2 +- .../kibana/definition/match_operator.json | 2 +- .../functions/kibana/docs/match_operator.md | 9 +- .../common/settings/Setting.java | 2 +- .../xpack/esql/heap_attack/HeapAttackIT.java | 35 +++- .../esql/core/expression/Expression.java | 12 +- .../esql/core/expression/Expressions.java | 6 +- .../esql/core/expression/FoldContext.java | 178 ++++++++++++++++++ .../xpack/esql/core/expression/Foldables.java | 4 +- .../xpack/esql/core/expression/Literal.java | 6 +- .../esql/core/expression/TypeResolutions.java | 10 +- .../function/scalar/UnaryScalarFunction.java | 3 +- .../expression/predicate/BinaryPredicate.java | 5 +- .../esql/core/expression/predicate/Range.java | 22 +-- .../expression/predicate/logical/Not.java | 5 +- .../expression/predicate/nulls/IsNotNull.java | 5 +- .../expression/predicate/nulls/IsNull.java | 5 +- .../predicate/operator/arithmetic/Neg.java | 5 +- .../predicate/regex/RegexMatch.java | 3 +- .../core/expression/FoldContextTests.java | 97 ++++++++++ .../core/expression/predicate/RangeTests.java | 8 +- .../core/optimizer/OptimizerRulesTests.java | 5 +- .../xpack/esql/EsqlTestUtils.java | 6 + .../xpack/esql/analysis/Analyzer.java | 15 +- .../xpack/esql/evaluator/EvalMapper.java | 39 ++-- .../evaluator/mapper/EvaluatorMapper.java | 104 ++++++++-- .../evaluator/mapper/ExpressionMapper.java | 3 +- .../xpack/esql/execution/PlanExecutor.java | 4 +- .../function/aggregate/AggregateFunction.java | 4 +- .../function/aggregate/CountDistinct.java | 5 +- .../function/aggregate/Percentile.java | 3 +- .../expression/function/aggregate/Rate.java | 3 +- .../expression/function/aggregate/Top.java | 5 +- .../function/aggregate/WeightedAvg.java | 13 +- .../function/fulltext/FullTextFunction.java | 3 +- .../expression/function/fulltext/Match.java | 3 +- .../expression/function/grouping/Bucket.java | 21 ++- .../function/grouping/GroupingFunction.java | 5 +- .../function/scalar/EsqlScalarFunction.java | 5 +- .../function/scalar/conditional/Case.java | 8 +- .../convert/FoldablesConvertFunction.java | 5 +- .../function/scalar/date/DateDiff.java | 2 +- .../function/scalar/date/DateExtract.java | 9 +- .../function/scalar/date/DateFormat.java | 2 +- .../function/scalar/date/DateParse.java | 2 +- .../function/scalar/date/DateTrunc.java | 2 +- .../expression/function/scalar/date/Now.java | 3 +- .../expression/function/scalar/math/E.java | 3 +- .../expression/function/scalar/math/Pi.java | 3 +- .../expression/function/scalar/math/Tau.java | 3 +- .../function/scalar/multivalue/MvConcat.java | 5 +- .../multivalue/MvPSeriesWeightedSum.java | 2 +- .../function/scalar/multivalue/MvSlice.java | 4 +- .../function/scalar/multivalue/MvSort.java | 16 +- .../scalar/spatial/SpatialContains.java | 7 +- .../scalar/spatial/SpatialDisjoint.java | 7 +- .../spatial/SpatialEvaluatorFactory.java | 12 +- .../scalar/spatial/SpatialIntersects.java | 7 +- .../scalar/spatial/SpatialRelatesUtils.java | 22 ++- .../scalar/spatial/SpatialWithin.java | 7 +- .../function/scalar/spatial/StDistance.java | 11 +- .../function/scalar/string/Hash.java | 2 +- .../function/scalar/string/RLike.java | 5 +- .../function/scalar/string/Repeat.java | 2 +- .../function/scalar/string/Replace.java | 2 +- .../function/scalar/string/Space.java | 2 +- .../function/scalar/string/Split.java | 7 +- .../function/scalar/string/WildcardLike.java | 5 +- .../function/scalar/util/Delay.java | 9 +- .../DateTimeArithmeticOperation.java | 29 ++- .../arithmetic/EsqlArithmeticOperation.java | 5 +- .../predicate/operator/arithmetic/Neg.java | 9 +- .../comparison/EsqlBinaryComparison.java | 5 +- .../predicate/operator/comparison/In.java | 5 +- .../comparison/InsensitiveEquals.java | 7 +- .../comparison/InsensitiveEqualsMapper.java | 14 +- .../LocalLogicalOptimizerContext.java | 5 +- .../LocalPhysicalOptimizerContext.java | 3 +- .../optimizer/LogicalOptimizerContext.java | 15 +- .../BooleanFunctionEqualsElimination.java | 3 +- .../rules/logical/BooleanSimplification.java | 3 +- .../logical/CombineBinaryComparisons.java | 34 ++-- .../rules/logical/CombineDisjunctions.java | 7 +- .../rules/logical/ConstantFolding.java | 5 +- .../rules/logical/ConvertStringToByteRef.java | 4 +- .../optimizer/rules/logical/FoldNull.java | 3 +- .../rules/logical/LiteralsOnTheRight.java | 3 +- .../rules/logical/OptimizerRules.java | 19 +- .../rules/logical/PartiallyFoldCase.java | 5 +- .../rules/logical/PropagateEmptyRelation.java | 25 ++- .../rules/logical/PropagateEquals.java | 38 ++-- .../rules/logical/PropagateEvalFoldables.java | 9 +- .../rules/logical/PropagateNullable.java | 3 +- .../logical/PushDownAndCombineLimits.java | 19 +- .../rules/logical/ReplaceRegexMatch.java | 3 +- .../logical/ReplaceRowAsLocalRelation.java | 10 +- .../ReplaceStatsFilteredAggWithEval.java | 2 +- ...laceStringCasingWithInsensitiveEquals.java | 22 ++- .../SimplifyComparisonsArithmetics.java | 38 ++-- .../rules/logical/SkipQueryOnLimitZero.java | 11 +- .../logical/SplitInWithFoldableValue.java | 5 +- .../logical/SubstituteFilteredExpression.java | 3 +- .../logical/SubstituteSpatialSurrogates.java | 3 +- .../local/LocalPropagateEmptyRelation.java | 12 +- .../local/EnableSpatialDistancePushdown.java | 41 ++-- .../physical/local/PushTopNToSource.java | 17 +- .../xpack/esql/parser/ExpressionBuilder.java | 13 +- .../xpack/esql/parser/LogicalPlanBuilder.java | 7 +- .../xpack/esql/plan/logical/Enrich.java | 3 +- .../AbstractPhysicalOperationProviders.java | 12 +- .../planner/EsPhysicalOperationProviders.java | 9 +- .../planner/EsqlExpressionTranslators.java | 17 +- .../esql/planner/LocalExecutionPlanner.java | 23 +-- .../xpack/esql/planner/PlannerUtils.java | 16 +- .../xpack/esql/planner/TypeConverter.java | 36 ++-- .../esql/planner/mapper/MapperUtils.java | 3 +- .../xpack/esql/plugin/ComputeService.java | 32 +++- .../xpack/esql/plugin/QueryPragmas.java | 15 ++ .../esql/plugin/TransportEsqlQueryAction.java | 4 + .../xpack/esql/session/Configuration.java | 8 + .../xpack/esql/session/EsqlSession.java | 8 +- .../esql/type/EsqlDataTypeConverter.java | 5 +- .../elasticsearch/xpack/esql/CsvTests.java | 35 ++-- .../xpack/esql/analysis/AnalyzerTests.java | 22 +-- .../mapper/EvaluatorMapperTests.java | 43 +++++ .../function/AbstractAggregationTestCase.java | 8 +- .../function/AbstractFunctionTestCase.java | 23 ++- .../AbstractScalarFunctionTestCase.java | 10 +- .../function/CheckLicenseTests.java | 5 +- .../expression/function/TestCaseSupplier.java | 6 +- .../scalar/conditional/CaseExtraTests.java | 61 +++--- .../scalar/conditional/CaseTests.java | 9 +- .../scalar/date/DateExtractTests.java | 3 +- .../function/scalar/nulls/CoalesceTests.java | 33 ++-- .../function/scalar/string/RLikeTests.java | 5 +- .../function/scalar/string/ToLowerTests.java | 3 +- .../function/scalar/string/ToUpperTests.java | 3 +- .../scalar/string/WildcardLikeTests.java | 5 +- .../operator/arithmetic/NegTests.java | 3 +- .../operator/comparison/InTests.java | 13 +- .../comparison/InsensitiveEqualsTests.java | 57 +++--- .../LocalLogicalPlanOptimizerTests.java | 16 +- .../LocalPhysicalPlanOptimizerTests.java | 40 ++-- .../optimizer/LogicalPlanOptimizerTests.java | 176 ++++++++--------- .../optimizer/PhysicalPlanOptimizerTests.java | 54 +++--- .../esql/optimizer/TestPlannerOptimizer.java | 12 +- .../rules/LogicalOptimizerContextTests.java | 62 ++++++ ...BooleanFunctionEqualsEliminationTests.java | 13 +- .../logical/BooleanSimplificationTests.java | 38 ++-- .../CombineBinaryComparisonsTests.java | 76 +++----- .../logical/CombineDisjunctionsTests.java | 30 +-- .../rules/logical/ConstantFoldingTests.java | 79 ++++---- .../rules/logical/FoldNullTests.java | 122 ++++++------ .../logical/LiteralsOnTheRightTests.java | 3 +- .../rules/logical/PropagateEqualsTests.java | 77 +++----- .../rules/logical/PropagateNullableTests.java | 27 ++- .../rules/logical/ReplaceRegexMatchTests.java | 18 +- .../physical/local/PushTopNToSourceTests.java | 3 +- .../xpack/esql/parser/ExpressionTests.java | 11 +- .../esql/parser/StatementParserTests.java | 39 ++-- .../xpack/esql/plan/QueryPlanTests.java | 23 +-- .../xpack/esql/planner/EvalMapperTests.java | 3 +- .../xpack/esql/planner/FilterTests.java | 4 +- .../planner/LocalExecutionPlannerTests.java | 6 +- .../TestPhysicalOperationProviders.java | 9 +- .../esql/plugin/ClusterRequestTests.java | 5 +- .../DataNodeRequestSerializationTests.java | 3 +- .../esql/stats/PlanExecutorMetricsTests.java | 3 + 170 files changed, 1858 insertions(+), 972 deletions(-) create mode 100644 docs/changelog/118602.yaml create mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FoldContext.java create mode 100644 x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FoldContextTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapperTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/LogicalOptimizerContextTests.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java index 9aab4a3e3210f..d3259b9604717 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java @@ -27,6 +27,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -71,12 +72,11 @@ public class EvalBenchmark { BigArrays.NON_RECYCLING_INSTANCE ); + private static final FoldContext FOLD_CONTEXT = FoldContext.small(); + private static final int BLOCK_LENGTH = 8 * 1024; - static final DriverContext driverContext = new DriverContext( - BigArrays.NON_RECYCLING_INSTANCE, - BlockFactory.getInstance(new NoopCircuitBreaker("noop"), BigArrays.NON_RECYCLING_INSTANCE) - ); + static final DriverContext driverContext = new DriverContext(BigArrays.NON_RECYCLING_INSTANCE, blockFactory); static { // Smoke test all the expected values and force loading subclasses more like prod @@ -114,11 +114,12 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) { return switch (operation) { case "abs" -> { FieldAttribute longField = longField(); - yield EvalMapper.toEvaluator(new Abs(Source.EMPTY, longField), layout(longField)).get(driverContext); + yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Abs(Source.EMPTY, longField), layout(longField)).get(driverContext); } case "add" -> { FieldAttribute longField = longField(); yield EvalMapper.toEvaluator( + FOLD_CONTEXT, new Add(Source.EMPTY, longField, new Literal(Source.EMPTY, 1L, DataType.LONG)), layout(longField) ).get(driverContext); @@ -126,6 +127,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) { case "add_double" -> { FieldAttribute doubleField = doubleField(); yield EvalMapper.toEvaluator( + FOLD_CONTEXT, new Add(Source.EMPTY, doubleField, new Literal(Source.EMPTY, 1D, DataType.DOUBLE)), layout(doubleField) ).get(driverContext); @@ -140,7 +142,8 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) { lhs = new Add(Source.EMPTY, lhs, new Literal(Source.EMPTY, 1L, DataType.LONG)); rhs = new Add(Source.EMPTY, rhs, new Literal(Source.EMPTY, 1L, DataType.LONG)); } - yield EvalMapper.toEvaluator(new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2)).get(driverContext); + yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2)) + .get(driverContext); } case "date_trunc" -> { FieldAttribute timestamp = new FieldAttribute( @@ -149,6 +152,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) { new EsField("timestamp", DataType.DATETIME, Map.of(), true) ); yield EvalMapper.toEvaluator( + FOLD_CONTEXT, new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION), timestamp), layout(timestamp) ).get(driverContext); @@ -156,6 +160,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) { case "equal_to_const" -> { FieldAttribute longField = longField(); yield EvalMapper.toEvaluator( + FOLD_CONTEXT, new Equals(Source.EMPTY, longField, new Literal(Source.EMPTY, 100_000L, DataType.LONG)), layout(longField) ).get(driverContext); @@ -163,21 +168,21 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) { case "long_equal_to_long" -> { FieldAttribute lhs = longField(); FieldAttribute rhs = longField(); - yield EvalMapper.toEvaluator(new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext); + yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext); } case "long_equal_to_int" -> { FieldAttribute lhs = longField(); FieldAttribute rhs = intField(); - yield EvalMapper.toEvaluator(new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext); + yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext); } case "mv_min", "mv_min_ascending" -> { FieldAttribute longField = longField(); - yield EvalMapper.toEvaluator(new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext); + yield EvalMapper.toEvaluator(FOLD_CONTEXT, new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext); } case "rlike" -> { FieldAttribute keywordField = keywordField(); RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar")); - yield EvalMapper.toEvaluator(rlike, layout(keywordField)).get(driverContext); + yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext); } default -> throw new UnsupportedOperationException(); }; diff --git a/docs/changelog/118602.yaml b/docs/changelog/118602.yaml new file mode 100644 index 0000000000000..a75c5dcf11da3 --- /dev/null +++ b/docs/changelog/118602.yaml @@ -0,0 +1,5 @@ +pr: 118602 +summary: Limit memory usage of `fold` +area: ES|QL +type: bug +issues: [] diff --git a/docs/reference/esql/functions/kibana/definition/bucket.json b/docs/reference/esql/functions/kibana/definition/bucket.json index 3d96de05c8407..f9c7f2f27d6f9 100644 --- a/docs/reference/esql/functions/kibana/definition/bucket.json +++ b/docs/reference/esql/functions/kibana/definition/bucket.json @@ -1599,7 +1599,7 @@ "FROM sample_data \n| WHERE @timestamp >= NOW() - 1 day and @timestamp < NOW()\n| STATS COUNT(*) BY bucket = BUCKET(@timestamp, 25, NOW() - 1 day, NOW())", "FROM employees\n| WHERE hire_date >= \"1985-01-01T00:00:00Z\" AND hire_date < \"1986-01-01T00:00:00Z\"\n| STATS AVG(salary) BY bucket = BUCKET(hire_date, 20, \"1985-01-01T00:00:00Z\", \"1986-01-01T00:00:00Z\")\n| SORT bucket", "FROM employees\n| STATS s1 = b1 + 1, s2 = BUCKET(salary / 1000 + 999, 50.) + 2 BY b1 = BUCKET(salary / 100 + 99, 50.), b2 = BUCKET(salary / 1000 + 999, 50.)\n| SORT b1, b2\n| KEEP s1, b1, s2, b2", - "FROM employees \n| STATS dates = VALUES(birth_date) BY b = BUCKET(birth_date + 1 HOUR, 1 YEAR) - 1 HOUR\n| EVAL d_count = MV_COUNT(dates)\n| SORT d_count\n| LIMIT 3" + "FROM employees\n| STATS dates = MV_SORT(VALUES(birth_date)) BY b = BUCKET(birth_date + 1 HOUR, 1 YEAR) - 1 HOUR\n| EVAL d_count = MV_COUNT(dates)\n| SORT d_count, b\n| LIMIT 3" ], "preview" : false, "snapshot_only" : false diff --git a/docs/reference/esql/functions/kibana/definition/match_operator.json b/docs/reference/esql/functions/kibana/definition/match_operator.json index 44233bbddb653..c8cbf1cf9d966 100644 --- a/docs/reference/esql/functions/kibana/definition/match_operator.json +++ b/docs/reference/esql/functions/kibana/definition/match_operator.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "operator", "name" : "match_operator", - "description" : "Performs a <> on the specified field. Returns true if the provided query matches the row.", + "description" : "Use `MATCH` to perform a <> on the specified field.\nUsing `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.\n\nMatch can be used on text fields, as well as other field types like boolean, dates, and numeric types.\n\nFor a simplified syntax, you can use the <> `:` operator instead of `MATCH`.\n\n`MATCH` returns true if the provided query matches the row.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/docs/match_operator.md b/docs/reference/esql/functions/kibana/docs/match_operator.md index b0b6196798087..7681c2d1ce231 100644 --- a/docs/reference/esql/functions/kibana/docs/match_operator.md +++ b/docs/reference/esql/functions/kibana/docs/match_operator.md @@ -3,7 +3,14 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MATCH_OPERATOR -Performs a <> on the specified field. Returns true if the provided query matches the row. +Use `MATCH` to perform a <> on the specified field. +Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. + +Match can be used on text fields, as well as other field types like boolean, dates, and numeric types. + +For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. + +`MATCH` returns true if the provided query matches the row. ``` FROM books diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index aec9c108d898d..16c6844f46402 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -1727,7 +1727,7 @@ public static > Setting enumSetting( * * @param key the key for the setting * @param defaultValue the default value for this setting - * @param properties properties properties for this setting like scope, filtering... + * @param properties properties for this setting like scope, filtering... * @return the setting object */ public static Setting memorySizeSetting(String key, ByteSizeValue defaultValue, Property... properties) { diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index ace3db377664c..958132b3e4076 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -194,6 +194,16 @@ private void assertCircuitBreaks(ThrowingRunnable r) throws IOException { ); } + private void assertFoldCircuitBreaks(ThrowingRunnable r) throws IOException { + ResponseException e = expectThrows(ResponseException.class, r); + Map map = responseAsMap(e.getResponse()); + logger.info("expected fold circuit breaking {}", map); + assertMap( + map, + matchesMap().entry("status", 400).entry("error", matchesMap().extraOk().entry("type", "fold_too_much_memory_exception")) + ); + } + private void assertParseFailure(ThrowingRunnable r) throws IOException { ResponseException e = expectThrows(ResponseException.class, r); Map map = responseAsMap(e.getResponse()); @@ -325,11 +335,23 @@ public void testManyConcatFromRow() throws IOException { assertManyStrings(resp, strings); } + /** + * Hits a circuit breaker by building many moderately long strings. + */ + public void testHugeManyConcatFromRow() throws IOException { + assertFoldCircuitBreaks( + () -> manyConcat( + "ROW a=9999999999999, b=99999999999999999, c=99999999999999999, d=99999999999999999, e=99999999999999999", + 5000 + ) + ); + } + /** * Fails to parse a huge huge query. */ public void testHugeHugeManyConcatFromRow() throws IOException { - assertParseFailure(() -> manyConcat("ROW a=9999, b=9999, c=9999, d=9999, e=9999", 50000)); + assertParseFailure(() -> manyConcat("ROW a=9999, b=9999, c=9999, d=9999, e=9999", 6000)); } /** @@ -387,13 +409,20 @@ public void testHugeManyRepeat() throws IOException { * Returns many moderately long strings. */ public void testManyRepeatFromRow() throws IOException { - int strings = 10000; + int strings = 300; Response resp = manyRepeat("ROW a = 99", strings); assertManyStrings(resp, strings); } /** - * Fails to parse a huge huge query. + * Hits a circuit breaker by building many moderately long strings. + */ + public void testHugeManyRepeatFromRow() throws IOException { + assertFoldCircuitBreaks(() -> manyRepeat("ROW a = 99", 400)); + } + + /** + * Fails to parse a huge, huge query. */ public void testHugeHugeManyRepeatFromRow() throws IOException { assertParseFailure(() -> manyRepeat("ROW a = 99", 100000)); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java index 00765a8c0528c..b254612a700df 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java @@ -78,12 +78,20 @@ public Expression(Source source, List children) { super(source, children); } - // whether the expression can be evaluated statically (folded) or not + /** + * Whether the expression can be evaluated statically, aka "folded", or not. + */ public boolean foldable() { return false; } - public Object fold() { + /** + * Evaluate this expression statically to a constant. It is an error to call + * this if {@link #foldable} returns false. + */ + public Object fold(FoldContext ctx) { + // TODO After removing FoldContext.unbounded from non-test code examine all calls + // for places we should use instanceof Literal instead throw new QlIllegalArgumentException("Should not fold expression"); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java index 4e4338aad3704..739333ded0fde 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java @@ -107,10 +107,10 @@ public static boolean foldable(List exps) { return true; } - public static List fold(List exps) { + public static List fold(FoldContext ctx, List exps) { List folded = new ArrayList<>(exps.size()); for (Expression exp : exps) { - folded.add(exp.fold()); + folded.add(exp.fold(ctx)); } return folded; @@ -135,7 +135,7 @@ public static String name(Expression e) { /** * Is this {@linkplain Expression} guaranteed to have * only the {@code null} value. {@linkplain Expression}s that - * {@link Expression#fold()} to {@code null} may + * {@link Expression#fold} to {@code null} may * return {@code false} here, but should eventually be folded * into a {@link Literal} containing {@code null} which will return * {@code true} from here. diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FoldContext.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FoldContext.java new file mode 100644 index 0000000000000..25da44c5fd226 --- /dev/null +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FoldContext.java @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.expression; + +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.MemorySizeValue; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.xpack.esql.core.QlClientException; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import java.util.Objects; + +/** + * Context passed to {@link Expression#fold}. This is not thread safe. + */ +public class FoldContext { + private static final long SMALL = MemorySizeValue.parseBytesSizeValueOrHeapRatio("5%", "small").getBytes(); + + /** + * {@link Expression#fold} using less than 5% of heap. Fine in tests but otherwise + * calling this is a signal that you either, shouldn't be calling {@link Expression#fold} + * at all, or should pass in a shared {@link FoldContext} made by {@code Configuration}. + */ + public static FoldContext small() { + return new FoldContext(SMALL); + } + + private final long initialAllowedBytes; + private long allowedBytes; + + public FoldContext(long allowedBytes) { + this.initialAllowedBytes = allowedBytes; + this.allowedBytes = allowedBytes; + } + + /** + * The maximum allowed bytes. {@link #allowedBytes()} will be the same as this + * for an unused context. + */ + public long initialAllowedBytes() { + return initialAllowedBytes; + } + + /** + * The remaining allowed bytes. + */ + long allowedBytes() { + return allowedBytes; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + FoldContext that = (FoldContext) o; + return initialAllowedBytes == that.initialAllowedBytes && allowedBytes == that.allowedBytes; + } + + @Override + public int hashCode() { + return Objects.hash(initialAllowedBytes, allowedBytes); + } + + @Override + public String toString() { + return "FoldContext[" + allowedBytes + '/' + initialAllowedBytes + ']'; + } + + /** + * Track an allocation. Best to call this before allocating + * if possible, but after is ok if the allocation is small. + *

+ * Note that, unlike {@link CircuitBreaker}, you don't have + * to free this allocation later. This is important because the query plan + * doesn't implement {@link Releasable} so it can't free + * consistently. But when you have to allocate big chunks of memory during + * folding and know that you are returning the memory it is kindest to + * call this with a negative number, effectively giving those bytes back. + *

+ */ + public void trackAllocation(Source source, long bytes) { + allowedBytes -= bytes; + assert allowedBytes <= initialAllowedBytes : "returned more bytes than it used"; + if (allowedBytes < 0) { + throw new FoldTooMuchMemoryException(source, bytes, initialAllowedBytes); + } + } + + /** + * Adapt this into a {@link CircuitBreaker} suitable for building bounded local + * DriverContext. This is absolutely an abuse of the {@link CircuitBreaker} contract + * and only methods used by BlockFactory are implemented. And this'll throw a + * {@link FoldTooMuchMemoryException} instead of the standard {@link CircuitBreakingException}. + * This works for the common folding implementation though. + */ + public CircuitBreaker circuitBreakerView(Source source) { + return new CircuitBreaker() { + @Override + public void circuitBreak(String fieldName, long bytesNeeded) { + throw new UnsupportedOperationException(); + } + + @Override + public void addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException { + trackAllocation(source, bytes); + } + + @Override + public void addWithoutBreaking(long bytes) { + assert bytes <= 0 : "we only expect this to be used for deallocation"; + allowedBytes -= bytes; + assert allowedBytes <= initialAllowedBytes : "returned more bytes than it used"; + } + + @Override + public long getUsed() { + /* + * This isn't expected to be used by we can implement it so we may as + * well. Maybe it'll be useful for debugging one day. + */ + return initialAllowedBytes - allowedBytes; + } + + @Override + public long getLimit() { + /* + * This isn't expected to be used by we can implement it so we may as + * well. Maybe it'll be useful for debugging one day. + */ + return initialAllowedBytes; + } + + @Override + public double getOverhead() { + return 1.0; + } + + @Override + public long getTrippedCount() { + return 0; + } + + @Override + public String getName() { + return REQUEST; + } + + @Override + public Durability getDurability() { + throw new UnsupportedOperationException(); + } + + @Override + public void setLimitAndOverhead(long limit, double overhead) { + throw new UnsupportedOperationException(); + } + }; + } + + public static class FoldTooMuchMemoryException extends QlClientException { + protected FoldTooMuchMemoryException(Source source, long bytesForExpression, long initialAllowedBytes) { + super( + "line {}:{}: Folding query used more than {}. The expression that pushed past the limit is [{}] which needed {}.", + source.source().getLineNumber(), + source.source().getColumnNumber(), + ByteSizeValue.ofBytes(initialAllowedBytes), + source.text(), + ByteSizeValue.ofBytes(bytesForExpression) + ); + } + } +} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Foldables.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Foldables.java index 601758bca5918..233113c3fe1b8 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Foldables.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Foldables.java @@ -10,9 +10,9 @@ public abstract class Foldables { - public static Object valueOf(Expression e) { + public static Object valueOf(FoldContext ctx, Expression e) { if (e.foldable()) { - return e.fold(); + return e.fold(ctx); } throw new QlIllegalArgumentException("Cannot determine value for {}", e); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java index 53f559c5c82fe..afe616489d81d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java @@ -98,7 +98,7 @@ public boolean resolved() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { return value; } @@ -138,7 +138,7 @@ public String nodeString() { * Utility method for creating a literal out of a foldable expression. * Throws an exception if the expression is not foldable. */ - public static Literal of(Expression foldable) { + public static Literal of(FoldContext ctx, Expression foldable) { if (foldable.foldable() == false) { throw new QlIllegalArgumentException("Foldable expression required for Literal creation; received unfoldable " + foldable); } @@ -147,7 +147,7 @@ public static Literal of(Expression foldable) { return (Literal) foldable; } - return new Literal(foldable.source(), foldable.fold(), foldable.dataType()); + return new Literal(foldable.source(), foldable.fold(ctx), foldable.dataType()); } public static Literal of(Expression source, Object value) { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java index b817ec17c7bda..842f3c0ddadd7 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java @@ -133,6 +133,14 @@ public static TypeResolution isFoldable(Expression e, String operationName, Para return TypeResolution.TYPE_RESOLVED; } + /** + * Is this {@link Expression#foldable()} and not {@code null}. + * + * @deprecated instead of calling this, check for a {@link Literal} containing + * {@code null}. Foldable expressions will be folded by other rules, + * eventually, to a {@link Literal}. + */ + @Deprecated public static TypeResolution isNotNullAndFoldable(Expression e, String operationName, ParamOrdinal paramOrd) { TypeResolution resolution = isFoldable(e, operationName, paramOrd); @@ -140,7 +148,7 @@ public static TypeResolution isNotNullAndFoldable(Expression e, String operation return resolution; } - if (e.dataType() == DataType.NULL || e.fold() == null) { + if (e.dataType() == DataType.NULL || e.fold(FoldContext.small()) == null) { resolution = new TypeResolution( format( null, diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java index 8704a42ed33e2..36517b1be9ce7 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; @@ -53,5 +54,5 @@ public boolean foldable() { } @Override - public abstract Object fold(); + public abstract Object fold(FoldContext ctx); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/BinaryPredicate.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/BinaryPredicate.java index be5caedacd50a..bf5549b31e5fa 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/BinaryPredicate.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/BinaryPredicate.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.core.expression.predicate; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -29,8 +30,8 @@ protected BinaryPredicate(Source source, Expression left, Expression right, F fu @SuppressWarnings("unchecked") @Override - public R fold() { - return function().apply((T) left().fold(), (U) right().fold()); + public R fold(FoldContext ctx) { + return function().apply((T) left().fold(ctx), (U) right().fold(ctx)); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java index 5de09f40437c7..a4e4685f764e8 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java @@ -8,6 +8,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; @@ -99,23 +100,24 @@ public boolean foldable() { } // We cannot fold the bounds here; but if they're already literals, we can check if the range is always empty. - if (lower() instanceof Literal && upper() instanceof Literal) { - return areBoundariesInvalid(); + if (lower() instanceof Literal l && upper() instanceof Literal u) { + return areBoundariesInvalid(l.value(), u.value()); } } - return false; } @Override - public Object fold() { - if (areBoundariesInvalid()) { + public Object fold(FoldContext ctx) { + Object lowerValue = lower.fold(ctx); + Object upperValue = upper.fold(ctx); + if (areBoundariesInvalid(lowerValue, upperValue)) { return Boolean.FALSE; } - Object val = value.fold(); - Integer lowerCompare = BinaryComparison.compare(lower.fold(), val); - Integer upperCompare = BinaryComparison.compare(val, upper().fold()); + Object val = value.fold(ctx); + Integer lowerCompare = BinaryComparison.compare(lower.fold(ctx), val); + Integer upperCompare = BinaryComparison.compare(val, upper().fold(ctx)); boolean lowerComparsion = lowerCompare == null ? false : (includeLower ? lowerCompare <= 0 : lowerCompare < 0); boolean upperComparsion = upperCompare == null ? false : (includeUpper ? upperCompare <= 0 : upperCompare < 0); return lowerComparsion && upperComparsion; @@ -125,9 +127,7 @@ public Object fold() { * Check whether the boundaries are invalid ( upper < lower) or not. * If they are, the value does not have to be evaluated. */ - protected boolean areBoundariesInvalid() { - Object lowerValue = lower.fold(); - Object upperValue = upper.fold(); + protected boolean areBoundariesInvalid(Object lowerValue, Object upperValue) { if (DataType.isDateTime(value.dataType()) || DataType.isDateTime(lower.dataType()) || DataType.isDateTime(upper.dataType())) { try { if (upperValue instanceof String upperString) { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java index c4983b49a6bc8..218f61856accc 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -56,8 +57,8 @@ protected TypeResolution resolveType() { } @Override - public Object fold() { - return apply(field().fold()); + public Object fold(FoldContext ctx) { + return apply(field().fold(ctx)); } private static Boolean apply(Object input) { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java index 9879a1f5ffc29..f5542ff7c3de5 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -49,8 +50,8 @@ protected IsNotNull replaceChild(Expression newChild) { } @Override - public Object fold() { - return field().fold() != null && DataType.isNull(field().dataType()) == false; + public Object fold(FoldContext ctx) { + return DataType.isNull(field().dataType()) == false && field().fold(ctx) != null; } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java index d88945045b03e..bb85791a9f85e 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -45,8 +46,8 @@ protected IsNull replaceChild(Expression newChild) { } @Override - public Object fold() { - return field().fold() == null || DataType.isNull(field().dataType()); + public Object fold(FoldContext ctx) { + return DataType.isNull(field().dataType()) || field().fold(ctx) == null; } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Neg.java index 9a8a14f320cd6..b0e79704f5fda 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Neg.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Neg.java @@ -8,6 +8,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -53,8 +54,8 @@ protected TypeResolution resolveType() { } @Override - public Object fold() { - return Arithmetics.negate((Number) field().fold()); + public Object fold(FoldContext ctx) { + return Arithmetics.negate((Number) field().fold(ctx)); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java index 0f9116ade5a31..a4a0a6217161e 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.core.expression.predicate.regex; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -62,7 +63,7 @@ public boolean foldable() { } @Override - public Boolean fold() { + public Boolean fold(FoldContext ctx) { throw new UnsupportedOperationException(); } diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FoldContextTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FoldContextTests.java new file mode 100644 index 0000000000000..2080f4007777c --- /dev/null +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FoldContextTests.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.expression; + +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import static org.hamcrest.Matchers.equalTo; + +public class FoldContextTests extends ESTestCase { + public void testEq() { + EqualsHashCodeTestUtils.checkEqualsAndHashCode(randomFoldContext(), this::copy, this::mutate); + } + + private FoldContext randomFoldContext() { + FoldContext ctx = new FoldContext(randomNonNegativeLong()); + if (randomBoolean()) { + ctx.trackAllocation(Source.EMPTY, randomLongBetween(0, ctx.initialAllowedBytes())); + } + return ctx; + } + + private FoldContext copy(FoldContext ctx) { + FoldContext copy = new FoldContext(ctx.initialAllowedBytes()); + copy.trackAllocation(Source.EMPTY, ctx.initialAllowedBytes() - ctx.allowedBytes()); + return copy; + } + + private FoldContext mutate(FoldContext ctx) { + if (randomBoolean()) { + FoldContext differentInitial = new FoldContext(ctx.initialAllowedBytes() + 1); + differentInitial.trackAllocation(Source.EMPTY, differentInitial.initialAllowedBytes() - ctx.allowedBytes()); + assertThat(differentInitial.allowedBytes(), equalTo(ctx.allowedBytes())); + return differentInitial; + } else { + FoldContext differentAllowed = new FoldContext(ctx.initialAllowedBytes()); + long allowed = randomValueOtherThan(ctx.allowedBytes(), () -> randomLongBetween(0, ctx.initialAllowedBytes())); + differentAllowed.trackAllocation(Source.EMPTY, ctx.initialAllowedBytes() - allowed); + assertThat(differentAllowed.allowedBytes(), equalTo(allowed)); + return differentAllowed; + } + } + + public void testTrackAllocation() { + FoldContext ctx = new FoldContext(10); + ctx.trackAllocation(Source.synthetic("shouldn't break"), 10); + Exception e = expectThrows( + FoldContext.FoldTooMuchMemoryException.class, + () -> ctx.trackAllocation(Source.synthetic("should break"), 1) + ); + assertThat( + e.getMessage(), + equalTo( + "line -1:-1: Folding query used more than 10b. " + + "The expression that pushed past the limit is [should break] which needed 1b." + ) + ); + } + + public void testCircuitBreakerViewBreaking() { + FoldContext ctx = new FoldContext(10); + ctx.circuitBreakerView(Source.synthetic("shouldn't break")).addEstimateBytesAndMaybeBreak(10, "test"); + Exception e = expectThrows( + FoldContext.FoldTooMuchMemoryException.class, + () -> ctx.circuitBreakerView(Source.synthetic("should break")).addEstimateBytesAndMaybeBreak(1, "test") + ); + assertThat( + e.getMessage(), + equalTo( + "line -1:-1: Folding query used more than 10b. " + + "The expression that pushed past the limit is [should break] which needed 1b." + ) + ); + } + + public void testCircuitBreakerViewWithoutBreaking() { + FoldContext ctx = new FoldContext(10); + CircuitBreaker view = ctx.circuitBreakerView(Source.synthetic("shouldn't break")); + view.addEstimateBytesAndMaybeBreak(10, "test"); + view.addWithoutBreaking(-1); + assertThat(view.getUsed(), equalTo(9L)); + } + + public void testToString() { + // Random looking numbers are indeed random. Just so we have consistent numbers to assert on in toString. + FoldContext ctx = new FoldContext(123); + ctx.trackAllocation(Source.EMPTY, 22); + assertThat(ctx.toString(), equalTo("FoldContext[101/123]")); + } +} diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java index ed4c6282368ca..cd15ed5a94cfc 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.core.expression.predicate; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.DateUtils; @@ -211,7 +212,11 @@ public void testAreBoundariesInvalid() { (Boolean) test[7], ZoneId.systemDefault() ); - assertEquals("failed on test " + i + ": " + Arrays.toString(test), test[8], range.areBoundariesInvalid()); + assertEquals( + "failed on test " + i + ": " + Arrays.toString(test), + test[8], + range.areBoundariesInvalid(range.lower().fold(FoldContext.small()), range.upper().fold(FoldContext.small())) + ); } } @@ -226,5 +231,4 @@ private static DataType randomNumericType() { private static DataType randomTextType() { return randomFrom(KEYWORD, TEXT); } - } diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java index d323174d2d3d9..91b0564a5b404 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.predicate.Range; @@ -104,7 +105,7 @@ public void testFoldExcludingRangeToFalse() { Range r = rangeOf(fa, SIX, false, FIVE, true); assertTrue(r.foldable()); - assertEquals(Boolean.FALSE, r.fold()); + assertEquals(Boolean.FALSE, r.fold(FoldContext.small())); } // 6 < a <= 5.5 -> FALSE @@ -113,7 +114,7 @@ public void testFoldExcludingRangeWithDifferentTypesToFalse() { Range r = rangeOf(fa, SIX, false, L(5.5d), true); assertTrue(r.foldable()); - assertEquals(Boolean.FALSE, r.fold()); + assertEquals(Boolean.FALSE, r.fold(FoldContext.small())); } // Conjunction diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 66fd7d3ee5eb5..7e25fb29fdb78 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.Range; @@ -61,6 +62,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.parser.QueryParam; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; @@ -350,6 +352,10 @@ public String toString() { public static final Configuration TEST_CFG = configuration(new QueryPragmas(Settings.EMPTY)); + public static LogicalOptimizerContext unboundLogicalOptimizerContext() { + return new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG, FoldContext.small()); + } + public static final Verifier TEST_VERIFIER = new Verifier(new Metrics(new EsqlFunctionRegistry()), new XPackLicenseState(() -> 0L)); public static final QueryBuilderResolver MOCK_QUERY_BUILDER_RESOLVER = new MockQueryBuilderResolver(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 3d1bfdfd0ef42..a11b511cb83b7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.Nullability; @@ -325,7 +326,7 @@ protected LogicalPlan rule(Enrich plan, AnalyzerContext context) { // the policy does not exist return plan; } - final String policyName = (String) plan.policyName().fold(); + final String policyName = (String) plan.policyName().fold(FoldContext.small() /* TODO remove me */); final var resolved = context.enrichResolution().getResolvedPolicy(policyName, plan.mode()); if (resolved != null) { var policy = new EnrichPolicy(resolved.matchType(), null, List.of(), resolved.matchField(), resolved.enrichFields()); @@ -1279,16 +1280,16 @@ private static boolean supportsStringImplicitCasting(DataType type) { private static UnresolvedAttribute unresolvedAttribute(Expression value, String type, Exception e) { String message = format( "Cannot convert string [{}] to [{}], error [{}]", - value.fold(), + value.fold(FoldContext.small() /* TODO remove me */), type, (e instanceof ParsingException pe) ? pe.getErrorMessage() : e.getMessage() ); - return new UnresolvedAttribute(value.source(), String.valueOf(value.fold()), message); + return new UnresolvedAttribute(value.source(), String.valueOf(value.fold(FoldContext.small() /* TODO remove me */)), message); } private static Expression castStringLiteralToTemporalAmount(Expression from) { try { - TemporalAmount result = maybeParseTemporalAmount(from.fold().toString().strip()); + TemporalAmount result = maybeParseTemporalAmount(from.fold(FoldContext.small() /* TODO remove me */).toString().strip()); if (result == null) { return from; } @@ -1304,7 +1305,11 @@ private static Expression castStringLiteral(Expression from, DataType target) { try { return isTemporalAmount(target) ? castStringLiteralToTemporalAmount(from) - : new Literal(from.source(), EsqlDataTypeConverter.convert(from.fold(), target), target); + : new Literal( + from.source(), + EsqlDataTypeConverter.convert(from.fold(FoldContext.small() /* TODO remove me */), target), + target + ); } catch (Exception e) { return unresolvedAttribute(from, target.toString(), e); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java index 9a2e9398f52fd..b9c2b92ea72dd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; @@ -50,13 +51,23 @@ public final class EvalMapper { private EvalMapper() {} @SuppressWarnings({ "rawtypes", "unchecked" }) - public static ExpressionEvaluator.Factory toEvaluator(Expression exp, Layout layout) { + public static ExpressionEvaluator.Factory toEvaluator(FoldContext foldCtx, Expression exp, Layout layout) { if (exp instanceof EvaluatorMapper m) { - return m.toEvaluator(e -> toEvaluator(e, layout)); + return m.toEvaluator(new EvaluatorMapper.ToEvaluator() { + @Override + public ExpressionEvaluator.Factory apply(Expression expression) { + return toEvaluator(foldCtx, expression, layout); + } + + @Override + public FoldContext foldCtx() { + return foldCtx; + } + }); } for (ExpressionMapper em : MAPPERS) { if (em.typeToken.isInstance(exp)) { - return em.map(exp, layout); + return em.map(foldCtx, exp, layout); } } throw new QlIllegalArgumentException("Unsupported expression [{}]", exp); @@ -64,9 +75,9 @@ public static ExpressionEvaluator.Factory toEvaluator(Expression exp, Layout lay static class BooleanLogic extends ExpressionMapper { @Override - public ExpressionEvaluator.Factory map(BinaryLogic bc, Layout layout) { - var leftEval = toEvaluator(bc.left(), layout); - var rightEval = toEvaluator(bc.right(), layout); + public ExpressionEvaluator.Factory map(FoldContext foldCtx, BinaryLogic bc, Layout layout) { + var leftEval = toEvaluator(foldCtx, bc.left(), layout); + var rightEval = toEvaluator(foldCtx, bc.right(), layout); /** * Evaluator for the three-valued boolean expressions. * We can't generate these with the {@link Evaluator} annotation because that @@ -142,8 +153,8 @@ public void close() { static class Nots extends ExpressionMapper { @Override - public ExpressionEvaluator.Factory map(Not not, Layout layout) { - var expEval = toEvaluator(not.field(), layout); + public ExpressionEvaluator.Factory map(FoldContext foldCtx, Not not, Layout layout) { + var expEval = toEvaluator(foldCtx, not.field(), layout); return dvrCtx -> new org.elasticsearch.xpack.esql.evaluator.predicate.operator.logical.NotEvaluator( not.source(), expEval.get(dvrCtx), @@ -154,7 +165,7 @@ public ExpressionEvaluator.Factory map(Not not, Layout layout) { static class Attributes extends ExpressionMapper { @Override - public ExpressionEvaluator.Factory map(Attribute attr, Layout layout) { + public ExpressionEvaluator.Factory map(FoldContext foldCtx, Attribute attr, Layout layout) { record Attribute(int channel) implements ExpressionEvaluator { @Override public Block eval(Page page) { @@ -189,7 +200,7 @@ public boolean eagerEvalSafeInLazy() { static class Literals extends ExpressionMapper { @Override - public ExpressionEvaluator.Factory map(Literal lit, Layout layout) { + public ExpressionEvaluator.Factory map(FoldContext foldCtx, Literal lit, Layout layout) { record LiteralsEvaluator(DriverContext context, Literal lit) implements ExpressionEvaluator { @Override public Block eval(Page page) { @@ -246,8 +257,8 @@ private static Block block(Literal lit, BlockFactory blockFactory, int positions static class IsNulls extends ExpressionMapper { @Override - public ExpressionEvaluator.Factory map(IsNull isNull, Layout layout) { - var field = toEvaluator(isNull.field(), layout); + public ExpressionEvaluator.Factory map(FoldContext foldCtx, IsNull isNull, Layout layout) { + var field = toEvaluator(foldCtx, isNull.field(), layout); return new IsNullEvaluatorFactory(field); } @@ -294,8 +305,8 @@ public String toString() { static class IsNotNulls extends ExpressionMapper { @Override - public ExpressionEvaluator.Factory map(IsNotNull isNotNull, Layout layout) { - return new IsNotNullEvaluatorFactory(toEvaluator(isNotNull.field(), layout)); + public ExpressionEvaluator.Factory map(FoldContext foldCtx, IsNotNull isNotNull, Layout layout) { + return new IsNotNullEvaluatorFactory(toEvaluator(foldCtx, isNotNull.field(), layout)); } record IsNotNullEvaluatorFactory(EvalOperator.ExpressionEvaluator.Factory field) implements ExpressionEvaluator.Factory { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java index d8692faef5290..5a8b3d32e7db0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java @@ -7,11 +7,20 @@ package org.elasticsearch.xpack.esql.evaluator.mapper; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.indices.breaker.AllCircuitBreakerStats; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.indices.breaker.CircuitBreakerStats; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.evaluator.EvalMapper; import org.elasticsearch.xpack.esql.planner.Layout; import static org.elasticsearch.compute.data.BlockUtils.fromArrayRow; @@ -23,9 +32,12 @@ public interface EvaluatorMapper { interface ToEvaluator { ExpressionEvaluator.Factory apply(Expression expression); + + FoldContext foldCtx(); } /** + * Convert this into an {@link ExpressionEvaluator}. *

* Note for implementors: * If you are implementing this function, you should call the passed-in @@ -35,8 +47,8 @@ interface ToEvaluator { *

* Note for Callers: * If you are attempting to call this method, and you have an - * {@link Expression} and a {@link org.elasticsearch.xpack.esql.planner.Layout}, - * you likely want to call {@link org.elasticsearch.xpack.esql.evaluator.EvalMapper#toEvaluator(Expression, Layout)} + * {@link Expression} and a {@link Layout}, + * you likely want to call {@link EvalMapper#toEvaluator} * instead. On the other hand, if you already have something that * looks like the parameter for this method, you should call this method * with that function. @@ -56,19 +68,89 @@ interface ToEvaluator { /** * Fold using {@link #toEvaluator} so you don't need a "by hand" - * implementation of fold. The evaluator that it makes is "funny" - * in that it'll always call {@link Expression#fold}, but that's - * good enough. + * implementation of {@link Expression#fold}. */ - default Object fold() { - return toJavaObject(toEvaluator(e -> driverContext -> new ExpressionEvaluator() { + default Object fold(Source source, FoldContext ctx) { + /* + * OK! So! We're going to build a bunch of *stuff* that so that we can + * call toEvaluator and use it without standing up an entire compute + * engine. + * + * Step 1 is creation of a `toEvaluator` which we'll soon use to turn + * the *children* of this Expression into ExpressionEvaluators. They + * have to be foldable or else we wouldn't have ended up here. So! + * We just call `fold` on them and turn the result of that into a + * Block. + * + * If the tree of expressions is pretty deep that `fold` call will + * likely end up being implemented by calling this method for the + * child. That's fine. Recursion is how you process trees. + */ + ToEvaluator foldChildren = new ToEvaluator() { @Override - public Block eval(Page page) { - return fromArrayRow(driverContext.blockFactory(), e.fold())[0]; + public ExpressionEvaluator.Factory apply(Expression expression) { + return driverContext -> new ExpressionEvaluator() { + @Override + public Block eval(Page page) { + return fromArrayRow(driverContext.blockFactory(), expression.fold(ctx))[0]; + } + + @Override + public void close() {} + }; } @Override - public void close() {} - }).get(DriverContext.getLocalDriver()).eval(new Page(1)), 0); + public FoldContext foldCtx() { + return ctx; + } + }; + + /* + * Step 2 is to create a DriverContext that we can pass to the above. + * This DriverContext is mostly about delegating to the FoldContext. + * That'll cause us to break if we attempt to allocate a huge amount + * of memory. Neat. + * + * Specifically, we make a CircuitBreaker view of the FoldContext, then + * we wrap it in a CircuitBreakerService so we can feed it to a BigArray + * so we can feed *that* into a DriverContext. It's a bit hacky, but + * that's what's going on here. + */ + CircuitBreaker breaker = ctx.circuitBreakerView(source); + BigArrays bigArrays = new BigArrays(null, new CircuitBreakerService() { + @Override + public CircuitBreaker getBreaker(String name) { + if (name.equals(CircuitBreaker.REQUEST) == false) { + throw new UnsupportedOperationException(); + } + return breaker; + } + + @Override + public AllCircuitBreakerStats stats() { + throw new UnsupportedOperationException(); + } + + @Override + public CircuitBreakerStats stats(String name) { + throw new UnsupportedOperationException(); + } + }, CircuitBreaker.REQUEST).withCircuitBreaking(); + DriverContext driverCtx = new DriverContext(bigArrays, new BlockFactory(breaker, bigArrays)); + + /* + * Finally we can call toEvaluator on ourselves! It'll fold our children, + * convert the result into Blocks, and then we'll run that with the memory + * breaking DriverContext. + * + * Then, finally finally, we turn the result into a java object to be compatible + * with the signature of `fold`. + */ + Block block = toEvaluator(foldChildren).get(driverCtx).eval(new Page(1)); + if (block.getPositionCount() != 1) { + throw new IllegalStateException("generated odd block from fold [" + block + "]"); + } + return toJavaObject(block, 0); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/ExpressionMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/ExpressionMapper.java index 5cd830058573f..5a76080e7995c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/ExpressionMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/ExpressionMapper.java @@ -9,6 +9,7 @@ import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.util.ReflectionUtils; import org.elasticsearch.xpack.esql.planner.Layout; @@ -19,5 +20,5 @@ public ExpressionMapper() { typeToken = ReflectionUtils.detectSuperTypeForRuleLike(getClass()); } - public abstract ExpressionEvaluator.Factory map(E expression, Layout layout); + public abstract ExpressionEvaluator.Factory map(FoldContext foldCtx, E expression, Layout layout); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java index 974f029eab2ef..94913581f696d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.analysis.PreAnalyzer; import org.elasticsearch.xpack.esql.analysis.Verifier; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; @@ -56,6 +57,7 @@ public void esql( EsqlQueryRequest request, String sessionId, Configuration cfg, + FoldContext foldContext, EnrichPolicyResolver enrichPolicyResolver, EsqlExecutionInfo executionInfo, IndicesExpressionGrouper indicesExpressionGrouper, @@ -71,7 +73,7 @@ public void esql( enrichPolicyResolver, preAnalyzer, functionRegistry, - new LogicalPlanOptimizer(new LogicalOptimizerContext(cfg)), + new LogicalPlanOptimizer(new LogicalOptimizerContext(cfg, foldContext)), mapper, verifier, planningMetrics, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java index 011fcaccf7fe4..8aa7f697489c6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.function.Function; @@ -92,7 +93,8 @@ public List parameters() { } public boolean hasFilter() { - return filter != null && (filter.foldable() == false || Boolean.TRUE.equals(filter.fold()) == false); + return filter != null + && (filter.foldable() == false || Boolean.TRUE.equals(filter.fold(FoldContext.small() /* TODO remove me */)) == false); } public Expression filter() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java index 7436db9e00dd2..3170ae8f132c2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java @@ -19,6 +19,7 @@ import org.elasticsearch.compute.aggregation.CountDistinctLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -210,7 +211,9 @@ protected TypeResolution resolveType() { @Override public AggregatorFunctionSupplier supplier(List inputChannels) { DataType type = field().dataType(); - int precision = this.precision == null ? DEFAULT_PRECISION : ((Number) this.precision.fold()).intValue(); + int precision = this.precision == null + ? DEFAULT_PRECISION + : ((Number) this.precision.fold(FoldContext.small() /* TODO remove me */)).intValue(); if (SUPPLIERS.containsKey(type) == false) { // If the type checking did its job, this should never happen throw EsqlIllegalArgumentException.illegalDataType(type); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java index 0d57267da1e29..8c943c991d501 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java @@ -16,6 +16,7 @@ import org.elasticsearch.compute.aggregation.PercentileIntAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.PercentileLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -170,7 +171,7 @@ protected AggregatorFunctionSupplier doubleSupplier(List inputChannels) } private int percentileValue() { - return ((Number) percentile.fold()).intValue(); + return ((Number) percentile.fold(FoldContext.small() /* TODO remove me */)).intValue(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java index 87ac9b77a6826..85ae65b6c5dc3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java @@ -18,6 +18,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -156,7 +157,7 @@ long unitInMillis() { } final Object foldValue; try { - foldValue = unit.fold(); + foldValue = unit.fold(FoldContext.small() /* TODO remove me */); } catch (Exception e) { throw new IllegalArgumentException("function [" + sourceText() + "] has invalid unit [" + unit.sourceText() + "]"); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java index 40777b4d78dc2..9be8c94266ee8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java @@ -21,6 +21,7 @@ import org.elasticsearch.compute.aggregation.TopLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -115,11 +116,11 @@ Expression orderField() { } private int limitValue() { - return (int) limitField().fold(); + return (int) limitField().fold(FoldContext.small() /* TODO remove me */); } private String orderRawValue() { - return BytesRefs.toString(orderField().fold()); + return BytesRefs.toString(orderField().fold(FoldContext.small() /* TODO remove me */)); } private boolean orderValue() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java index 56d034a2eae1d..bab65653ba576 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -114,9 +115,15 @@ protected Expression.TypeResolution resolveType() { return resolution; } - if (weight.dataType() == DataType.NULL - || (weight.foldable() && (weight.fold() == null || weight.fold().equals(0) || weight.fold().equals(0.0)))) { - return new TypeResolution(format(null, invalidWeightError, SECOND, sourceText(), weight.foldable() ? weight.fold() : null)); + if (weight.dataType() == DataType.NULL) { + return new TypeResolution(format(null, invalidWeightError, SECOND, sourceText(), null)); + } + if (weight.foldable() == false) { + return TypeResolution.TYPE_RESOLVED; + } + Object weightVal = weight.fold(FoldContext.small()/* TODO remove me*/); + if (weightVal == null || weightVal.equals(0) || weightVal.equals(0.0)) { + return new TypeResolution(format(null, invalidWeightError, SECOND, sourceText(), weightVal)); } return TypeResolution.TYPE_RESOLVED; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java index 07c4bb282ba71..4da7c01139c24 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; @@ -111,7 +112,7 @@ public Expression query() { * @return query expression as an object */ public Object queryAsObject() { - Object queryAsObject = query().fold(); + Object queryAsObject = query().fold(FoldContext.small() /* TODO remove me */); if (queryAsObject instanceof BytesRef bytesRef) { return bytesRef.utf8ToString(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java index ce23860dbdba7..7552b100119f0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator; import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; @@ -222,7 +223,7 @@ public void postLogicalOptimizationVerification(Failures failures) { @Override public Object queryAsObject() { - Object queryAsObject = query().fold(); + Object queryAsObject = query().fold(FoldContext.small() /* TODO remove me */); // Convert BytesRef to string for string-based values if (queryAsObject instanceof BytesRef bytesRef) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java index 113989323eff2..7a3e080f5c830 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Foldables; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; @@ -255,25 +256,25 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (field.dataType() == DataType.DATETIME || field.dataType() == DataType.DATE_NANOS) { Rounding.Prepared preparedRounding; if (buckets.dataType().isWholeNumber()) { - int b = ((Number) buckets.fold()).intValue(); - long f = foldToLong(from); - long t = foldToLong(to); + int b = ((Number) buckets.fold(toEvaluator.foldCtx())).intValue(); + long f = foldToLong(toEvaluator.foldCtx(), from); + long t = foldToLong(toEvaluator.foldCtx(), to); preparedRounding = new DateRoundingPicker(b, f, t).pickRounding().prepareForUnknown(); } else { assert DataType.isTemporalAmount(buckets.dataType()) : "Unexpected span data type [" + buckets.dataType() + "]"; - preparedRounding = DateTrunc.createRounding(buckets.fold(), DEFAULT_TZ); + preparedRounding = DateTrunc.createRounding(buckets.fold(toEvaluator.foldCtx()), DEFAULT_TZ); } return DateTrunc.evaluator(field.dataType(), source(), toEvaluator.apply(field), preparedRounding); } if (field.dataType().isNumeric()) { double roundTo; if (from != null) { - int b = ((Number) buckets.fold()).intValue(); - double f = ((Number) from.fold()).doubleValue(); - double t = ((Number) to.fold()).doubleValue(); + int b = ((Number) buckets.fold(toEvaluator.foldCtx())).intValue(); + double f = ((Number) from.fold(toEvaluator.foldCtx())).doubleValue(); + double t = ((Number) to.fold(toEvaluator.foldCtx())).doubleValue(); roundTo = pickRounding(b, f, t); } else { - roundTo = ((Number) buckets.fold()).doubleValue(); + roundTo = ((Number) buckets.fold(toEvaluator.foldCtx())).doubleValue(); } Literal rounding = new Literal(source(), roundTo, DataType.DOUBLE); @@ -416,8 +417,8 @@ public void postLogicalOptimizationVerification(Failures failures) { .add(to != null ? isFoldable(to, operation, FOURTH) : null); } - private long foldToLong(Expression e) { - Object value = Foldables.valueOf(e); + private long foldToLong(FoldContext ctx, Expression e) { + Object value = Foldables.valueOf(ctx, e); return DataType.isDateTime(e.dataType()) ? ((Number) value).longValue() : dateTimeToLong(((BytesRef) value).utf8ToString()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/GroupingFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/GroupingFunction.java index 0fee65d32ca98..fd025e5e67a7c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/GroupingFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/GroupingFunction.java @@ -10,6 +10,7 @@ import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; @@ -28,8 +29,8 @@ protected GroupingFunction(Source source, List fields) { } @Override - public Object fold() { - return EvaluatorMapper.super.fold(); + public Object fold(FoldContext ctx) { + return EvaluatorMapper.super.fold(source(), ctx); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java index 404ce7e3900c9..85d15f82f458a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.expression.function.scalar; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; @@ -34,7 +35,7 @@ protected EsqlScalarFunction(Source source, List fields) { } @Override - public Object fold() { - return EvaluatorMapper.super.fold(); + public Object fold(FoldContext ctx) { + return EvaluatorMapper.super.fold(source(), ctx); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java index 824f02ca7ccbb..236e625f7abe1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java @@ -23,6 +23,7 @@ import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; @@ -227,7 +228,7 @@ public boolean foldable() { if (condition.condition.foldable() == false) { return false; } - if (Boolean.TRUE.equals(condition.condition.fold())) { + if (Boolean.TRUE.equals(condition.condition.fold(FoldContext.small() /* TODO remove me - use literal true?*/))) { /* * `fold` can make four things here: * 1. `TRUE` @@ -264,7 +265,8 @@ public boolean foldable() { * And those two combine so {@code EVAL c=CASE(false, foo, b, bar, true, bort, el)} becomes * {@code EVAL c=CASE(b, bar, bort)}. */ - public Expression partiallyFold() { + public Expression partiallyFold(FoldContext ctx) { + // TODO don't throw away the results of any `fold`. That might mean looking for literal TRUE on the conditions. List newChildren = new ArrayList<>(children().size()); boolean modified = false; for (Condition condition : conditions) { @@ -274,7 +276,7 @@ public Expression partiallyFold() { continue; } modified = true; - if (Boolean.TRUE.equals(condition.condition.fold())) { + if (Boolean.TRUE.equals(condition.condition.fold(ctx))) { /* * `fold` can make four things here: * 1. `TRUE` diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java index 842e899ebdac6..57f362f86ff4c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -65,8 +66,8 @@ protected final Map factories() { } @Override - public final Object fold() { - return foldToTemporalAmount(field(), sourceText(), dataType()); + public final Object fold(FoldContext ctx) { + return foldToTemporalAmount(ctx, field(), sourceText(), dataType()); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java index f6a23a5d5962e..b588832aba4cb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java @@ -232,7 +232,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (unit.foldable()) { try { - Part datePartField = Part.resolve(((BytesRef) unit.fold()).utf8ToString()); + Part datePartField = Part.resolve(((BytesRef) unit.fold(toEvaluator.foldCtx())).utf8ToString()); return new DateDiffConstantEvaluator.Factory(source(), datePartField, startTimestampEvaluator, endTimestampEvaluator); } catch (IllegalArgumentException e) { throw new InvalidArgumentException("invalid unit format for [{}]: {}", sourceText(), e.getMessage()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java index 501dfd431f106..7fc5d82441802 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java @@ -16,6 +16,7 @@ import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.InvalidArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -110,9 +111,9 @@ public String getWriteableName() { public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(children().get(1)); if (children().get(0).foldable()) { - ChronoField chrono = chronoField(); + ChronoField chrono = chronoField(toEvaluator.foldCtx()); if (chrono == null) { - BytesRef field = (BytesRef) children().get(0).fold(); + BytesRef field = (BytesRef) children().get(0).fold(toEvaluator.foldCtx()); throw new InvalidArgumentException("invalid date field for [{}]: {}", sourceText(), field.utf8ToString()); } return new DateExtractConstantEvaluator.Factory(source(), fieldEvaluator, chrono, configuration().zoneId()); @@ -121,14 +122,14 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new DateExtractEvaluator.Factory(source(), fieldEvaluator, chronoEvaluator, configuration().zoneId()); } - private ChronoField chronoField() { + private ChronoField chronoField(FoldContext ctx) { // chronoField's never checked (the return is). The foldability test is done twice and type is checked in resolveType() already. // TODO: move the slimmed down code here to toEvaluator? if (chronoField == null) { Expression field = children().get(0); try { if (field.foldable() && DataType.isString(field.dataType())) { - chronoField = (ChronoField) STRING_TO_CHRONO_FIELD.convert(field.fold()); + chronoField = (ChronoField) STRING_TO_CHRONO_FIELD.convert(field.fold(ctx)); } } catch (Exception e) { return null; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java index 920a3bb1f4a13..29648d55cadd8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java @@ -147,7 +147,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { throw new IllegalArgumentException("unsupported data type for format [" + format.dataType() + "]"); } if (format.foldable()) { - DateFormatter formatter = toFormatter(format.fold(), configuration().locale()); + DateFormatter formatter = toFormatter(format.fold(toEvaluator.foldCtx()), configuration().locale()); return new DateFormatConstantEvaluator.Factory(source(), fieldEvaluator, formatter); } var formatEvaluator = toEvaluator.apply(format); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java index e09fabab98d0f..7c38b54ed232b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java @@ -143,7 +143,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } if (format.foldable()) { try { - DateFormatter formatter = toFormatter(format.fold()); + DateFormatter formatter = toFormatter(format.fold(toEvaluator.foldCtx())); return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, formatter); } catch (IllegalArgumentException e) { throw new InvalidArgumentException(e, "invalid date pattern for [{}]: {}", sourceText(), e.getMessage()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java index a35b67d7ac3fd..7983c38cc4288 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java @@ -225,7 +225,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } Object foldedInterval; try { - foldedInterval = interval.fold(); + foldedInterval = interval.fold(toEvaluator.foldCtx()); if (foldedInterval == null) { throw new IllegalArgumentException("Interval cannot not be null"); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java index d259fc6ae57ce..74c2da450995c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java @@ -14,6 +14,7 @@ import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -59,7 +60,7 @@ public String getWriteableName() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { return now; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/E.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/E.java index 757b67b47ce72..e1eceef7ed1f5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/E.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/E.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; @@ -49,7 +50,7 @@ public String getWriteableName() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { return Math.E; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pi.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pi.java index 90a4f1f091e91..32b7a0ab88b4e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pi.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pi.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; @@ -49,7 +50,7 @@ public String getWriteableName() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { return Math.PI; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Tau.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Tau.java index 17e5b027270d1..1a7669b7391e1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Tau.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Tau.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; @@ -51,7 +52,7 @@ public String getWriteableName() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { return TAU; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java index 1996744a76567..26211258e6ca6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java @@ -17,6 +17,7 @@ import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -91,8 +92,8 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } @Override - public Object fold() { - return EvaluatorMapper.super.fold(); + public Object fold(FoldContext ctx) { + return EvaluatorMapper.super.fold(source(), ctx); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java index 4dd447f938880..d5093964145b7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java @@ -115,7 +115,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua source(), toEvaluator.apply(field), ctx -> new CompensatedSum(), - (Double) p.fold() + (Double) p.fold(toEvaluator.foldCtx()) ); case NULL -> EvalOperator.CONSTANT_NULL_FACTORY; default -> throw EsqlIllegalArgumentException.illegalDataType(field.dataType()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java index f4f9679dc3704..4a04524d1b23d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java @@ -187,8 +187,8 @@ public boolean foldable() { @Override public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (start.foldable() && end.foldable()) { - int startOffset = stringToInt(String.valueOf(start.fold())); - int endOffset = stringToInt(String.valueOf(end.fold())); + int startOffset = stringToInt(String.valueOf(start.fold(toEvaluator.foldCtx()))); + int endOffset = stringToInt(String.valueOf(end.fold(toEvaluator.foldCtx()))); checkStartEnd(startOffset, endOffset); } return switch (PlannerUtils.toElementType(field.dataType())) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java index 86538c828ece7..b68718acfcd0a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -155,12 +156,12 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua sourceText(), ASC.value(), DESC.value(), - ((BytesRef) order.fold()).utf8ToString() + ((BytesRef) order.fold(toEvaluator.foldCtx())).utf8ToString() ) ); } if (order != null && order.foldable()) { - ordering = ((BytesRef) order.fold()).utf8ToString().equalsIgnoreCase((String) ASC.value()); + ordering = ((BytesRef) order.fold(toEvaluator.foldCtx())).utf8ToString().equalsIgnoreCase((String) ASC.value()); } return switch (PlannerUtils.toElementType(field.dataType())) { @@ -238,7 +239,14 @@ public void postLogicalOptimizationVerification(Failures failures) { failures.add(isFoldable(order, operation, SECOND)); if (isValidOrder() == false) { failures.add( - Failure.fail(order, INVALID_ORDER_ERROR, sourceText(), ASC.value(), DESC.value(), ((BytesRef) order.fold()).utf8ToString()) + Failure.fail( + order, + INVALID_ORDER_ERROR, + sourceText(), + ASC.value(), + DESC.value(), + ((BytesRef) order.fold(FoldContext.small() /* TODO remove me */)).utf8ToString() + ) ); } } @@ -246,7 +254,7 @@ public void postLogicalOptimizationVerification(Failures failures) { private boolean isValidOrder() { boolean isValidOrder = true; if (order != null && order.foldable()) { - Object obj = order.fold(); + Object obj = order.fold(FoldContext.small() /* TODO remove me */); String o = null; if (obj instanceof BytesRef ob) { o = ob.utf8ToString(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java index 9189c6a7b8f70..b15d04aa792d9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java @@ -26,6 +26,7 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -214,10 +215,10 @@ protected NodeInfo info() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { try { - GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left()); - Geometry rightGeom = makeGeometryFromLiteral(right()); + GeometryDocValueReader docValueReader = asGeometryDocValueReader(ctx, crsType(), left()); + Geometry rightGeom = makeGeometryFromLiteral(ctx, right()); Component2D[] components = asLuceneComponent2Ds(crsType(), rightGeom); return (crsType() == SpatialCrsType.GEO) ? GEO.geometryRelatesGeometries(docValueReader, components) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java index ee78f50c4d6bd..3e16fa163fcd6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java @@ -23,6 +23,7 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -129,10 +130,10 @@ protected NodeInfo info() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { try { - GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left()); - Component2D component2D = asLuceneComponent2D(crsType(), right()); + GeometryDocValueReader docValueReader = asGeometryDocValueReader(ctx, crsType(), left()); + Component2D component2D = asLuceneComponent2D(ctx, crsType(), right()); return (crsType() == SpatialCrsType.GEO) ? GEO.geometryRelatesGeometry(docValueReader, component2D) : CARTESIAN.geometryRelatesGeometry(docValueReader, component2D); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java index 1a51af8dfeeb4..dcd53075cf69c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java @@ -171,7 +171,11 @@ protected static class SpatialEvaluatorWithConstantFactory extends SpatialEvalua @Override public EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier s, EvaluatorMapper.ToEvaluator toEvaluator) { - return factoryCreator.apply(s.source(), toEvaluator.apply(s.left()), asLuceneComponent2D(s.crsType(), s.right())); + return factoryCreator.apply( + s.source(), + toEvaluator.apply(s.left()), + asLuceneComponent2D(toEvaluator.foldCtx(), s.crsType(), s.right()) + ); } } @@ -197,7 +201,11 @@ protected static class SpatialEvaluatorWithConstantArrayFactory extends SpatialE @Override public EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier s, EvaluatorMapper.ToEvaluator toEvaluator) { - return factoryCreator.apply(s.source(), toEvaluator.apply(s.left()), asLuceneComponent2Ds(s.crsType(), s.right())); + return factoryCreator.apply( + s.source(), + toEvaluator.apply(s.left()), + asLuceneComponent2Ds(toEvaluator.foldCtx(), s.crsType(), s.right()) + ); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java index 8d54e5ee443c2..601550cd173bb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java @@ -23,6 +23,7 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -127,10 +128,10 @@ protected NodeInfo info() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { try { - GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left()); - Component2D component2D = asLuceneComponent2D(crsType(), right()); + GeometryDocValueReader docValueReader = asGeometryDocValueReader(ctx, crsType(), left()); + Component2D component2D = asLuceneComponent2D(ctx, crsType(), right()); return (crsType() == SpatialCrsType.GEO) ? GEO.geometryRelatesGeometry(docValueReader, component2D) : CARTESIAN.geometryRelatesGeometry(docValueReader, component2D); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java index 6ae99ea8165cd..1b06c6dfd3dd5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java @@ -29,6 +29,7 @@ import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.lucene.spatial.GeometryDocValueWriter; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; @@ -42,8 +43,8 @@ public class SpatialRelatesUtils { /** Converts a {@link Expression} into a {@link Component2D}. */ - static Component2D asLuceneComponent2D(BinarySpatialFunction.SpatialCrsType crsType, Expression expression) { - return asLuceneComponent2D(crsType, makeGeometryFromLiteral(expression)); + static Component2D asLuceneComponent2D(FoldContext ctx, BinarySpatialFunction.SpatialCrsType crsType, Expression expression) { + return asLuceneComponent2D(crsType, makeGeometryFromLiteral(ctx, expression)); } /** Converts a {@link Geometry} into a {@link Component2D}. */ @@ -66,8 +67,8 @@ static Component2D asLuceneComponent2D(BinarySpatialFunction.SpatialCrsType type * Converts a {@link Expression} at a given {@code position} into a {@link Component2D} array. * The reason for generating an array instead of a single component is for multi-shape support with ST_CONTAINS. */ - static Component2D[] asLuceneComponent2Ds(BinarySpatialFunction.SpatialCrsType crsType, Expression expression) { - return asLuceneComponent2Ds(crsType, makeGeometryFromLiteral(expression)); + static Component2D[] asLuceneComponent2Ds(FoldContext ctx, BinarySpatialFunction.SpatialCrsType crsType, Expression expression) { + return asLuceneComponent2Ds(crsType, makeGeometryFromLiteral(ctx, expression)); } /** @@ -90,9 +91,12 @@ static Component2D[] asLuceneComponent2Ds(BinarySpatialFunction.SpatialCrsType t } /** Converts a {@link Expression} into a {@link GeometryDocValueReader} */ - static GeometryDocValueReader asGeometryDocValueReader(BinarySpatialFunction.SpatialCrsType crsType, Expression expression) - throws IOException { - Geometry geometry = makeGeometryFromLiteral(expression); + static GeometryDocValueReader asGeometryDocValueReader( + FoldContext ctx, + BinarySpatialFunction.SpatialCrsType crsType, + Expression expression + ) throws IOException { + Geometry geometry = makeGeometryFromLiteral(ctx, expression); if (crsType == BinarySpatialFunction.SpatialCrsType.GEO) { return asGeometryDocValueReader( CoordinateEncoder.GEO, @@ -167,8 +171,8 @@ private static Geometry asGeometry(BytesRefBlock valueBlock, int position) { * This function is used in two places, when evaluating a spatial constant in the SpatialRelatesFunction, as well as when * we do lucene-pushdown of spatial functions. */ - public static Geometry makeGeometryFromLiteral(Expression expr) { - return makeGeometryFromLiteralValue(valueOf(expr), expr.dataType()); + public static Geometry makeGeometryFromLiteral(FoldContext ctx, Expression expr) { + return makeGeometryFromLiteralValue(valueOf(ctx, expr), expr.dataType()); } private static Geometry makeGeometryFromLiteralValue(Object value, DataType dataType) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java index 2005709cd37e9..9fcece1ce65bc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java @@ -23,6 +23,7 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -129,10 +130,10 @@ protected NodeInfo info() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { try { - GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left()); - Component2D component2D = asLuceneComponent2D(crsType(), right()); + GeometryDocValueReader docValueReader = asGeometryDocValueReader(ctx, crsType(), left()); + Component2D component2D = asLuceneComponent2D(ctx, crsType(), right()); return (crsType() == SpatialCrsType.GEO) ? GEO.geometryRelatesGeometry(docValueReader, component2D) : CARTESIAN.geometryRelatesGeometry(docValueReader, component2D); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java index 3cf042a2db828..f0c25e3289cc1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java @@ -23,6 +23,7 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -280,18 +281,18 @@ protected NodeInfo info() { } @Override - public Object fold() { - var leftGeom = makeGeometryFromLiteral(left()); - var rightGeom = makeGeometryFromLiteral(right()); + public Object fold(FoldContext ctx) { + var leftGeom = makeGeometryFromLiteral(ctx, left()); + var rightGeom = makeGeometryFromLiteral(ctx, right()); return (crsType() == SpatialCrsType.GEO) ? GEO.distance(leftGeom, rightGeom) : CARTESIAN.distance(leftGeom, rightGeom); } @Override public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (right().foldable()) { - return toEvaluator(toEvaluator, left(), makeGeometryFromLiteral(right()), leftDocValues); + return toEvaluator(toEvaluator, left(), makeGeometryFromLiteral(toEvaluator.foldCtx(), right()), leftDocValues); } else if (left().foldable()) { - return toEvaluator(toEvaluator, right(), makeGeometryFromLiteral(left()), rightDocValues); + return toEvaluator(toEvaluator, right(), makeGeometryFromLiteral(toEvaluator.foldCtx(), left()), rightDocValues); } else { EvalOperator.ExpressionEvaluator.Factory leftE = toEvaluator.apply(left()); EvalOperator.ExpressionEvaluator.Factory rightE = toEvaluator.apply(right()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java index 52d33c0fc9d3d..be0a7b2fe27b2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java @@ -146,7 +146,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua if (algorithm.foldable()) { try { // hash function is created here in order to validate the algorithm is valid before evaluator is created - var hf = HashFunction.create((BytesRef) algorithm.fold()); + var hf = HashFunction.create((BytesRef) algorithm.fold(toEvaluator.foldCtx())); return new HashConstantEvaluator.Factory( source(), context -> new BreakingBytesRefBuilder(context.breaker(), "hash"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java index 996c90a8e40bc..fb0aac0c85b38 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -94,8 +95,8 @@ protected TypeResolution resolveType() { } @Override - public Boolean fold() { - return (Boolean) EvaluatorMapper.super.fold(); + public Boolean fold(FoldContext ctx) { + return (Boolean) EvaluatorMapper.super.fold(source(), ctx); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java index e91f03de3dd7e..363991d1556f1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java @@ -151,7 +151,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { ExpressionEvaluator.Factory strExpr = toEvaluator.apply(str); if (number.foldable()) { - int num = (int) number.fold(); + int num = (int) number.fold(toEvaluator.foldCtx()); if (num < 0) { throw new IllegalArgumentException("Number parameter cannot be negative, found [" + number + "]"); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java index 4fa191244cb42..4b963b794aef0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java @@ -152,7 +152,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (regex.foldable() && regex.dataType() == DataType.KEYWORD) { Pattern regexPattern; try { - regexPattern = Pattern.compile(((BytesRef) regex.fold()).utf8ToString()); + regexPattern = Pattern.compile(((BytesRef) regex.fold(toEvaluator.foldCtx())).utf8ToString()); } catch (PatternSyntaxException pse) { // TODO this is not right (inconsistent). See also https://github.com/elastic/elasticsearch/issues/100038 // this should generate a header warning and return null (as do the rest of this functionality in evaluators), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java index 3b9a466966911..e46c0a730431d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java @@ -113,7 +113,7 @@ protected NodeInfo info() { @Override public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (field.foldable()) { - Object folded = field.fold(); + Object folded = field.fold(toEvaluator.foldCtx()); if (folded instanceof Integer num) { checkNumber(num); return toEvaluator.apply(new Literal(source(), " ".repeat(num), KEYWORD)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java index 24762122f755b..d0c1035978ff3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java @@ -17,6 +17,7 @@ import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.InvalidArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -102,8 +103,8 @@ public boolean foldable() { } @Override - public Object fold() { - return EvaluatorMapper.super.fold(); + public Object fold(FoldContext ctx) { + return EvaluatorMapper.super.fold(source(), ctx); } @Evaluator(extraName = "SingleByte") @@ -163,7 +164,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (right().foldable() == false) { return new SplitVariableEvaluator.Factory(source(), str, toEvaluator.apply(right()), context -> new BytesRef()); } - BytesRef delim = (BytesRef) right().fold(); + BytesRef delim = (BytesRef) right().fold(toEvaluator.foldCtx()); checkDelimiter(delim); return new SplitSingleByteEvaluator.Factory(source(), str, delim.bytes[delim.offset], context -> new BytesRef()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java index d2edb0f92e8f2..65455c708cc9b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -100,8 +101,8 @@ protected TypeResolution resolveType() { } @Override - public Boolean fold() { - return (Boolean) EvaluatorMapper.super.fold(); + public Boolean fold(FoldContext ctx) { + return (Boolean) EvaluatorMapper.super.fold(source(), ctx); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/util/Delay.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/util/Delay.java index 1d03f09c86409..3b17133bf4974 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/util/Delay.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/util/Delay.java @@ -14,6 +14,7 @@ import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -84,15 +85,15 @@ public boolean foldable() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { return null; } - private long msValue() { + private long msValue(FoldContext ctx) { if (field().foldable() == false) { throw new IllegalArgumentException("function [" + sourceText() + "] has invalid argument [" + field().sourceText() + "]"); } - var ms = field().fold(); + var ms = field().fold(ctx); if (ms instanceof Duration duration) { return duration.toMillis(); } @@ -101,7 +102,7 @@ private long msValue() { @Override public ExpressionEvaluator.Factory toEvaluator(EvaluatorMapper.ToEvaluator toEvaluator) { - return new DelayEvaluator.Factory(source(), msValue()); + return new DelayEvaluator.Factory(source(), msValue(toEvaluator.foldCtx())); } @Evaluator diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java index 8bb166fac60bb..424c080c905e3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java @@ -11,6 +11,7 @@ import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.ExceptionUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -111,7 +112,7 @@ protected TypeResolution checkCompatibility() { /** * Override this to allow processing literals of type {@link DataType#DATE_PERIOD} when folding constants. - * Used in {@link DateTimeArithmeticOperation#fold()}. + * Used in {@link DateTimeArithmeticOperation#fold}. * @param left the left period * @param right the right period * @return the result of the evaluation @@ -120,7 +121,7 @@ protected TypeResolution checkCompatibility() { /** * Override this to allow processing literals of type {@link DataType#TIME_DURATION} when folding constants. - * Used in {@link DateTimeArithmeticOperation#fold()}. + * Used in {@link DateTimeArithmeticOperation#fold}. * @param left the left duration * @param right the right duration * @return the result of the evaluation @@ -128,13 +129,13 @@ protected TypeResolution checkCompatibility() { abstract Duration fold(Duration left, Duration right); @Override - public final Object fold() { + public final Object fold(FoldContext ctx) { DataType leftDataType = left().dataType(); DataType rightDataType = right().dataType(); if (leftDataType == DATE_PERIOD && rightDataType == DATE_PERIOD) { // Both left and right expressions are temporal amounts; we can assume they are both foldable. - var l = left().fold(); - var r = right().fold(); + var l = left().fold(ctx); + var r = right().fold(ctx); if (l instanceof Collection || r instanceof Collection) { return null; } @@ -148,8 +149,8 @@ public final Object fold() { } if (leftDataType == TIME_DURATION && rightDataType == TIME_DURATION) { // Both left and right expressions are temporal amounts; we can assume they are both foldable. - Duration l = (Duration) left().fold(); - Duration r = (Duration) right().fold(); + Duration l = (Duration) left().fold(ctx); + Duration r = (Duration) right().fold(ctx); try { return fold(l, r); } catch (ArithmeticException e) { @@ -161,7 +162,7 @@ public final Object fold() { if (isNull(leftDataType) || isNull(rightDataType)) { return null; } - return super.fold(); + return super.fold(ctx); } @Override @@ -178,7 +179,11 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { temporalAmountArgument = left(); } - return millisEvaluator.apply(source(), toEvaluator.apply(datetimeArgument), (TemporalAmount) temporalAmountArgument.fold()); + return millisEvaluator.apply( + source(), + toEvaluator.apply(datetimeArgument), + (TemporalAmount) temporalAmountArgument.fold(toEvaluator.foldCtx()) + ); } else if (dataType() == DATE_NANOS) { // One of the arguments has to be a date_nanos and the other a temporal amount. Expression dateNanosArgument; @@ -191,7 +196,11 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { temporalAmountArgument = left(); } - return nanosEvaluator.apply(source(), toEvaluator.apply(dateNanosArgument), (TemporalAmount) temporalAmountArgument.fold()); + return nanosEvaluator.apply( + source(), + toEvaluator.apply(dateNanosArgument), + (TemporalAmount) temporalAmountArgument.fold(toEvaluator.foldCtx()) + ); } else { return super.toEvaluator(toEvaluator); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java index 74394d796855f..e3248665ad486 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java @@ -12,6 +12,7 @@ import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryArithmeticOperation; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -120,8 +121,8 @@ public interface BinaryEvaluator { } @Override - public Object fold() { - return EvaluatorMapper.super.fold(); + public Object fold(FoldContext ctx) { + return EvaluatorMapper.super.fold(source(), ctx); } public DataType dataType() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java index fb32282005f02..6663ccf0ef7b6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.ExceptionUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -87,12 +88,12 @@ else if (type == DataType.LONG) { } @Override - public final Object fold() { + public final Object fold(FoldContext ctx) { DataType dataType = field().dataType(); // For date periods and time durations, we need to treat folding differently. These types are unrepresentable, so there is no // evaluator for them - but the default folding requires an evaluator. if (dataType == DATE_PERIOD) { - Period fieldValue = (Period) field().fold(); + Period fieldValue = (Period) field().fold(ctx); try { return fieldValue.negated(); } catch (ArithmeticException e) { @@ -102,7 +103,7 @@ public final Object fold() { } } if (dataType == TIME_DURATION) { - Duration fieldValue = (Duration) field().fold(); + Duration fieldValue = (Duration) field().fold(ctx); try { return fieldValue.negated(); } catch (ArithmeticException e) { @@ -111,7 +112,7 @@ public final Object fold() { throw ExceptionUtils.math(source(), e); } } - return super.fold(); + return super.fold(ctx); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index 3e2a21664aa7e..e56c19b26a902 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -13,6 +13,7 @@ import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -204,8 +205,8 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua } @Override - public Boolean fold() { - return (Boolean) EvaluatorMapper.super.fold(); + public Boolean fold(FoldContext ctx) { + return (Boolean) EvaluatorMapper.super.fold(source(), ctx); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java index f596d589cdde2..2061c2626aa45 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java @@ -17,6 +17,7 @@ import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.Comparisons; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -205,11 +206,11 @@ public boolean foldable() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { if (Expressions.isGuaranteedNull(value) || list.stream().allMatch(Expressions::isGuaranteedNull)) { return null; } - return super.fold(); + return super.fold(ctx); } protected boolean areCompatible(DataType left, DataType right) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java index c731e44197f2e..01564644bf5c7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java @@ -16,6 +16,7 @@ import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -77,9 +78,9 @@ public static Automaton automaton(BytesRef val) { } @Override - public Boolean fold() { - BytesRef leftVal = BytesRefs.toBytesRef(left().fold()); - BytesRef rightVal = BytesRefs.toBytesRef(right().fold()); + public Boolean fold(FoldContext ctx) { + BytesRef leftVal = BytesRefs.toBytesRef(left().fold(ctx)); + BytesRef rightVal = BytesRefs.toBytesRef(right().fold(ctx)); if (leftVal == null || rightVal == null) { return null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java index f5704239993f9..7ea95c764f36c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.evaluator.mapper.ExpressionMapper; @@ -28,15 +29,15 @@ public class InsensitiveEqualsMapper extends ExpressionMapper InsensitiveEqualsEvaluator.Factory::new; @Override - public final ExpressionEvaluator.Factory map(InsensitiveEquals bc, Layout layout) { + public final ExpressionEvaluator.Factory map(FoldContext foldCtx, InsensitiveEquals bc, Layout layout) { DataType leftType = bc.left().dataType(); DataType rightType = bc.right().dataType(); - var leftEval = toEvaluator(bc.left(), layout); - var rightEval = toEvaluator(bc.right(), layout); + var leftEval = toEvaluator(foldCtx, bc.left(), layout); + var rightEval = toEvaluator(foldCtx, bc.right(), layout); if (DataType.isString(leftType)) { if (bc.right().foldable() && DataType.isString(rightType)) { - BytesRef rightVal = BytesRefs.toBytesRef(bc.right().fold()); + BytesRef rightVal = BytesRefs.toBytesRef(bc.right().fold(FoldContext.small() /* TODO remove me */)); Automaton automaton = InsensitiveEquals.automaton(rightVal); return dvrCtx -> new InsensitiveEqualsConstantEvaluator( bc.source(), @@ -51,13 +52,14 @@ public final ExpressionEvaluator.Factory map(InsensitiveEquals bc, Layout layout } public static ExpressionEvaluator.Factory castToEvaluator( + FoldContext foldCtx, InsensitiveEquals op, Layout layout, DataType required, TriFunction factory ) { - var lhs = Cast.cast(op.source(), op.left().dataType(), required, toEvaluator(op.left(), layout)); - var rhs = Cast.cast(op.source(), op.right().dataType(), required, toEvaluator(op.right(), layout)); + var lhs = Cast.cast(op.source(), op.left().dataType(), required, toEvaluator(foldCtx, op.left(), layout)); + var rhs = Cast.cast(op.source(), op.right().dataType(), required, toEvaluator(foldCtx, op.right(), layout)); return factory.apply(op.source(), lhs, rhs); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java index ef5cf50c76541..183008f900c5d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.optimizer; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.stats.SearchStats; @@ -15,8 +16,8 @@ public final class LocalLogicalOptimizerContext extends LogicalOptimizerContext { private final SearchStats searchStats; - public LocalLogicalOptimizerContext(Configuration configuration, SearchStats searchStats) { - super(configuration); + public LocalLogicalOptimizerContext(Configuration configuration, FoldContext foldCtx, SearchStats searchStats) { + super(configuration, foldCtx); this.searchStats = searchStats; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java index c11e1a4ec49e4..22e07b45310fb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java @@ -7,7 +7,8 @@ package org.elasticsearch.xpack.esql.optimizer; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.stats.SearchStats; -public record LocalPhysicalOptimizerContext(Configuration configuration, SearchStats searchStats) {} +public record LocalPhysicalOptimizerContext(Configuration configuration, FoldContext foldCtx, SearchStats searchStats) {} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java index 67148e67cbc19..da2d583674a90 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java @@ -7,37 +7,44 @@ package org.elasticsearch.xpack.esql.optimizer; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.session.Configuration; import java.util.Objects; public class LogicalOptimizerContext { private final Configuration configuration; + private final FoldContext foldCtx; - public LogicalOptimizerContext(Configuration configuration) { + public LogicalOptimizerContext(Configuration configuration, FoldContext foldCtx) { this.configuration = configuration; + this.foldCtx = foldCtx; } public Configuration configuration() { return configuration; } + public FoldContext foldCtx() { + return foldCtx; + } + @Override public boolean equals(Object obj) { if (obj == this) return true; if (obj == null || obj.getClass() != this.getClass()) return false; var that = (LogicalOptimizerContext) obj; - return Objects.equals(this.configuration, that.configuration); + return this.configuration.equals(that.configuration) && this.foldCtx.equals(that.foldCtx); } @Override public int hashCode() { - return Objects.hash(configuration); + return Objects.hash(configuration, foldCtx); } @Override public String toString() { - return "LogicalOptimizerContext[" + "configuration=" + configuration + ']'; + return "LogicalOptimizerContext[configuration=" + configuration + ", foldCtx=" + foldCtx + ']'; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsElimination.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsElimination.java index 3152f9b574767..5f463f2aa4c78 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsElimination.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsElimination.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; @@ -28,7 +29,7 @@ public BooleanFunctionEqualsElimination() { } @Override - public Expression rule(BinaryComparison bc) { + public Expression rule(BinaryComparison bc, LogicalOptimizerContext ctx) { if ((bc instanceof Equals || bc instanceof NotEquals) && bc.left() instanceof Function) { // for expression "==" or "!=" TRUE/FALSE, return the expression itself or its negated variant diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplification.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplification.java index 73d1ea1fb6e8f..e1803872fd606 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplification.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplification.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.util.List; @@ -35,7 +36,7 @@ public BooleanSimplification() { } @Override - public Expression rule(ScalarFunction e) { + public Expression rule(ScalarFunction e, LogicalOptimizerContext ctx) { if (e instanceof And || e instanceof Or) { return simplifyAndOr((BinaryPredicate) e); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisons.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisons.java index 3f47c74aaf814..1c290a7c4c4fd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisons.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisons.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic; @@ -20,6 +21,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.util.ArrayList; import java.util.List; @@ -31,17 +33,17 @@ public CombineBinaryComparisons() { } @Override - public Expression rule(BinaryLogic e) { + public Expression rule(BinaryLogic e, LogicalOptimizerContext ctx) { if (e instanceof And and) { - return combine(and); + return combine(ctx.foldCtx(), and); } else if (e instanceof Or or) { - return combine(or); + return combine(ctx.foldCtx(), or); } return e; } // combine conjunction - private static Expression combine(And and) { + private static Expression combine(FoldContext ctx, And and) { List bcs = new ArrayList<>(); List exps = new ArrayList<>(); boolean changed = false; @@ -58,13 +60,13 @@ private static Expression combine(And and) { }); for (Expression ex : andExps) { if (ex instanceof BinaryComparison bc && (ex instanceof Equals || ex instanceof NotEquals) == false) { - if (bc.right().foldable() && (findExistingComparison(bc, bcs, true))) { + if (bc.right().foldable() && (findExistingComparison(ctx, bc, bcs, true))) { changed = true; } else { bcs.add(bc); } } else if (ex instanceof NotEquals neq) { - if (neq.right().foldable() && notEqualsIsRemovableFromConjunction(neq, bcs)) { + if (neq.right().foldable() && notEqualsIsRemovableFromConjunction(ctx, neq, bcs)) { // the non-equality can simply be dropped: either superfluous or has been merged with an updated range/inequality changed = true; } else { // not foldable OR not overlapping @@ -78,13 +80,13 @@ private static Expression combine(And and) { } // combine disjunction - private static Expression combine(Or or) { + private static Expression combine(FoldContext ctx, Or or) { List bcs = new ArrayList<>(); List exps = new ArrayList<>(); boolean changed = false; for (Expression ex : Predicates.splitOr(or)) { if (ex instanceof BinaryComparison bc) { - if (bc.right().foldable() && findExistingComparison(bc, bcs, false)) { + if (bc.right().foldable() && findExistingComparison(ctx, bc, bcs, false)) { changed = true; } else { bcs.add(bc); @@ -100,8 +102,8 @@ private static Expression combine(Or or) { * Find commonalities between the given comparison in the given list. * The method can be applied both for conjunctive (AND) or disjunctive purposes (OR). */ - private static boolean findExistingComparison(BinaryComparison main, List bcs, boolean conjunctive) { - Object value = main.right().fold(); + private static boolean findExistingComparison(FoldContext ctx, BinaryComparison main, List bcs, boolean conjunctive) { + Object value = main.right().fold(ctx); // NB: the loop modifies the list (hence why the int is used) for (int i = 0; i < bcs.size(); i++) { BinaryComparison other = bcs.get(i); @@ -113,7 +115,7 @@ private static boolean findExistingComparison(BinaryComparison main, List bcs) { - Object neqVal = notEquals.right().fold(); + private static boolean notEqualsIsRemovableFromConjunction(FoldContext ctx, NotEquals notEquals, List bcs) { + Object neqVal = notEquals.right().fold(ctx); Integer comp; // check on "condition-overlapping" inequalities: @@ -183,7 +185,7 @@ private static boolean notEqualsIsRemovableFromConjunction(NotEquals notEquals, BinaryComparison bc = bcs.get(i); if (notEquals.left().semanticEquals(bc.left())) { if (bc instanceof LessThan || bc instanceof LessThanOrEqual) { - comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold()) : null; + comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold(ctx)) : null; if (comp != null) { if (comp >= 0) { if (comp == 0 && bc instanceof LessThanOrEqual) { // a != 2 AND a <= 2 -> a < 2 @@ -193,7 +195,7 @@ private static boolean notEqualsIsRemovableFromConjunction(NotEquals notEquals, } // else: comp < 0 : a != 2 AND a nop } // else: non-comparable, nop } else if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) { - comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold()) : null; + comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold(ctx)) : null; if (comp != null) { if (comp <= 0) { if (comp == 0 && bc instanceof GreaterThanOrEqual) { // a != 2 AND a >= 2 -> a > 2 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctions.java index 5cb377de47efc..e1cda9cb149d4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctions.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctions.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.time.ZoneId; import java.util.ArrayList; @@ -61,7 +62,7 @@ protected static CIDRMatch createCIDRMatch(Expression k, List v) { } @Override - public Expression rule(Or or) { + public Expression rule(Or or, LogicalOptimizerContext ctx) { Expression e = or; // look only at equals, In and CIDRMatch List exps = splitOr(e); @@ -78,7 +79,7 @@ public Expression rule(Or or) { if (eq.right().foldable()) { ins.computeIfAbsent(eq.left(), k -> new LinkedHashSet<>()).add(eq.right()); if (eq.left().dataType() == DataType.IP) { - Object value = eq.right().fold(); + Object value = eq.right().fold(ctx.foldCtx()); // ImplicitCasting and ConstantFolding(includes explicit casting) are applied before CombineDisjunctions. // They fold the input IP string to an internal IP format. These happen to Equals and IN, but not for CIDRMatch, // as CIDRMatch takes strings as input, ImplicitCasting does not apply to it, and the first input to CIDRMatch is a @@ -101,7 +102,7 @@ public Expression rule(Or or) { if (in.value().dataType() == DataType.IP) { List values = new ArrayList<>(in.list().size()); for (Expression i : in.list()) { - Object value = i.fold(); + Object value = i.fold(ctx.foldCtx()); // Same as Equals. if (value instanceof BytesRef bytesRef) { value = ipToString(bytesRef); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFolding.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFolding.java index 82fe2c6bddf50..27eec8de59020 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFolding.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFolding.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; public final class ConstantFolding extends OptimizerRules.OptimizerExpressionRule { @@ -17,7 +18,7 @@ public ConstantFolding() { } @Override - public Expression rule(Expression e) { - return e.foldable() ? Literal.of(e) : e; + public Expression rule(Expression e, LogicalOptimizerContext ctx) { + return e.foldable() ? Literal.of(ctx.foldCtx(), e) : e; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConvertStringToByteRef.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConvertStringToByteRef.java index 0604750883f14..b716d8f012d21 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConvertStringToByteRef.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConvertStringToByteRef.java @@ -10,6 +10,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.util.ArrayList; import java.util.List; @@ -21,7 +22,8 @@ public ConvertStringToByteRef() { } @Override - protected Expression rule(Literal lit) { + protected Expression rule(Literal lit, LogicalOptimizerContext ctx) { + // TODO we shouldn't be emitting String into Literals at all Object value = lit.value(); if (value == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java index 747864625e65c..cf4c7f19baafe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; public class FoldNull extends OptimizerRules.OptimizerExpressionRule { @@ -23,7 +24,7 @@ public FoldNull() { } @Override - public Expression rule(Expression e) { + public Expression rule(Expression e, LogicalOptimizerContext ctx) { Expression result = tryReplaceIsNullIsNotNull(e); // convert an aggregate null filter into a false diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRight.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRight.java index d96c73d5ee4f1..6504e6042c33a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRight.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRight.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; public final class LiteralsOnTheRight extends OptimizerRules.OptimizerExpressionRule> { @@ -17,7 +18,7 @@ public LiteralsOnTheRight() { } @Override - public BinaryOperator rule(BinaryOperator be) { + public BinaryOperator rule(BinaryOperator be, LogicalOptimizerContext ctx) { return be.left() instanceof Literal && (be.right() instanceof Literal) == false ? be.swapLeftAndRight() : be; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/OptimizerRules.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/OptimizerRules.java index 2a0b2a6af36aa..169ac2ac8c0fe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/OptimizerRules.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/OptimizerRules.java @@ -8,6 +8,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.util.ReflectionUtils; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.rule.ParameterizedRule; import org.elasticsearch.xpack.esql.rule.Rule; @@ -36,7 +37,10 @@ public final LogicalPlan apply(LogicalPlan plan) { protected abstract LogicalPlan rule(SubPlan plan); } - public abstract static class OptimizerExpressionRule extends Rule { + public abstract static class OptimizerExpressionRule extends ParameterizedRule< + LogicalPlan, + LogicalPlan, + LogicalOptimizerContext> { private final TransformDirection direction; // overriding type token which returns the correct class but does an uncheck cast to LogicalPlan due to its generic bound @@ -49,17 +53,13 @@ public OptimizerExpressionRule(TransformDirection direction) { } @Override - public final LogicalPlan apply(LogicalPlan plan) { + public final LogicalPlan apply(LogicalPlan plan, LogicalOptimizerContext ctx) { return direction == TransformDirection.DOWN - ? plan.transformExpressionsDown(expressionTypeToken, this::rule) - : plan.transformExpressionsUp(expressionTypeToken, this::rule); + ? plan.transformExpressionsDown(expressionTypeToken, e -> rule(e, ctx)) + : plan.transformExpressionsUp(expressionTypeToken, e -> rule(e, ctx)); } - protected LogicalPlan rule(LogicalPlan plan) { - return plan; - } - - protected abstract Expression rule(E e); + protected abstract Expression rule(E e, LogicalOptimizerContext ctx); public Class expressionToken() { return expressionTypeToken; @@ -82,6 +82,7 @@ protected ParameterizedOptimizerRule(TransformDirection direction) { this.direction = direction; } + @Override public final LogicalPlan apply(LogicalPlan plan, P context) { return direction == TransformDirection.DOWN ? plan.transformDown(typeToken(), t -> rule(t, context)) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PartiallyFoldCase.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PartiallyFoldCase.java index 118e4fc170520..0111c7cdd806a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PartiallyFoldCase.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PartiallyFoldCase.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import static org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.TransformDirection.DOWN; @@ -28,7 +29,7 @@ public PartiallyFoldCase() { } @Override - protected Expression rule(Case c) { - return c.partiallyFold(); + protected Expression rule(Case c, LogicalOptimizerContext ctx) { + return c.partiallyFold(ctx.foldCtx()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEmptyRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEmptyRelation.java index 8437b79454884..b6f185c856693 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEmptyRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEmptyRelation.java @@ -12,9 +12,11 @@ import org.elasticsearch.compute.data.BlockUtils; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; @@ -26,15 +28,18 @@ import java.util.List; @SuppressWarnings("removal") -public class PropagateEmptyRelation extends OptimizerRules.OptimizerRule { +public class PropagateEmptyRelation extends OptimizerRules.ParameterizedOptimizerRule { + public PropagateEmptyRelation() { + super(OptimizerRules.TransformDirection.DOWN); + } @Override - protected LogicalPlan rule(UnaryPlan plan) { + protected LogicalPlan rule(UnaryPlan plan, LogicalOptimizerContext ctx) { LogicalPlan p = plan; if (plan.child() instanceof LocalRelation local && local.supplier() == LocalSupplier.EMPTY) { // only care about non-grouped aggs might return something (count) if (plan instanceof Aggregate agg && agg.groupings().isEmpty()) { - List emptyBlocks = aggsFromEmpty(agg.aggregates()); + List emptyBlocks = aggsFromEmpty(ctx.foldCtx(), agg.aggregates()); p = replacePlanByRelation(plan, LocalSupplier.of(emptyBlocks.toArray(Block[]::new))); } else { p = PruneEmptyPlans.skipPlan(plan); @@ -43,14 +48,14 @@ protected LogicalPlan rule(UnaryPlan plan) { return p; } - private List aggsFromEmpty(List aggs) { + private List aggsFromEmpty(FoldContext foldCtx, List aggs) { List blocks = new ArrayList<>(); var blockFactory = PlannerUtils.NON_BREAKING_BLOCK_FACTORY; int i = 0; for (var agg : aggs) { // there needs to be an alias if (Alias.unwrap(agg) instanceof AggregateFunction aggFunc) { - aggOutput(agg, aggFunc, blockFactory, blocks); + aggOutput(foldCtx, agg, aggFunc, blockFactory, blocks); } else { throw new EsqlIllegalArgumentException("Did not expect a non-aliased aggregation {}", agg); } @@ -61,9 +66,15 @@ private List aggsFromEmpty(List aggs) { /** * The folded aggregation output - this variant is for the coordinator/final. */ - protected void aggOutput(NamedExpression agg, AggregateFunction aggFunc, BlockFactory blockFactory, List blocks) { + protected void aggOutput( + FoldContext foldCtx, + NamedExpression agg, + AggregateFunction aggFunc, + BlockFactory blockFactory, + List blocks + ) { // look for count(literal) with literal != null - Object value = aggFunc instanceof Count count && (count.foldable() == false || count.fold() != null) ? 0L : null; + Object value = aggFunc instanceof Count count && (count.foldable() == false || count.fold(foldCtx) != null) ? 0L : null; var wrapper = BlockUtils.wrapperFor(blockFactory, PlannerUtils.toElementType(aggFunc.dataType()), 1); wrapper.accept(value); blocks.add(wrapper.builder().build()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEquals.java index 0bd98db1e1d7a..5a1677f2759e3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEquals.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.util.ArrayList; import java.util.Iterator; @@ -41,17 +42,18 @@ public PropagateEquals() { super(OptimizerRules.TransformDirection.DOWN); } - public Expression rule(BinaryLogic e) { + @Override + public Expression rule(BinaryLogic e, LogicalOptimizerContext ctx) { if (e instanceof And) { - return propagate((And) e); + return propagate((And) e, ctx); } else if (e instanceof Or) { - return propagate((Or) e); + return propagate((Or) e, ctx); } return e; } // combine conjunction - private static Expression propagate(And and) { + private static Expression propagate(And and, LogicalOptimizerContext ctx) { List ranges = new ArrayList<>(); // Only equalities, not-equalities and inequalities with a foldable .right are extracted separately; // the others go into the general 'exps'. @@ -72,7 +74,7 @@ private static Expression propagate(And and) { if (otherEq.right().foldable() && DataType.isDateTime(otherEq.left().dataType()) == false) { for (BinaryComparison eq : equals) { if (otherEq.left().semanticEquals(eq.left())) { - Integer comp = BinaryComparison.compare(eq.right().fold(), otherEq.right().fold()); + Integer comp = BinaryComparison.compare(eq.right().fold(ctx.foldCtx()), otherEq.right().fold(ctx.foldCtx())); if (comp != null) { // var cannot be equal to two different values at the same time if (comp != 0) { @@ -108,7 +110,7 @@ private static Expression propagate(And and) { // check for (BinaryComparison eq : equals) { - Object eqValue = eq.right().fold(); + Object eqValue = eq.right().fold(ctx.foldCtx()); for (Iterator iterator = ranges.iterator(); iterator.hasNext();) { Range range = iterator.next(); @@ -116,7 +118,7 @@ private static Expression propagate(And and) { if (range.value().semanticEquals(eq.left())) { // if equals is outside the interval, evaluate the whole expression to FALSE if (range.lower().foldable()) { - Integer compare = BinaryComparison.compare(range.lower().fold(), eqValue); + Integer compare = BinaryComparison.compare(range.lower().fold(ctx.foldCtx()), eqValue); if (compare != null && ( // eq outside the lower boundary compare > 0 || @@ -126,7 +128,7 @@ private static Expression propagate(And and) { } } if (range.upper().foldable()) { - Integer compare = BinaryComparison.compare(range.upper().fold(), eqValue); + Integer compare = BinaryComparison.compare(range.upper().fold(ctx.foldCtx()), eqValue); if (compare != null && ( // eq outside the upper boundary compare < 0 || @@ -146,7 +148,7 @@ private static Expression propagate(And and) { for (Iterator iter = notEquals.iterator(); iter.hasNext();) { NotEquals neq = iter.next(); if (eq.left().semanticEquals(neq.left())) { - Integer comp = BinaryComparison.compare(eqValue, neq.right().fold()); + Integer comp = BinaryComparison.compare(eqValue, neq.right().fold(ctx.foldCtx())); if (comp != null) { if (comp == 0) { // clashing and conflicting: a = 1 AND a != 1 return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN); @@ -162,7 +164,7 @@ private static Expression propagate(And and) { for (Iterator iter = inequalities.iterator(); iter.hasNext();) { BinaryComparison bc = iter.next(); if (eq.left().semanticEquals(bc.left())) { - Integer compare = BinaryComparison.compare(eqValue, bc.right().fold()); + Integer compare = BinaryComparison.compare(eqValue, bc.right().fold(ctx.foldCtx())); if (compare != null) { if (bc instanceof LessThan || bc instanceof LessThanOrEqual) { // a = 2 AND a a < 3; a = 2 OR a < 1 -> nop // a = 2 OR 3 < a < 5 -> nop; a = 2 OR 1 < a < 3 -> 1 < a < 3; a = 2 OR 0 < a < 1 -> nop // a = 2 OR a != 2 -> TRUE; a = 2 OR a = 5 -> nop; a = 2 OR a != 5 -> a != 5 - private static Expression propagate(Or or) { + private static Expression propagate(Or or, LogicalOptimizerContext ctx) { List exps = new ArrayList<>(); List equals = new ArrayList<>(); // foldable right term Equals List notEquals = new ArrayList<>(); // foldable right term NotEquals @@ -230,13 +232,13 @@ private static Expression propagate(Or or) { // evaluate the impact of each Equal over the different types of Expressions for (Iterator iterEq = equals.iterator(); iterEq.hasNext();) { Equals eq = iterEq.next(); - Object eqValue = eq.right().fold(); + Object eqValue = eq.right().fold(ctx.foldCtx()); boolean removeEquals = false; // Equals OR NotEquals for (NotEquals neq : notEquals) { if (eq.left().semanticEquals(neq.left())) { // a = 2 OR a != ? -> ... - Integer comp = BinaryComparison.compare(eqValue, neq.right().fold()); + Integer comp = BinaryComparison.compare(eqValue, neq.right().fold(ctx.foldCtx())); if (comp != null) { if (comp == 0) { // a = 2 OR a != 2 -> TRUE return TRUE; @@ -257,8 +259,12 @@ private static Expression propagate(Or or) { for (int i = 0; i < ranges.size(); i++) { // might modify list, so use index loop Range range = ranges.get(i); if (eq.left().semanticEquals(range.value())) { - Integer lowerComp = range.lower().foldable() ? BinaryComparison.compare(eqValue, range.lower().fold()) : null; - Integer upperComp = range.upper().foldable() ? BinaryComparison.compare(eqValue, range.upper().fold()) : null; + Integer lowerComp = range.lower().foldable() + ? BinaryComparison.compare(eqValue, range.lower().fold(ctx.foldCtx())) + : null; + Integer upperComp = range.upper().foldable() + ? BinaryComparison.compare(eqValue, range.upper().fold(ctx.foldCtx())) + : null; if (lowerComp != null && lowerComp == 0) { if (range.includeLower() == false) { // a = 2 OR 2 < a < ? -> 2 <= a < ? @@ -312,7 +318,7 @@ private static Expression propagate(Or or) { for (int i = 0; i < inequalities.size(); i++) { BinaryComparison bc = inequalities.get(i); if (eq.left().semanticEquals(bc.left())) { - Integer comp = BinaryComparison.compare(eqValue, bc.right().fold()); + Integer comp = BinaryComparison.compare(eqValue, bc.right().fold(ctx.foldCtx())); if (comp != null) { if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) { if (comp < 0) { // a = 1 OR a > 2 -> nop diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEvalFoldables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEvalFoldables.java index 73eaa9220fd84..66cdc992a91cb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEvalFoldables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEvalFoldables.java @@ -12,19 +12,20 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Filter; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.esql.rule.Rule; +import org.elasticsearch.xpack.esql.rule.ParameterizedRule; /** * Replace any reference attribute with its source, if it does not affect the result. * This avoids ulterior look-ups between attributes and its source across nodes. */ -public final class PropagateEvalFoldables extends Rule { +public final class PropagateEvalFoldables extends ParameterizedRule { @Override - public LogicalPlan apply(LogicalPlan plan) { + public LogicalPlan apply(LogicalPlan plan, LogicalOptimizerContext ctx) { var collectRefs = new AttributeMap(); java.util.function.Function replaceReference = r -> collectRefs.resolve(r, r); @@ -39,7 +40,7 @@ public LogicalPlan apply(LogicalPlan plan) { shouldCollect = c.foldable(); } if (shouldCollect) { - collectRefs.put(a.toAttribute(), Literal.of(c)); + collectRefs.put(a.toAttribute(), Literal.of(ctx.foldCtx(), c)); } }); if (collectRefs.isEmpty()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullable.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullable.java index 738ca83b47e42..e3165180e331c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullable.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullable.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.util.ArrayList; import java.util.LinkedHashSet; @@ -33,7 +34,7 @@ public PropagateNullable() { } @Override - public Expression rule(And and) { + public Expression rule(And and, LogicalOptimizerContext ctx) { List splits = Predicates.splitAnd(and); Set nullExpressions = new LinkedHashSet<>(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimits.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimits.java index 1cacebdf27cd2..969a6bb713eca 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimits.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimits.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.Eval; @@ -20,14 +21,18 @@ import org.elasticsearch.xpack.esql.plan.logical.join.Join; import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; -public final class PushDownAndCombineLimits extends OptimizerRules.OptimizerRule { +public final class PushDownAndCombineLimits extends OptimizerRules.ParameterizedOptimizerRule { + + public PushDownAndCombineLimits() { + super(OptimizerRules.TransformDirection.DOWN); + } @Override - public LogicalPlan rule(Limit limit) { + public LogicalPlan rule(Limit limit, LogicalOptimizerContext ctx) { if (limit.child() instanceof Limit childLimit) { var limitSource = limit.limit(); - var l1 = (int) limitSource.fold(); - var l2 = (int) childLimit.limit().fold(); + var l1 = (int) limitSource.fold(ctx.foldCtx()); + var l2 = (int) childLimit.limit().fold(ctx.foldCtx()); return new Limit(limit.source(), Literal.of(limitSource, Math.min(l1, l2)), childLimit.child()); } else if (limit.child() instanceof UnaryPlan unary) { if (unary instanceof Eval || unary instanceof Project || unary instanceof RegexExtract || unary instanceof Enrich) { @@ -41,7 +46,7 @@ public LogicalPlan rule(Limit limit) { // we add an inner limit to MvExpand and just push down the existing limit, ie. // | MV_EXPAND | LIMIT N -> | LIMIT N | MV_EXPAND (with limit N) var limitSource = limit.limit(); - var limitVal = (int) limitSource.fold(); + var limitVal = (int) limitSource.fold(ctx.foldCtx()); Integer mvxLimit = mvx.limit(); if (mvxLimit == null || mvxLimit > limitVal) { mvx = new MvExpand(mvx.source(), mvx.child(), mvx.target(), mvx.expanded(), limitVal); @@ -54,8 +59,8 @@ public LogicalPlan rule(Limit limit) { else { Limit descendantLimit = descendantLimit(unary); if (descendantLimit != null) { - var l1 = (int) limit.limit().fold(); - var l2 = (int) descendantLimit.limit().fold(); + var l1 = (int) limit.limit().fold(ctx.foldCtx()); + var l2 = (int) descendantLimit.limit().fold(ctx.foldCtx()); if (l2 <= l1) { return new Limit(limit.source(), Literal.of(limit.limit(), l2), limit.child()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatch.java index 1a8f8a164cc1b..7953b2b28eaaa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatch.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatch.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.regex.StringPattern; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.parser.ParsingException; public final class ReplaceRegexMatch extends OptimizerRules.OptimizerExpressionRule> { @@ -23,7 +24,7 @@ public ReplaceRegexMatch() { } @Override - public Expression rule(RegexMatch regexMatch) { + public Expression rule(RegexMatch regexMatch, LogicalOptimizerContext ctx) { Expression e = regexMatch; StringPattern pattern = regexMatch.pattern(); boolean matchesAll; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRowAsLocalRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRowAsLocalRelation.java index eebeb1dc14f48..9e7b6ce80422d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRowAsLocalRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRowAsLocalRelation.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical; import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; @@ -17,13 +18,16 @@ import java.util.ArrayList; import java.util.List; -public final class ReplaceRowAsLocalRelation extends OptimizerRules.OptimizerRule { +public final class ReplaceRowAsLocalRelation extends OptimizerRules.ParameterizedOptimizerRule { + public ReplaceRowAsLocalRelation() { + super(OptimizerRules.TransformDirection.DOWN); + } @Override - protected LogicalPlan rule(Row row) { + protected LogicalPlan rule(Row row, LogicalOptimizerContext context) { var fields = row.fields(); List values = new ArrayList<>(fields.size()); - fields.forEach(f -> values.add(f.child().fold())); + fields.forEach(f -> values.add(f.child().fold(context.foldCtx()))); var blocks = BlockUtils.fromListRow(PlannerUtils.NON_BREAKING_BLOCK_FACTORY, values); return new LocalRelation(row.source(), row.output(), LocalSupplier.of(blocks)); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsFilteredAggWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsFilteredAggWithEval.java index 2cafcc2e07052..a7e56a5f25fc8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsFilteredAggWithEval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsFilteredAggWithEval.java @@ -49,7 +49,7 @@ protected LogicalPlan rule(Aggregate aggregate) { && alias.child() instanceof AggregateFunction aggFunction && aggFunction.hasFilter() && aggFunction.filter() instanceof Literal literal - && Boolean.FALSE.equals(literal.fold())) { + && Boolean.FALSE.equals(literal.value())) { Object value = aggFunction instanceof Count || aggFunction instanceof CountDistinct ? 0L : null; Alias newAlias = alias.replaceChild(Literal.of(aggFunction, value)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java index 0fea7cf8ddc1f..053441bce5e1f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.InsensitiveEquals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; public class ReplaceStringCasingWithInsensitiveEquals extends OptimizerRules.OptimizerExpressionRule { @@ -26,30 +27,35 @@ public ReplaceStringCasingWithInsensitiveEquals() { } @Override - protected Expression rule(ScalarFunction sf) { + protected Expression rule(ScalarFunction sf, LogicalOptimizerContext ctx) { Expression e = sf; if (sf instanceof BinaryComparison bc) { - e = rewriteBinaryComparison(sf, bc, false); + e = rewriteBinaryComparison(ctx, sf, bc, false); } else if (sf instanceof Not not && not.field() instanceof BinaryComparison bc) { - e = rewriteBinaryComparison(sf, bc, true); + e = rewriteBinaryComparison(ctx, sf, bc, true); } return e; } - private static Expression rewriteBinaryComparison(ScalarFunction sf, BinaryComparison bc, boolean negated) { + private static Expression rewriteBinaryComparison( + LogicalOptimizerContext ctx, + ScalarFunction sf, + BinaryComparison bc, + boolean negated + ) { Expression e = sf; if (bc.left() instanceof ChangeCase changeCase && bc.right().foldable()) { if (bc instanceof Equals) { - e = replaceChangeCase(bc, changeCase, negated); + e = replaceChangeCase(ctx, bc, changeCase, negated); } else if (bc instanceof NotEquals) { // not actually used currently, `!=` is built as `NOT(==)` already - e = replaceChangeCase(bc, changeCase, negated == false); + e = replaceChangeCase(ctx, bc, changeCase, negated == false); } } return e; } - private static Expression replaceChangeCase(BinaryComparison bc, ChangeCase changeCase, boolean negated) { - var foldedRight = BytesRefs.toString(bc.right().fold()); + private static Expression replaceChangeCase(LogicalOptimizerContext ctx, BinaryComparison bc, ChangeCase changeCase, boolean negated) { + var foldedRight = BytesRefs.toString(bc.right().fold(ctx.foldCtx())); var field = unwrapCase(changeCase.field()); var e = changeCase.caseType().matchesCase(foldedRight) ? new InsensitiveEquals(bc.source(), field, bc.right()) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SimplifyComparisonsArithmetics.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SimplifyComparisonsArithmetics.java index d3a9970896c16..60ff161651f2d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SimplifyComparisonsArithmetics.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SimplifyComparisonsArithmetics.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -15,6 +16,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.BinaryComparisonInversible; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.time.DateTimeException; import java.util.List; @@ -41,20 +43,20 @@ public SimplifyComparisonsArithmetics(BiFunction ty } @Override - protected Expression rule(BinaryComparison bc) { + protected Expression rule(BinaryComparison bc, LogicalOptimizerContext ctx) { // optimize only once the expression has a literal on the right side of the binary comparison if (bc.right() instanceof Literal) { if (bc.left() instanceof ArithmeticOperation) { - return simplifyBinaryComparison(bc); + return simplifyBinaryComparison(ctx.foldCtx(), bc); } if (bc.left() instanceof Neg) { - return foldNegation(bc); + return foldNegation(ctx.foldCtx(), bc); } } return bc; } - private Expression simplifyBinaryComparison(BinaryComparison comparison) { + private Expression simplifyBinaryComparison(FoldContext foldContext, BinaryComparison comparison) { ArithmeticOperation operation = (ArithmeticOperation) comparison.left(); // Use symbol comp: SQL operations aren't available in this package (as dependencies) String opSymbol = operation.symbol(); @@ -64,9 +66,9 @@ private Expression simplifyBinaryComparison(BinaryComparison comparison) { } OperationSimplifier simplification = null; if (isMulOrDiv(opSymbol)) { - simplification = new MulDivSimplifier(comparison); + simplification = new MulDivSimplifier(foldContext, comparison); } else if (opSymbol.equals(ADD.symbol()) || opSymbol.equals(SUB.symbol())) { - simplification = new AddSubSimplifier(comparison); + simplification = new AddSubSimplifier(foldContext, comparison); } return (simplification == null || simplification.isUnsafe(typesCompatible)) ? comparison : simplification.apply(); @@ -76,16 +78,16 @@ private static boolean isMulOrDiv(String opSymbol) { return opSymbol.equals(MUL.symbol()) || opSymbol.equals(DIV.symbol()); } - private static Expression foldNegation(BinaryComparison bc) { + private static Expression foldNegation(FoldContext ctx, BinaryComparison bc) { Literal bcLiteral = (Literal) bc.right(); - Expression literalNeg = tryFolding(new Neg(bcLiteral.source(), bcLiteral)); + Expression literalNeg = tryFolding(ctx, new Neg(bcLiteral.source(), bcLiteral)); return literalNeg == null ? bc : bc.reverse().replaceChildren(asList(((Neg) bc.left()).field(), literalNeg)); } - private static Expression tryFolding(Expression expression) { + private static Expression tryFolding(FoldContext ctx, Expression expression) { if (expression.foldable()) { try { - expression = new Literal(expression.source(), expression.fold(), expression.dataType()); + expression = new Literal(expression.source(), expression.fold(ctx), expression.dataType()); } catch (ArithmeticException | DateTimeException e) { // null signals that folding would result in an over-/underflow (such as Long.MAX_VALUE+1); the optimisation is skipped. expression = null; @@ -95,6 +97,7 @@ private static Expression tryFolding(Expression expression) { } private abstract static class OperationSimplifier { + final FoldContext foldContext; final BinaryComparison comparison; final Literal bcLiteral; final ArithmeticOperation operation; @@ -102,7 +105,8 @@ private abstract static class OperationSimplifier { final Expression opRight; final Literal opLiteral; - OperationSimplifier(BinaryComparison comparison) { + OperationSimplifier(FoldContext foldContext, BinaryComparison comparison) { + this.foldContext = foldContext; this.comparison = comparison; operation = (ArithmeticOperation) comparison.left(); bcLiteral = (Literal) comparison.right(); @@ -151,7 +155,7 @@ final Expression apply() { Expression bcRightExpression = ((BinaryComparisonInversible) operation).binaryComparisonInverse() .create(bcl.source(), bcl, opRight); - bcRightExpression = tryFolding(bcRightExpression); + bcRightExpression = tryFolding(foldContext, bcRightExpression); return bcRightExpression != null ? postProcess((BinaryComparison) comparison.replaceChildren(List.of(opLeft, bcRightExpression))) : comparison; @@ -169,8 +173,8 @@ Expression postProcess(BinaryComparison binaryComparison) { private static class AddSubSimplifier extends OperationSimplifier { - AddSubSimplifier(BinaryComparison comparison) { - super(comparison); + AddSubSimplifier(FoldContext foldContext, BinaryComparison comparison) { + super(foldContext, comparison); } @Override @@ -182,7 +186,7 @@ boolean isOpUnsafe() { if (operation.symbol().equals(SUB.symbol()) && opRight instanceof Literal == false) { // such as: 1 - x > -MAX // if next simplification step would fail on overflow anyways, skip the optimisation already - return tryFolding(new Sub(EMPTY, opLeft, bcLiteral)) == null; + return tryFolding(foldContext, new Sub(EMPTY, opLeft, bcLiteral)) == null; } return false; @@ -194,8 +198,8 @@ private static class MulDivSimplifier extends OperationSimplifier { private final boolean isDiv; // and not MUL. private final int opRightSign; // sign of the right operand in: (left) (op) (right) (comp) (literal) - MulDivSimplifier(BinaryComparison comparison) { - super(comparison); + MulDivSimplifier(FoldContext foldContext, BinaryComparison comparison) { + super(foldContext, comparison); isDiv = operation.symbol().equals(DIV.symbol()); opRightSign = sign(opRight); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnLimitZero.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnLimitZero.java index 5d98d941bb207..c6d62dee0ba42 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnLimitZero.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnLimitZero.java @@ -7,14 +7,19 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.plan.logical.Limit; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; -public final class SkipQueryOnLimitZero extends OptimizerRules.OptimizerRule { +public final class SkipQueryOnLimitZero extends OptimizerRules.ParameterizedOptimizerRule { + public SkipQueryOnLimitZero() { + super(OptimizerRules.TransformDirection.DOWN); + } + @Override - protected LogicalPlan rule(Limit limit) { + protected LogicalPlan rule(Limit limit, LogicalOptimizerContext ctx) { if (limit.limit().foldable()) { - if (Integer.valueOf(0).equals((limit.limit().fold()))) { + if (Integer.valueOf(0).equals((limit.limit().fold(ctx.foldCtx())))) { return PruneEmptyPlans.skipPlan(limit); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java index 9e9ae6a9a559d..870464feb4867 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.util.ArrayList; import java.util.List; @@ -25,7 +26,7 @@ public SplitInWithFoldableValue() { } @Override - public Expression rule(In in) { + public Expression rule(In in, LogicalOptimizerContext ctx) { if (in.value().foldable()) { List foldables = new ArrayList<>(in.list().size()); List nonFoldables = new ArrayList<>(in.list().size()); @@ -36,7 +37,7 @@ public Expression rule(In in) { nonFoldables.add(e); } }); - if (foldables.size() > 0 && nonFoldables.size() > 0) { + if (foldables.isEmpty() == false && nonFoldables.isEmpty() == false) { In withFoldables = new In(in.source(), in.value(), foldables); In withoutFoldables = new In(in.source(), in.value(), nonFoldables); return new Or(in.source(), withFoldables, withoutFoldables); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java index c8369d2b08a34..62a00b79d7333 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.OptimizerExpressionRule; import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.TransformDirection; @@ -21,7 +22,7 @@ public SubstituteFilteredExpression() { } @Override - protected Expression rule(FilteredExpression filteredExpression) { + protected Expression rule(FilteredExpression filteredExpression, LogicalOptimizerContext ctx) { return filteredExpression.surrogate(); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java index 93512d80e1708..4b68ee941bc92 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; /** * Currently this works similarly to SurrogateExpression, leaving the logic inside the expressions, @@ -23,7 +24,7 @@ public SubstituteSpatialSurrogates() { } @Override - protected SpatialRelatesFunction rule(SpatialRelatesFunction function) { + protected SpatialRelatesFunction rule(SpatialRelatesFunction function, LogicalOptimizerContext ctx) { return function.surrogate(); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/LocalPropagateEmptyRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/LocalPropagateEmptyRelation.java index d29da1354ef3c..9259a50d5ff9e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/LocalPropagateEmptyRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/LocalPropagateEmptyRelation.java @@ -11,6 +11,7 @@ import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BlockUtils; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; @@ -25,19 +26,24 @@ * Local aggregation can only produce intermediate state that get wired into the global agg. */ public class LocalPropagateEmptyRelation extends PropagateEmptyRelation { - /** * Local variant of the aggregation that returns the intermediate value. */ @Override - protected void aggOutput(NamedExpression agg, AggregateFunction aggFunc, BlockFactory blockFactory, List blocks) { + protected void aggOutput( + FoldContext foldCtx, + NamedExpression agg, + AggregateFunction aggFunc, + BlockFactory blockFactory, + List blocks + ) { List output = AbstractPhysicalOperationProviders.intermediateAttributes(List.of(agg), List.of()); for (Attribute o : output) { DataType dataType = o.dataType(); // boolean right now is used for the internal #seen so always return true var value = dataType == DataType.BOOLEAN ? true // look for count(literal) with literal != null - : aggFunc instanceof Count count && (count.foldable() == false || count.fold() != null) ? 0L + : aggFunc instanceof Count count && (count.foldable() == false || count.fold(foldCtx) != null) ? 0L // otherwise nullify : null; var wrapper = BlockUtils.wrapperFor(blockFactory, PlannerUtils.toElementType(dataType), 1); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java index dfb1dbc8bc8f3..afeab28745c65 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeMap; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; @@ -76,22 +77,33 @@ public class EnableSpatialDistancePushdown extends PhysicalOptimizerRules.Parame protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext ctx) { PhysicalPlan plan = filterExec; if (filterExec.child() instanceof EsQueryExec esQueryExec) { - plan = rewrite(filterExec, esQueryExec, LucenePushdownPredicates.from(ctx.searchStats())); + plan = rewrite(ctx.foldCtx(), filterExec, esQueryExec, LucenePushdownPredicates.from(ctx.searchStats())); } else if (filterExec.child() instanceof EvalExec evalExec && evalExec.child() instanceof EsQueryExec esQueryExec) { - plan = rewriteBySplittingFilter(filterExec, evalExec, esQueryExec, LucenePushdownPredicates.from(ctx.searchStats())); + plan = rewriteBySplittingFilter( + ctx.foldCtx(), + filterExec, + evalExec, + esQueryExec, + LucenePushdownPredicates.from(ctx.searchStats()) + ); } return plan; } - private FilterExec rewrite(FilterExec filterExec, EsQueryExec esQueryExec, LucenePushdownPredicates lucenePushdownPredicates) { + private FilterExec rewrite( + FoldContext ctx, + FilterExec filterExec, + EsQueryExec esQueryExec, + LucenePushdownPredicates lucenePushdownPredicates + ) { // Find and rewrite any binary comparisons that involve a distance function and a literal var rewritten = filterExec.condition().transformDown(EsqlBinaryComparison.class, comparison -> { ComparisonType comparisonType = ComparisonType.from(comparison.getFunctionType()); if (comparison.left() instanceof StDistance dist && comparison.right().foldable()) { - return rewriteComparison(comparison, dist, comparison.right(), comparisonType); + return rewriteComparison(ctx, comparison, dist, comparison.right(), comparisonType); } else if (comparison.right() instanceof StDistance dist && comparison.left().foldable()) { - return rewriteComparison(comparison, dist, comparison.left(), ComparisonType.invert(comparisonType)); + return rewriteComparison(ctx, comparison, dist, comparison.left(), ComparisonType.invert(comparisonType)); } return comparison; }); @@ -120,6 +132,7 @@ private FilterExec rewrite(FilterExec filterExec, EsQueryExec esQueryExec, Lucen * */ private PhysicalPlan rewriteBySplittingFilter( + FoldContext ctx, FilterExec filterExec, EvalExec evalExec, EsQueryExec esQueryExec, @@ -142,7 +155,7 @@ private PhysicalPlan rewriteBySplittingFilter( for (Expression exp : splitAnd(filterExec.condition())) { Expression resExp = exp.transformUp(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r)); // Find and rewrite any binary comparisons that involve a distance function and a literal - var rewritten = rewriteDistanceFilters(resExp, distances); + var rewritten = rewriteDistanceFilters(ctx, resExp, distances); // If all pushable StDistance functions were found and re-written, we need to re-write the FILTER/EVAL combination if (rewritten.equals(resExp) == false && canPushToSource(rewritten, lucenePushdownPredicates)) { pushable.add(rewritten); @@ -181,40 +194,42 @@ private Map getPushableDistances(List aliases, Lucene return distances; } - private Expression rewriteDistanceFilters(Expression expr, Map distances) { + private Expression rewriteDistanceFilters(FoldContext ctx, Expression expr, Map distances) { return expr.transformDown(EsqlBinaryComparison.class, comparison -> { ComparisonType comparisonType = ComparisonType.from(comparison.getFunctionType()); if (comparison.left() instanceof ReferenceAttribute r && distances.containsKey(r.id()) && comparison.right().foldable()) { StDistance dist = distances.get(r.id()); - return rewriteComparison(comparison, dist, comparison.right(), comparisonType); + return rewriteComparison(ctx, comparison, dist, comparison.right(), comparisonType); } else if (comparison.right() instanceof ReferenceAttribute r && distances.containsKey(r.id()) && comparison.left().foldable()) { StDistance dist = distances.get(r.id()); - return rewriteComparison(comparison, dist, comparison.left(), ComparisonType.invert(comparisonType)); + return rewriteComparison(ctx, comparison, dist, comparison.left(), ComparisonType.invert(comparisonType)); } return comparison; }); } private Expression rewriteComparison( + FoldContext ctx, EsqlBinaryComparison comparison, StDistance dist, Expression literal, ComparisonType comparisonType ) { - Object value = literal.fold(); + Object value = literal.fold(ctx); if (value instanceof Number number) { if (dist.right().foldable()) { - return rewriteDistanceFilter(comparison, dist.left(), dist.right(), number, comparisonType); + return rewriteDistanceFilter(ctx, comparison, dist.left(), dist.right(), number, comparisonType); } else if (dist.left().foldable()) { - return rewriteDistanceFilter(comparison, dist.right(), dist.left(), number, comparisonType); + return rewriteDistanceFilter(ctx, comparison, dist.right(), dist.left(), number, comparisonType); } } return comparison; } private Expression rewriteDistanceFilter( + FoldContext ctx, EsqlBinaryComparison comparison, Expression spatialExp, Expression literalExp, @@ -222,7 +237,7 @@ private Expression rewriteDistanceFilter( ComparisonType comparisonType ) { DataType shapeDataType = getShapeDataType(spatialExp); - Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(literalExp); + Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(ctx, literalExp); if (geometry instanceof Point point) { double distance = number.doubleValue(); Source source = comparison.source(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java index 2b531257e594a..24df3c1db234e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.core.expression.AttributeMap; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; @@ -61,7 +62,7 @@ public class PushTopNToSource extends PhysicalOptimizerRules.ParameterizedOptimi @Override protected PhysicalPlan rule(TopNExec topNExec, LocalPhysicalOptimizerContext ctx) { - Pushable pushable = evaluatePushable(topNExec, LucenePushdownPredicates.from(ctx.searchStats())); + Pushable pushable = evaluatePushable(ctx.foldCtx(), topNExec, LucenePushdownPredicates.from(ctx.searchStats())); return pushable.rewrite(topNExec); } @@ -95,18 +96,18 @@ private EsQueryExec.Sort sort() { return new EsQueryExec.GeoDistanceSort(fieldAttribute.exactAttribute(), order.direction(), point.getLat(), point.getLon()); } - private static PushableGeoDistance from(StDistance distance, Order order) { + private static PushableGeoDistance from(FoldContext ctx, StDistance distance, Order order) { if (distance.left() instanceof Attribute attr && distance.right().foldable()) { - return from(attr, distance.right(), order); + return from(ctx, attr, distance.right(), order); } else if (distance.right() instanceof Attribute attr && distance.left().foldable()) { - return from(attr, distance.left(), order); + return from(ctx, attr, distance.left(), order); } return null; } - private static PushableGeoDistance from(Attribute attr, Expression foldable, Order order) { + private static PushableGeoDistance from(FoldContext ctx, Attribute attr, Expression foldable, Order order) { if (attr instanceof FieldAttribute fieldAttribute) { - Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(foldable); + Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(ctx, foldable); if (geometry instanceof Point point) { return new PushableGeoDistance(fieldAttribute, order, point); } @@ -122,7 +123,7 @@ public PhysicalPlan rewrite(TopNExec topNExec) { } } - private static Pushable evaluatePushable(TopNExec topNExec, LucenePushdownPredicates lucenePushdownPredicates) { + private static Pushable evaluatePushable(FoldContext ctx, TopNExec topNExec, LucenePushdownPredicates lucenePushdownPredicates) { PhysicalPlan child = topNExec.child(); if (child instanceof EsQueryExec queryExec && queryExec.canPushSorts() @@ -164,7 +165,7 @@ && canPushDownOrders(topNExec.order(), lucenePushdownPredicates)) { if (distances.containsKey(resolvedAttribute.id())) { StDistance distance = distances.get(resolvedAttribute.id()); StDistance d = (StDistance) distance.transformDown(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r)); - PushableGeoDistance pushableGeoDistance = PushableGeoDistance.from(d, order); + PushableGeoDistance pushableGeoDistance = PushableGeoDistance.from(ctx, d, order); if (pushableGeoDistance != null) { pushableSorts.add(pushableGeoDistance.sort()); } else { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index 81d43bc68b79e..eb81446f9ddea 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; @@ -686,12 +687,20 @@ public Expression visitRegexBooleanExpression(EsqlBaseParser.RegexBooleanExpress RegexMatch result = switch (type) { case EsqlBaseParser.LIKE -> { try { - yield new WildcardLike(source, left, new WildcardPattern(pattern.fold().toString())); + yield new WildcardLike( + source, + left, + new WildcardPattern(pattern.fold(FoldContext.small() /* TODO remove me */).toString()) + ); } catch (InvalidArgumentException e) { throw new ParsingException(source, "Invalid pattern for LIKE [{}]: [{}]", pattern, e.getMessage()); } } - case EsqlBaseParser.RLIKE -> new RLike(source, left, new RLikePattern(pattern.fold().toString())); + case EsqlBaseParser.RLIKE -> new RLike( + source, + left, + new RLikePattern(pattern.fold(FoldContext.small() /* TODO remove me */).toString()) + ); default -> throw new ParsingException("Invalid predicate type for [{}]", source.text()); }; return ctx.NOT() == null ? result : new Not(source, result); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index 49d77bc36fb2e..4edd0470058db 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; @@ -157,7 +158,7 @@ public PlanFactory visitEvalCommand(EsqlBaseParser.EvalCommandContext ctx) { public PlanFactory visitGrokCommand(EsqlBaseParser.GrokCommandContext ctx) { return p -> { Source source = source(ctx); - String pattern = visitString(ctx.string()).fold().toString(); + String pattern = visitString(ctx.string()).fold(FoldContext.small() /* TODO remove me */).toString(); Grok.Parser grokParser; try { grokParser = Grok.pattern(source, pattern); @@ -188,7 +189,7 @@ private void validateGrokPattern(Source source, Grok.Parser grokParser, String p @Override public PlanFactory visitDissectCommand(EsqlBaseParser.DissectCommandContext ctx) { return p -> { - String pattern = visitString(ctx.string()).fold().toString(); + String pattern = visitString(ctx.string()).fold(FoldContext.small() /* TODO remove me */).toString(); Map options = visitCommandOptions(ctx.commandOptions()); String appendSeparator = ""; for (Map.Entry item : options.entrySet()) { @@ -243,7 +244,7 @@ public Map visitCommandOptions(EsqlBaseParser.CommandOptionsCont } Map result = new HashMap<>(); for (EsqlBaseParser.CommandOptionContext option : ctx.commandOption()) { - result.put(visitIdentifier(option.identifier()), expression(option.constant()).fold()); + result.put(visitIdentifier(option.identifier()), expression(option.constant()).fold(FoldContext.small() /* TODO remove me */)); } return result; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java index 6755a7fa30af9..9b81060349815 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; @@ -149,7 +150,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeNamedWriteable(policyName()); out.writeNamedWriteable(matchField()); if (out.getTransportVersion().before(TransportVersions.V_8_13_0)) { - out.writeString(BytesRefs.toString(policyName().fold())); // old policy name + out.writeString(BytesRefs.toString(policyName().fold(FoldContext.small() /* TODO remove me */))); // old policy name } policy().writeTo(out); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java index 57ba1c8016feb..072bae21da2a3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.evaluator.EvalMapper; @@ -47,9 +48,11 @@ public abstract class AbstractPhysicalOperationProviders implements PhysicalOperationProviders { private final AggregateMapper aggregateMapper = new AggregateMapper(); + private final FoldContext foldContext; private final AnalysisRegistry analysisRegistry; - AbstractPhysicalOperationProviders(AnalysisRegistry analysisRegistry) { + AbstractPhysicalOperationProviders(FoldContext foldContext, AnalysisRegistry analysisRegistry) { + this.foldContext = foldContext; this.analysisRegistry = analysisRegistry; } @@ -251,6 +254,7 @@ public static List intermediateAttributes(List aggregates, AggregatorMode mode, Layout layout, @@ -311,7 +315,11 @@ else if (mode == AggregatorMode.FINAL || mode == AggregatorMode.INTERMEDIATE) { // apply the filter only in the initial phase - as the rest of the data is already filtered if (aggregateFunction.hasFilter() && mode.isInputPartial() == false) { - EvalOperator.ExpressionEvaluator.Factory evalFactory = EvalMapper.toEvaluator(aggregateFunction.filter(), layout); + EvalOperator.ExpressionEvaluator.Factory evalFactory = EvalMapper.toEvaluator( + foldContext, + aggregateFunction.filter(), + layout + ); aggSupplier = new FilteredAggregatorFunctionSupplier(aggSupplier, evalFactory); } consumer.accept(new AggFunctionSupplierContext(aggSupplier, mode)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index b1fe0e7a7cf54..8b63a146f2e5d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -47,6 +47,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; @@ -94,8 +95,8 @@ public interface ShardContext extends org.elasticsearch.compute.lucene.ShardCont private final List shardContexts; - public EsPhysicalOperationProviders(List shardContexts, AnalysisRegistry analysisRegistry) { - super(analysisRegistry); + public EsPhysicalOperationProviders(FoldContext foldContext, List shardContexts, AnalysisRegistry analysisRegistry) { + super(foldContext, analysisRegistry); this.shardContexts = shardContexts; } @@ -161,7 +162,7 @@ public final PhysicalOperation sourcePhysicalOperation(EsQueryExec esQueryExec, List sorts = esQueryExec.sorts(); assert esQueryExec.estimatedRowSize() != null : "estimated row size not initialized"; int rowEstimatedSize = esQueryExec.estimatedRowSize(); - int limit = esQueryExec.limit() != null ? (Integer) esQueryExec.limit().fold() : NO_LIMIT; + int limit = esQueryExec.limit() != null ? (Integer) esQueryExec.limit().fold(context.foldCtx()) : NO_LIMIT; boolean scoring = esQueryExec.attrs() .stream() .anyMatch(a -> a instanceof MetadataAttribute && a.name().equals(MetadataAttribute.SCORE)); @@ -217,7 +218,7 @@ public LuceneCountOperator.Factory countSource(LocalExecutionPlannerContext cont querySupplier(queryBuilder), context.queryPragmas().dataPartitioning(), context.queryPragmas().taskConcurrency(), - limit == null ? NO_LIMIT : (Integer) limit.fold() + limit == null ? NO_LIMIT : (Integer) limit.fold(context.foldCtx()) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index a1765977ee9c2..c185bd5729879 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; @@ -144,7 +145,7 @@ public static void checkInsensitiveComparison(InsensitiveEquals bc) { static Query translate(InsensitiveEquals bc) { TypedAttribute attribute = checkIsPushableAttribute(bc.left()); Source source = bc.source(); - BytesRef value = BytesRefs.toBytesRef(valueOf(bc.right())); + BytesRef value = BytesRefs.toBytesRef(valueOf(FoldContext.small() /* TODO remove me */, bc.right())); String name = pushableAttributeName(attribute); return new TermQuery(source, name, value.utf8ToString(), true); } @@ -188,7 +189,7 @@ static Query translate(BinaryComparison bc, TranslatorHandler handler) { TypedAttribute attribute = checkIsPushableAttribute(bc.left()); Source source = bc.source(); String name = handler.nameOf(attribute); - Object result = bc.right().fold(); + Object result = bc.right().fold(FoldContext.small() /* TODO remove me */); Object value = result; String format = null; boolean isDateLiteralComparison = false; @@ -269,7 +270,7 @@ private static Query translateOutOfRangeComparisons(BinaryComparison bc) { return null; } Source source = bc.source(); - Object value = valueOf(bc.right()); + Object value = valueOf(FoldContext.small() /* TODO remove me */, bc.right()); // Comparisons with multi-values always return null in ESQL. if (value instanceof List) { @@ -369,7 +370,7 @@ public static Query doTranslate(ScalarFunction f, TranslatorHandler handler) { if (f instanceof CIDRMatch cm) { if (cm.ipField() instanceof FieldAttribute fa && Expressions.foldable(cm.matches())) { String targetFieldName = handler.nameOf(fa.exactAttribute()); - Set set = new LinkedHashSet<>(Expressions.fold(cm.matches())); + Set set = new LinkedHashSet<>(Expressions.fold(FoldContext.small() /* TODO remove me */, cm.matches())); Query query = new TermsQuery(f.source(), targetFieldName, set); // CIDR_MATCH applies only to single values. @@ -420,7 +421,7 @@ static Query translate( String name = handler.nameOf(attribute); try { - Geometry shape = SpatialRelatesUtils.makeGeometryFromLiteral(constantExpression); + Geometry shape = SpatialRelatesUtils.makeGeometryFromLiteral(FoldContext.small() /* TODO remove me */, constantExpression); return new SpatialRelatesQuery(bc.source(), name, bc.queryRelation(), shape, attribute.dataType()); } catch (IllegalArgumentException e) { throw new QlIllegalArgumentException(e.getMessage(), e); @@ -461,7 +462,7 @@ private static Query translate(In in, TranslatorHandler handler) { queries.add(query); } } else { - terms.add(valueOf(rhs)); + terms.add(valueOf(FoldContext.small() /* TODO remove me */, rhs)); } } } @@ -487,8 +488,8 @@ public static Query doTranslate(Range r, TranslatorHandler handler) { } private static RangeQuery translate(Range r, TranslatorHandler handler) { - Object lower = valueOf(r.lower()); - Object upper = valueOf(r.upper()); + Object lower = valueOf(FoldContext.small() /* TODO remove me */, r.lower()); + Object upper = valueOf(FoldContext.small() /* TODO remove me */, r.upper()); String format = null; DataType dataType = r.value().dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index af38551c1ad06..ecd0284c7cb57 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -22,7 +22,6 @@ import org.elasticsearch.compute.operator.Driver; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator.EvalOperatorFactory; -import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.compute.operator.FilterOperator.FilterOperatorFactory; import org.elasticsearch.compute.operator.LocalSourceOperator; import org.elasticsearch.compute.operator.LocalSourceOperator.LocalSourceFactory; @@ -55,6 +54,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; @@ -161,13 +161,14 @@ public LocalExecutionPlanner( /** * turn the given plan into a list of drivers to execute */ - public LocalExecutionPlan plan(PhysicalPlan localPhysicalPlan) { + public LocalExecutionPlan plan(FoldContext foldCtx, PhysicalPlan localPhysicalPlan) { var context = new LocalExecutionPlannerContext( new ArrayList<>(), new Holder<>(DriverParallelism.SINGLE), configuration.pragmas(), bigArrays, blockFactory, + foldCtx, settings ); @@ -397,7 +398,7 @@ private PhysicalOperation planEval(EvalExec eval, LocalExecutionPlannerContext c PhysicalOperation source = plan(eval.child(), context); for (Alias field : eval.fields()) { - var evaluatorSupplier = EvalMapper.toEvaluator(field.child(), source.layout); + var evaluatorSupplier = EvalMapper.toEvaluator(context.foldCtx(), field.child(), source.layout); Layout.Builder layout = source.layout.builder(); layout.append(field.toAttribute()); source = source.with(new EvalOperatorFactory(evaluatorSupplier), layout.build()); @@ -418,7 +419,7 @@ private PhysicalOperation planDissect(DissectExec dissect, LocalExecutionPlanner source = source.with( new StringExtractOperator.StringExtractOperatorFactory( patternNames, - EvalMapper.toEvaluator(expr, layout), + EvalMapper.toEvaluator(context.foldCtx(), expr, layout), () -> (input) -> dissect.parser().parser().parse(input) ), layout @@ -450,7 +451,7 @@ private PhysicalOperation planGrok(GrokExec grok, LocalExecutionPlannerContext c source = source.with( new ColumnExtractOperator.Factory( types, - EvalMapper.toEvaluator(grok.inputExpression(), layout), + EvalMapper.toEvaluator(context.foldCtx(), grok.inputExpression(), layout), () -> new GrokEvaluatorExtracter(grok.pattern().grok(), grok.pattern().pattern(), fieldToPos, fieldToType) ), layout @@ -599,10 +600,6 @@ private PhysicalOperation planLookupJoin(LookupJoinExec join, LocalExecutionPlan ); } - private ExpressionEvaluator.Factory toEvaluator(Expression exp, Layout layout) { - return EvalMapper.toEvaluator(exp, layout); - } - private PhysicalOperation planLocal(LocalSourceExec localSourceExec, LocalExecutionPlannerContext context) { Layout.Builder layout = new Layout.Builder(); layout.append(localSourceExec.output()); @@ -657,12 +654,15 @@ private PhysicalOperation planProject(ProjectExec project, LocalExecutionPlanner private PhysicalOperation planFilter(FilterExec filter, LocalExecutionPlannerContext context) { PhysicalOperation source = plan(filter.child(), context); // TODO: should this be extracted into a separate eval block? - return source.with(new FilterOperatorFactory(toEvaluator(filter.condition(), source.layout)), source.layout); + return source.with( + new FilterOperatorFactory(EvalMapper.toEvaluator(context.foldCtx(), filter.condition(), source.layout)), + source.layout + ); } private PhysicalOperation planLimit(LimitExec limit, LocalExecutionPlannerContext context) { PhysicalOperation source = plan(limit.child(), context); - return source.with(new Factory((Integer) limit.limit().fold()), source.layout); + return source.with(new Factory((Integer) limit.limit().fold(context.foldCtx)), source.layout); } private PhysicalOperation planMvExpand(MvExpandExec mvExpandExec, LocalExecutionPlannerContext context) { @@ -783,6 +783,7 @@ public record LocalExecutionPlannerContext( QueryPragmas queryPragmas, BigArrays bigArrays, BlockFactory blockFactory, + FoldContext foldCtx, Settings settings ) { void addDriverFactory(DriverFactory driverFactory) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index 5325145a77ade..2f4368155069f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -159,13 +160,18 @@ private static , E extends T> void forEachUpWithChildren( } } - public static PhysicalPlan localPlan(List searchContexts, Configuration configuration, PhysicalPlan plan) { - return localPlan(configuration, plan, SearchContextStats.from(searchContexts)); + public static PhysicalPlan localPlan( + List searchContexts, + Configuration configuration, + FoldContext foldCtx, + PhysicalPlan plan + ) { + return localPlan(configuration, foldCtx, plan, SearchContextStats.from(searchContexts)); } - public static PhysicalPlan localPlan(Configuration configuration, PhysicalPlan plan, SearchStats searchStats) { - final var logicalOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, searchStats)); - var physicalOptimizer = new LocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(configuration, searchStats)); + public static PhysicalPlan localPlan(Configuration configuration, FoldContext foldCtx, PhysicalPlan plan, SearchStats searchStats) { + final var logicalOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, foldCtx, searchStats)); + var physicalOptimizer = new LocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(configuration, foldCtx, searchStats)); return localPlan(plan, logicalOptimizer, physicalOptimizer); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java index 334875927eb96..4dea8a50b5c17 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java @@ -14,6 +14,9 @@ import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; class TypeConverter { @@ -33,19 +36,26 @@ public static TypeConverter fromConvertFunction(AbstractConvertFunction convertF BigArrays.NON_RECYCLING_INSTANCE ) ); - return new TypeConverter( - convertFunction.functionName(), - convertFunction.toEvaluator(e -> driverContext -> new ExpressionEvaluator() { - @Override - public org.elasticsearch.compute.data.Block eval(Page page) { - // This is a pass-through evaluator, since it sits directly on the source loading (no prior expressions) - return page.getBlock(0); - } - - @Override - public void close() {} - }).get(driverContext1) - ); + return new TypeConverter(convertFunction.functionName(), convertFunction.toEvaluator(new EvaluatorMapper.ToEvaluator() { + @Override + public ExpressionEvaluator.Factory apply(Expression expression) { + return driverContext -> new ExpressionEvaluator() { + @Override + public org.elasticsearch.compute.data.Block eval(Page page) { + // This is a pass-through evaluator, since it sits directly on the source loading (no prior expressions) + return page.getBlock(0); + } + + @Override + public void close() {} + }; + } + + @Override + public FoldContext foldCtx() { + throw new IllegalStateException("not folding"); + } + }).get(driverContext1)); } public Block convert(Block block) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/MapperUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/MapperUtils.java index e881eabb38c43..b8f539ea307c9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/MapperUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/MapperUtils.java @@ -11,6 +11,7 @@ import org.elasticsearch.compute.aggregation.AggregatorMode; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -90,7 +91,7 @@ static PhysicalPlan mapUnary(UnaryPlan p, PhysicalPlan child) { enrich.mode(), enrich.policy().getType(), enrich.matchField(), - BytesRefs.toString(enrich.policyName().fold()), + BytesRefs.toString(enrich.policyName().fold(FoldContext.small() /* TODO remove me */)), enrich.policy().getMatchField(), enrich.concreteIndices(), enrich.enrichFields() diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index 7223e6988bb19..a38236fe60954 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -61,6 +61,7 @@ import org.elasticsearch.xpack.esql.action.EsqlQueryAction; import org.elasticsearch.xpack.esql.action.EsqlSearchShardsAction; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.enrich.EnrichLookupService; import org.elasticsearch.xpack.esql.enrich.LookupFromIndexService; import org.elasticsearch.xpack.esql.plan.physical.ExchangeSinkExec; @@ -140,6 +141,7 @@ public void execute( CancellableTask rootTask, PhysicalPlan physicalPlan, Configuration configuration, + FoldContext foldContext, EsqlExecutionInfo execInfo, ActionListener listener ) { @@ -174,6 +176,7 @@ public void execute( RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, List.of(), configuration, + foldContext, null, null ); @@ -226,6 +229,7 @@ public void execute( RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, List.of(), configuration, + foldContext, exchangeSource, null ), @@ -460,16 +464,16 @@ public SourceProvider createSourceProvider() { context.exchangeSink(), enrichLookupService, lookupFromIndexService, - new EsPhysicalOperationProviders(contexts, searchService.getIndicesService().getAnalysis()) + new EsPhysicalOperationProviders(context.foldCtx(), contexts, searchService.getIndicesService().getAnalysis()) ); LOGGER.debug("Received physical plan:\n{}", plan); - plan = PlannerUtils.localPlan(context.searchExecutionContexts(), context.configuration, plan); + plan = PlannerUtils.localPlan(context.searchExecutionContexts(), context.configuration, context.foldCtx(), plan); // the planner will also set the driver parallelism in LocalExecutionPlanner.LocalExecutionPlan (used down below) // it's doing this in the planning of EsQueryExec (the source of the data) // see also EsPhysicalOperationProviders.sourcePhysicalOperation - LocalExecutionPlanner.LocalExecutionPlan localExecutionPlan = planner.plan(plan); + LocalExecutionPlanner.LocalExecutionPlan localExecutionPlan = planner.plan(context.foldCtx(), plan); if (LOGGER.isDebugEnabled()) { LOGGER.debug("Local execution plan:\n{}", localExecutionPlan.describe()); } @@ -715,7 +719,15 @@ public void onFailure(Exception e) { }; acquireSearchContexts(clusterAlias, shardIds, configuration, request.aliasFilters(), ActionListener.wrap(searchContexts -> { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH, ESQL_WORKER_THREAD_POOL_NAME); - var computeContext = new ComputeContext(sessionId, clusterAlias, searchContexts, configuration, null, exchangeSink); + var computeContext = new ComputeContext( + sessionId, + clusterAlias, + searchContexts, + configuration, + configuration.newFoldContext(), + null, + exchangeSink + ); runCompute(parentTask, computeContext, request.plan(), batchListener); }, batchListener::onFailure)); } @@ -766,6 +778,7 @@ private void runComputeOnDataNode( request.clusterAlias(), List.of(), request.configuration(), + new FoldContext(request.pragmas().foldLimit().getBytes()), exchangeSource, externalSink ), @@ -901,7 +914,15 @@ void runComputeOnRemoteCluster( exchangeSink.addCompletionListener(computeListener.acquireAvoid()); runCompute( parentTask, - new ComputeContext(localSessionId, clusterAlias, List.of(), configuration, exchangeSource, exchangeSink), + new ComputeContext( + localSessionId, + clusterAlias, + List.of(), + configuration, + configuration.newFoldContext(), + exchangeSource, + exchangeSink + ), coordinatorPlan, computeListener.acquireCompute(clusterAlias) ); @@ -925,6 +946,7 @@ record ComputeContext( String clusterAlias, List searchContexts, Configuration configuration, + FoldContext foldCtx, ExchangeSourceHandler exchangeSource, ExchangeSinkHandler exchangeSink ) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java index 58e80e569ee5e..2443c3f2cda62 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java @@ -12,12 +12,14 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.compute.lucene.DataPartitioning; import org.elasticsearch.compute.operator.Driver; import org.elasticsearch.compute.operator.DriverStatus; import org.elasticsearch.core.TimeValue; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.esql.core.expression.Expression; import java.io.IOException; import java.util.Objects; @@ -57,6 +59,8 @@ public final class QueryPragmas implements Writeable { public static final Setting NODE_LEVEL_REDUCTION = Setting.boolSetting("node_level_reduction", true); + public static final Setting FOLD_LIMIT = Setting.memorySizeSetting("fold_limit", "5%"); + public static final QueryPragmas EMPTY = new QueryPragmas(Settings.EMPTY); private final Settings settings; @@ -134,6 +138,17 @@ public boolean nodeLevelReduction() { return NODE_LEVEL_REDUCTION.get(settings); } + /** + * The maximum amount of memory we can use for {@link Expression#fold} during planing. This + * defaults to 5% of memory available on the current node. If this method is called on the + * coordinating node, this is 5% of the coordinating node's memory. If it's called on a data + * node, it's 5% of the data node. That's an exciting inconsistency. But it's + * important. Bigger nodes have more space to do folding. + */ + public ByteSizeValue foldLimit() { + return FOLD_LIMIT.get(settings); + } + public boolean isEmpty() { return settings.isEmpty(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index b44e249e38006..84173eeecc060 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -43,6 +43,7 @@ import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; import org.elasticsearch.xpack.esql.action.EsqlQueryTask; import org.elasticsearch.xpack.esql.core.async.AsyncTaskManagementService; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.enrich.EnrichLookupService; import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver; import org.elasticsearch.xpack.esql.enrich.LookupFromIndexService; @@ -189,11 +190,13 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener computeService.execute( sessionId, (CancellableTask) task, plan, configuration, + foldCtx, executionInfo, resultListener ); @@ -201,6 +204,7 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener new EnrichPolicyResolver.UnresolvedPolicy((String) e.policyName().fold(), e.mode())) + .map( + e -> new EnrichPolicyResolver.UnresolvedPolicy( + (String) e.policyName().fold(FoldContext.small() /* TODO remove me*/), + e.mode() + ) + ) .collect(Collectors.toSet()); final List indices = preAnalysis.indices; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 0847f71b1fb01..95ee6ab337bd6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.Converter; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -239,9 +240,9 @@ public static Converter converterFor(DataType from, DataType to) { return null; } - public static TemporalAmount foldToTemporalAmount(Expression field, String sourceText, DataType expectedType) { + public static TemporalAmount foldToTemporalAmount(FoldContext ctx, Expression field, String sourceText, DataType expectedType) { if (field.foldable()) { - Object v = field.fold(); + Object v = field.fold(ctx); if (v instanceof BytesRef b) { try { return EsqlDataTypeConverter.parseTemporalAmount(b.utf8ToString(), expectedType); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 7d4374934ab82..ee5073c05cab1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -55,6 +55,7 @@ import org.elasticsearch.xpack.esql.analysis.EnrichResolution; import org.elasticsearch.xpack.esql.analysis.PreAnalyzer; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; @@ -482,15 +483,16 @@ private static CsvTestsDataLoader.MultiIndexTestDataset testDatasets(LogicalPlan return new CsvTestsDataLoader.MultiIndexTestDataset(indexName, datasets); } - private static TestPhysicalOperationProviders testOperationProviders(CsvTestsDataLoader.MultiIndexTestDataset datasets) - throws Exception { - var indexResolution = loadIndexResolution(datasets); + private static TestPhysicalOperationProviders testOperationProviders( + FoldContext foldCtx, + CsvTestsDataLoader.MultiIndexTestDataset datasets + ) throws Exception { var indexPages = new ArrayList(); for (CsvTestsDataLoader.TestDataset dataset : datasets.datasets()) { var testData = loadPageFromCsv(CsvTests.class.getResource("/data/" + dataset.dataFileName()), dataset.typeMapping()); indexPages.add(new TestPhysicalOperationProviders.IndexPage(dataset.indexName(), testData.v1(), testData.v2())); } - return TestPhysicalOperationProviders.create(indexPages); + return TestPhysicalOperationProviders.create(foldCtx, indexPages); } private ActualResults executePlan(BigArrays bigArrays) throws Exception { @@ -498,6 +500,7 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception { var testDatasets = testDatasets(parsed); LogicalPlan analyzed = analyzedPlan(parsed, testDatasets); + FoldContext foldCtx = FoldContext.small(); EsqlSession session = new EsqlSession( getTestName(), configuration, @@ -505,21 +508,21 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception { null, null, functionRegistry, - new LogicalPlanOptimizer(new LogicalOptimizerContext(configuration)), + new LogicalPlanOptimizer(new LogicalOptimizerContext(configuration, foldCtx)), mapper, TEST_VERIFIER, new PlanningMetrics(), null, EsqlTestUtils.MOCK_QUERY_BUILDER_RESOLVER ); - TestPhysicalOperationProviders physicalOperationProviders = testOperationProviders(testDatasets); + TestPhysicalOperationProviders physicalOperationProviders = testOperationProviders(foldCtx, testDatasets); PlainActionFuture listener = new PlainActionFuture<>(); session.executeOptimizedPlan( new EsqlQueryRequest(), new EsqlExecutionInfo(randomBoolean()), - planRunner(bigArrays, physicalOperationProviders), + planRunner(bigArrays, foldCtx, physicalOperationProviders), session.optimizedPlan(analyzed), listener.delegateFailureAndWrap( // Wrap so we can capture the warnings in the calling thread @@ -579,12 +582,13 @@ private void assertWarnings(List warnings) { testCase.assertWarnings(false).assertWarnings(normalized); } - PlanRunner planRunner(BigArrays bigArrays, TestPhysicalOperationProviders physicalOperationProviders) { - return (physicalPlan, listener) -> executeSubPlan(bigArrays, physicalOperationProviders, physicalPlan, listener); + PlanRunner planRunner(BigArrays bigArrays, FoldContext foldCtx, TestPhysicalOperationProviders physicalOperationProviders) { + return (physicalPlan, listener) -> executeSubPlan(bigArrays, foldCtx, physicalOperationProviders, physicalPlan, listener); } void executeSubPlan( BigArrays bigArrays, + FoldContext foldCtx, TestPhysicalOperationProviders physicalOperationProviders, PhysicalPlan physicalPlan, ActionListener listener @@ -630,12 +634,17 @@ void executeSubPlan( // replace fragment inside the coordinator plan List drivers = new ArrayList<>(); - LocalExecutionPlan coordinatorNodeExecutionPlan = executionPlanner.plan(new OutputExec(coordinatorPlan, collectedPages::add)); + LocalExecutionPlan coordinatorNodeExecutionPlan = executionPlanner.plan( + foldCtx, + new OutputExec(coordinatorPlan, collectedPages::add) + ); drivers.addAll(coordinatorNodeExecutionPlan.createDrivers(getTestName())); if (dataNodePlan != null) { var searchStats = new DisabledSearchStats(); - var logicalTestOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, searchStats)); - var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(configuration, searchStats)); + var logicalTestOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, foldCtx, searchStats)); + var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer( + new LocalPhysicalOptimizerContext(configuration, foldCtx, searchStats) + ); var csvDataNodePhysicalPlan = PlannerUtils.localPlan(dataNodePlan, logicalTestOptimizer, physicalTestOptimizer); exchangeSource.addRemoteSink( @@ -646,7 +655,7 @@ void executeSubPlan( throw new AssertionError("expected no failure", e); }) ); - LocalExecutionPlan dataNodeExecutionPlan = executionPlanner.plan(csvDataNodePhysicalPlan); + LocalExecutionPlan dataNodeExecutionPlan = executionPlanner.plan(foldCtx, csvDataNodePhysicalPlan); drivers.addAll(dataNodeExecutionPlan.createDrivers(getTestName())); Randomness.shuffle(drivers); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index dc4120f357725..2df6e30e96081 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1077,7 +1077,7 @@ public void testImplicitLimit() { from test """); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(DEFAULT_LIMIT)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(DEFAULT_LIMIT)); as(limit.child(), EsRelation.class); } @@ -1085,7 +1085,7 @@ public void testImplicitMaxLimitAfterLimit() { for (int i = -1; i <= 1; i++) { var plan = analyze("from test | limit " + (MAX_LIMIT + i)); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(MAX_LIMIT)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(MAX_LIMIT)); limit = as(limit.child(), Limit.class); as(limit.child(), EsRelation.class); } @@ -1102,7 +1102,7 @@ public void testImplicitMaxLimitAfterLimitAndNonLimit() { for (int i = -1; i <= 1; i++) { var plan = analyze("from test | limit " + (MAX_LIMIT + i) + " | eval s = salary * 10 | where s > 0"); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(MAX_LIMIT)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(MAX_LIMIT)); var filter = as(limit.child(), Filter.class); var eval = as(filter.child(), Eval.class); limit = as(eval.child(), Limit.class); @@ -1114,7 +1114,7 @@ public void testImplicitDefaultLimitAfterLimitAndBreaker() { for (var breaker : List.of("stats c = count(salary) by last_name", "sort salary")) { var plan = analyze("from test | limit 100000 | " + breaker); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(MAX_LIMIT)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(MAX_LIMIT)); } } @@ -1122,7 +1122,7 @@ public void testImplicitDefaultLimitAfterBreakerAndNonBreakers() { for (var breaker : List.of("stats c = count(salary) by last_name", "eval c = salary | sort c")) { var plan = analyze("from test | " + breaker + " | eval cc = c * 10 | where cc > 0"); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(DEFAULT_LIMIT)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(DEFAULT_LIMIT)); } } @@ -1428,7 +1428,7 @@ public void testEmptyEsRelationOnLimitZeroWithCount() throws IOException { var plan = analyzeWithEmptyFieldCapsResponse(query); var limit = as(plan, Limit.class); limit = as(limit.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(0)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(0)); var orderBy = as(limit.child(), OrderBy.class); var agg = as(orderBy.child(), Aggregate.class); assertEmptyEsRelation(agg.child()); @@ -1443,7 +1443,7 @@ public void testEmptyEsRelationOnConstantEvalAndKeep() throws IOException { var plan = analyzeWithEmptyFieldCapsResponse(query); var limit = as(plan, Limit.class); limit = as(limit.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(2)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(2)); var project = as(limit.child(), EsqlProject.class); var eval = as(project.child(), Eval.class); assertEmptyEsRelation(eval.child()); @@ -1460,7 +1460,7 @@ public void testEmptyEsRelationOnConstantEvalAndStats() throws IOException { var agg = as(limit.child(), Aggregate.class); var eval = as(agg.child(), Eval.class); limit = as(eval.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(10)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(10)); assertEmptyEsRelation(limit.child()); } @@ -2054,10 +2054,10 @@ public void testLookup() { } LogicalPlan plan = analyze(query); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(1000)); var lookup = as(limit.child(), Lookup.class); - assertThat(lookup.tableName().fold(), equalTo("int_number_names")); + assertThat(as(lookup.tableName(), Literal.class).value(), equalTo("int_number_names")); assertMap(lookup.matchFields().stream().map(Object::toString).toList(), matchesList().item(startsWith("int{r}"))); assertThat( lookup.localRelation().output().stream().map(Object::toString).toList(), @@ -2336,7 +2336,7 @@ public void testCoalesceWithMixedNumericTypes() { projection = as(projections.get(3), ReferenceAttribute.class); assertEquals(projection.name(), "w"); assertEquals(projection.dataType(), DataType.DOUBLE); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(1000)); } public void testNamedParamsForIdentifiers() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapperTests.java new file mode 100644 index 0000000000000..828f9e061686b --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapperTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.evaluator.mapper; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; +import org.hamcrest.Matchers; + +public class EvaluatorMapperTests extends ESTestCase { + public void testFoldCompletesWithPlentyOfMemory() { + Add add = new Add( + Source.synthetic("shouldn't break"), + new Literal(Source.EMPTY, 1, DataType.INTEGER), + new Literal(Source.EMPTY, 3, DataType.INTEGER) + ); + assertEquals(add.fold(new FoldContext(100)), 4); + } + + public void testFoldBreaksWithLittleMemory() { + Add add = new Add( + Source.synthetic("should break"), + new Literal(Source.EMPTY, 1, DataType.INTEGER), + new Literal(Source.EMPTY, 3, DataType.INTEGER) + ); + Exception e = expectThrows(FoldContext.FoldTooMuchMemoryException.class, () -> add.fold(new FoldContext(10))); + assertThat( + e.getMessage(), + Matchers.equalTo( + "line -1:-1: Folding query used more than 10b. " + + "The expression that pushed past the limit is [should break] which needed 32b." + ) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java index c086245d6fd61..87ea6315d4f3b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java @@ -22,6 +22,7 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.NumericUtils; @@ -40,6 +41,7 @@ import java.util.stream.IntStream; import static org.elasticsearch.compute.data.BlockUtils.toJavaObject; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -263,12 +265,12 @@ private void evaluate(Expression evaluableExpression) { assertTrue(evaluableExpression.foldable()); if (testCase.foldingExceptionClass() != null) { - Throwable t = expectThrows(testCase.foldingExceptionClass(), evaluableExpression::fold); + Throwable t = expectThrows(testCase.foldingExceptionClass(), () -> evaluableExpression.fold(FoldContext.small())); assertThat(t.getMessage(), equalTo(testCase.foldingExceptionMessage())); return; } - Object result = evaluableExpression.fold(); + Object result = evaluableExpression.fold(FoldContext.small()); // Decode unsigned longs into BigIntegers if (testCase.expectedType() == DataType.UNSIGNED_LONG && result != null) { result = NumericUtils.unsignedLongAsBigInteger((Long) result); @@ -289,7 +291,7 @@ private void resolveExpression(Expression expression, Consumer onAgg expression = resolveSurrogates(expression); // As expressions may be composed of multiple functions, we need to fold nulls bottom-up - expression = expression.transformUp(e -> new FoldNull().rule(e)); + expression = expression.transformUp(e -> new FoldNull().rule(e, unboundLogicalOptimizerContext())); assertThat(expression.dataType(), equalTo(testCase.expectedType())); Expression.TypeResolution resolution = expression.typeResolved(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 03b9dba298951..1cf087cf55ccd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; @@ -44,6 +45,7 @@ import org.elasticsearch.xpack.esql.core.util.NumericUtils; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.evaluator.EvalMapper; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; @@ -97,6 +99,7 @@ import static java.util.Map.entry; import static org.elasticsearch.compute.data.BlockUtils.toJavaObject; import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.endsWith; @@ -530,14 +533,28 @@ protected final Expression buildLiteralExpression(TestCaseSupplier.TestCase test return build(testCase.getSource(), testCase.getDataAsLiterals()); } + public static EvaluatorMapper.ToEvaluator toEvaluator() { + return new EvaluatorMapper.ToEvaluator() { + @Override + public ExpressionEvaluator.Factory apply(Expression expression) { + return evaluator(expression); + } + + @Override + public FoldContext foldCtx() { + return FoldContext.small(); + } + }; + } + /** * Convert an {@link Expression} tree into a {@link ExpressionEvaluator.Factory} * for {@link ExpressionEvaluator}s in the same way as our planner. */ public static ExpressionEvaluator.Factory evaluator(Expression e) { - e = new FoldNull().rule(e); + e = new FoldNull().rule(e, unboundLogicalOptimizerContext()); if (e.foldable()) { - e = new Literal(e.source(), e.fold(), e.dataType()); + e = new Literal(e.source(), e.fold(FoldContext.small()), e.dataType()); } Layout.Builder builder = new Layout.Builder(); buildLayout(builder, e); @@ -545,7 +562,7 @@ public static ExpressionEvaluator.Factory evaluator(Expression e) { if (resolution.unresolved()) { throw new AssertionError("expected resolved " + resolution.message()); } - return EvalMapper.toEvaluator(e, builder.build()); + return EvalMapper.toEvaluator(FoldContext.small(), e, builder.build()); } protected final Page row(List values) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java index 65b9c447170f4..64086334b7251 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java @@ -20,6 +20,7 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.NumericUtils; import org.elasticsearch.xpack.esql.optimizer.rules.logical.FoldNull; @@ -38,6 +39,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -132,7 +134,7 @@ public final void testEvaluate() { if (resolution.unresolved()) { throw new AssertionError("expected resolved " + resolution.message()); } - expression = new FoldNull().rule(expression); + expression = new FoldNull().rule(expression, unboundLogicalOptimizerContext()); assertThat(expression.dataType(), equalTo(testCase.expectedType())); logger.info("Result type: " + expression.dataType()); @@ -363,11 +365,11 @@ public void testFold() { return; } assertFalse("expected resolved", expression.typeResolved().unresolved()); - Expression nullOptimized = new FoldNull().rule(expression); + Expression nullOptimized = new FoldNull().rule(expression, unboundLogicalOptimizerContext()); assertThat(nullOptimized.dataType(), equalTo(testCase.expectedType())); assertTrue(nullOptimized.foldable()); if (testCase.foldingExceptionClass() == null) { - Object result = nullOptimized.fold(); + Object result = nullOptimized.fold(FoldContext.small()); // Decode unsigned longs into BigIntegers if (testCase.expectedType() == DataType.UNSIGNED_LONG && result != null) { result = NumericUtils.unsignedLongAsBigInteger((Long) result); @@ -380,7 +382,7 @@ public void testFold() { assertWarnings(testCase.getExpectedWarnings()); } } else { - Throwable t = expectThrows(testCase.foldingExceptionClass(), nullOptimized::fold); + Throwable t = expectThrows(testCase.foldingExceptionClass(), () -> nullOptimized.fold(FoldContext.small())); assertThat(t.getMessage(), equalTo(testCase.foldingExceptionMessage())); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java index 19af9892015b2..e507640c7b23c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.analysis.Verifier; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -80,7 +81,9 @@ public EsqlFunctionRegistry snapshotRegistry() { var plan = parser.createStatement(esql); plan = plan.transformDown( Limit.class, - l -> Objects.equals(l.limit().fold(), 10) ? new LicensedLimit(l.source(), l.limit(), l.child(), functionLicenseFeature) : l + l -> Objects.equals(l.limit().fold(FoldContext.small()), 10) + ? new LicensedLimit(l.source(), l.limit(), l.child(), functionLicenseFeature) + : l ); return analyzer(registry, operationMode).analyze(plan); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index f2bae0c5a4979..9bf063518d4ba 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -1412,7 +1412,7 @@ public static final class TestCase { /** * Warnings that are added by calling {@link AbstractFunctionTestCase#evaluator} - * or {@link Expression#fold()} on the expression built by this. + * or {@link Expression#fold} on the expression built by this. */ private final String[] expectedBuildEvaluatorWarnings; @@ -1542,7 +1542,7 @@ public String[] getExpectedWarnings() { /** * Warnings that are added by calling {@link AbstractFunctionTestCase#evaluator} - * or {@link Expression#fold()} on the expression built by this. + * or {@link Expression#fold} on the expression built by this. */ public String[] getExpectedBuildEvaluatorWarnings() { return expectedBuildEvaluatorWarnings; @@ -1624,7 +1624,7 @@ public TestCase withWarning(String warning) { /** * Warnings that are added by calling {@link AbstractFunctionTestCase#evaluator} - * or {@link Expression#fold()} on the expression built by this. + * or {@link Expression#fold} on the expression built by this. */ public TestCase withBuildEvaluatorWarning(String warning) { return new TestCase( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java index de84086e3cb4e..911878a645b42 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java @@ -19,9 +19,11 @@ import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.junit.After; @@ -67,7 +69,7 @@ public void testPartialFoldDropsFirstFalse() { ); assertThat(c.foldable(), equalTo(false)); assertThat( - c.partiallyFold(), + c.partiallyFold(FoldContext.small()), equalTo(new Case(Source.synthetic("case"), field("last_cond", DataType.BOOLEAN), List.of(field("last", DataType.LONG)))) ); } @@ -80,7 +82,7 @@ public void testPartialFoldMv() { ); assertThat(c.foldable(), equalTo(false)); assertThat( - c.partiallyFold(), + c.partiallyFold(FoldContext.small()), equalTo(new Case(Source.synthetic("case"), field("last_cond", DataType.BOOLEAN), List.of(field("last", DataType.LONG)))) ); } @@ -92,7 +94,7 @@ public void testPartialFoldNoop() { List.of(field("first", DataType.LONG), field("last", DataType.LONG)) ); assertThat(c.foldable(), equalTo(false)); - assertThat(c.partiallyFold(), sameInstance(c)); + assertThat(c.partiallyFold(FoldContext.small()), sameInstance(c)); } public void testPartialFoldFirst() { @@ -102,7 +104,7 @@ public void testPartialFoldFirst() { List.of(field("first", DataType.LONG), field("last", DataType.LONG)) ); assertThat(c.foldable(), equalTo(false)); - assertThat(c.partiallyFold(), equalTo(field("first", DataType.LONG))); + assertThat(c.partiallyFold(FoldContext.small()), equalTo(field("first", DataType.LONG))); } public void testPartialFoldFirstAfterKeepingUnknown() { @@ -118,7 +120,7 @@ public void testPartialFoldFirstAfterKeepingUnknown() { ); assertThat(c.foldable(), equalTo(false)); assertThat( - c.partiallyFold(), + c.partiallyFold(FoldContext.small()), equalTo( new Case( Source.synthetic("case"), @@ -141,7 +143,7 @@ public void testPartialFoldSecond() { ) ); assertThat(c.foldable(), equalTo(false)); - assertThat(c.partiallyFold(), equalTo(field("second", DataType.LONG))); + assertThat(c.partiallyFold(FoldContext.small()), equalTo(field("second", DataType.LONG))); } public void testPartialFoldSecondAfterDroppingFalse() { @@ -156,7 +158,7 @@ public void testPartialFoldSecondAfterDroppingFalse() { ) ); assertThat(c.foldable(), equalTo(false)); - assertThat(c.partiallyFold(), equalTo(field("second", DataType.LONG))); + assertThat(c.partiallyFold(FoldContext.small()), equalTo(field("second", DataType.LONG))); } public void testPartialFoldLast() { @@ -171,7 +173,7 @@ public void testPartialFoldLast() { ) ); assertThat(c.foldable(), equalTo(false)); - assertThat(c.partiallyFold(), equalTo(field("last", DataType.LONG))); + assertThat(c.partiallyFold(FoldContext.small()), equalTo(field("last", DataType.LONG))); } public void testPartialFoldLastAfterKeepingUnknown() { @@ -187,7 +189,7 @@ public void testPartialFoldLastAfterKeepingUnknown() { ); assertThat(c.foldable(), equalTo(false)); assertThat( - c.partiallyFold(), + c.partiallyFold(FoldContext.small()), equalTo( new Case( Source.synthetic("case"), @@ -203,7 +205,7 @@ public void testEvalCase() { DriverContext driverContext = driverContext(); Page page = new Page(driverContext.blockFactory().newConstantIntBlockWith(0, 1)); try ( - EvalOperator.ExpressionEvaluator eval = caseExpr.toEvaluator(AbstractFunctionTestCase::evaluator).get(driverContext); + EvalOperator.ExpressionEvaluator eval = caseExpr.toEvaluator(AbstractFunctionTestCase.toEvaluator()).get(driverContext); Block block = eval.eval(page) ) { return toJavaObject(block, 0); @@ -216,7 +218,7 @@ public void testEvalCase() { public void testFoldCase() { testCase(caseExpr -> { assertTrue(caseExpr.foldable()); - return caseExpr.fold(); + return caseExpr.fold(FoldContext.small()); }); } @@ -265,22 +267,31 @@ public void testCaseWithIncompatibleTypes() { public void testCaseIsLazy() { Case caseExpr = caseExpr(true, 1, true, 2); DriverContext driveContext = driverContext(); - EvalOperator.ExpressionEvaluator evaluator = caseExpr.toEvaluator(child -> { - Object value = child.fold(); - if (value != null && value.equals(2)) { - return dvrCtx -> new EvalOperator.ExpressionEvaluator() { - @Override - public Block eval(Page page) { - fail("Unexpected evaluation of 4th argument"); - return null; - } + EvaluatorMapper.ToEvaluator toEvaluator = new EvaluatorMapper.ToEvaluator() { + @Override + public EvalOperator.ExpressionEvaluator.Factory apply(Expression expression) { + Object value = expression.fold(FoldContext.small()); + if (value != null && value.equals(2)) { + return dvrCtx -> new EvalOperator.ExpressionEvaluator() { + @Override + public Block eval(Page page) { + fail("Unexpected evaluation of 4th argument"); + return null; + } - @Override - public void close() {} - }; + @Override + public void close() {} + }; + } + return AbstractFunctionTestCase.evaluator(expression); } - return AbstractFunctionTestCase.evaluator(child); - }).get(driveContext); + + @Override + public FoldContext foldCtx() { + return FoldContext.small(); + } + }; + EvalOperator.ExpressionEvaluator evaluator = caseExpr.toEvaluator(toEvaluator).get(driveContext); Page page = new Page(driveContext.blockFactory().newConstantIntBlockWith(0, 1)); try (Block block = evaluator.eval(page)) { assertEquals(1, toJavaObject(block, 0)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java index 05923246520fc..23a0f2307171c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.NumericUtils; @@ -779,7 +780,7 @@ public void testFancyFolding() { return; } assertThat(e.foldable(), equalTo(true)); - Object result = e.fold(); + Object result = e.fold(FoldContext.small()); if (testCase.getExpectedBuildEvaluatorWarnings() != null) { assertWarnings(testCase.getExpectedBuildEvaluatorWarnings()); } @@ -799,18 +800,18 @@ public void testPartialFold() { } Case c = (Case) buildFieldExpression(testCase); if (extra().expectedPartialFold == null) { - assertThat(c.partiallyFold(), sameInstance(c)); + assertThat(c.partiallyFold(FoldContext.small()), sameInstance(c)); return; } if (extra().expectedPartialFold.size() == 1) { - assertThat(c.partiallyFold(), equalTo(extra().expectedPartialFold.get(0).asField())); + assertThat(c.partiallyFold(FoldContext.small()), equalTo(extra().expectedPartialFold.get(0).asField())); return; } Case expected = build( Source.synthetic("expected"), extra().expectedPartialFold.stream().map(TestCaseSupplier.TypedData::asField).toList() ); - assertThat(c.partiallyFold(), equalTo(expected)); + assertThat(c.partiallyFold(FoldContext.small()), equalTo(expected)); } private static Function addWarnings(List warnings) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java index be978eda06758..cd27ce511b317 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.InvalidArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -99,7 +100,7 @@ public void testAllChronoFields() { EsqlTestUtils.TEST_CFG ); - assertThat(instance.fold(), is(date.getLong(value))); + assertThat(instance.fold(FoldContext.small()), is(date.getLong(value))); assertThat( DateExtract.process(epochMilli, new BytesRef(value.name()), EsqlTestUtils.TEST_CFG.zoneId()), is(date.getLong(value)) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java index 797c99992815e..688341ebaa2b7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -187,19 +188,27 @@ public void testCoalesceIsLazy() { Layout.Builder builder = new Layout.Builder(); buildLayout(builder, exp); Layout layout = builder.build(); - EvaluatorMapper.ToEvaluator toEvaluator = child -> { - if (child == evil) { - return dvrCtx -> new EvalOperator.ExpressionEvaluator() { - @Override - public Block eval(Page page) { - throw new AssertionError("shouldn't be called"); - } - - @Override - public void close() {} - }; + EvaluatorMapper.ToEvaluator toEvaluator = new EvaluatorMapper.ToEvaluator() { + @Override + public EvalOperator.ExpressionEvaluator.Factory apply(Expression expression) { + if (expression == evil) { + return dvrCtx -> new EvalOperator.ExpressionEvaluator() { + @Override + public Block eval(Page page) { + throw new AssertionError("shouldn't be called"); + } + + @Override + public void close() {} + }; + } + return EvalMapper.toEvaluator(FoldContext.small(), expression, layout); + } + + @Override + public FoldContext foldCtx() { + return FoldContext.small(); } - return EvalMapper.toEvaluator(child, layout); }; try ( EvalOperator.ExpressionEvaluator eval = exp.toEvaluator(toEvaluator).get(driverContext()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java index 4f8adf3abaae6..6c41552a9fc52 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -159,8 +160,8 @@ protected Expression build(Source source, List args) { Expression expression = args.get(0); Literal pattern = (Literal) args.get(1); Literal caseInsensitive = args.size() > 2 ? (Literal) args.get(2) : null; - String patternString = ((BytesRef) pattern.fold()).utf8ToString(); - boolean caseInsensitiveBool = caseInsensitive != null ? (boolean) caseInsensitive.fold() : false; + String patternString = ((BytesRef) pattern.fold(FoldContext.small())).utf8ToString(); + boolean caseInsensitiveBool = caseInsensitive != null ? (boolean) caseInsensitive.fold(FoldContext.small()) : false; logger.info("pattern={} caseInsensitive={}", patternString, caseInsensitiveBool); return caseInsensitiveBool diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java index b355feb6130a3..f779dd038454d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -54,7 +55,7 @@ public void testRandomLocale() { String testString = randomAlphaOfLength(10); Configuration cfg = randomLocaleConfig(); ToLower func = new ToLower(Source.EMPTY, new Literal(Source.EMPTY, testString, DataType.KEYWORD), cfg); - assertThat(BytesRefs.toBytesRef(testString.toLowerCase(cfg.locale())), equalTo(func.fold())); + assertThat(BytesRefs.toBytesRef(testString.toLowerCase(cfg.locale())), equalTo(func.fold(FoldContext.small()))); } private Configuration randomLocaleConfig() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java index fdae4f953a0fa..3957c2e1fb2c0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -54,7 +55,7 @@ public void testRandomLocale() { String testString = randomAlphaOfLength(10); Configuration cfg = randomLocaleConfig(); ToUpper func = new ToUpper(Source.EMPTY, new Literal(Source.EMPTY, testString, DataType.KEYWORD), cfg); - assertThat(BytesRefs.toBytesRef(testString.toUpperCase(cfg.locale())), equalTo(func.fold())); + assertThat(BytesRefs.toBytesRef(testString.toUpperCase(cfg.locale())), equalTo(func.fold(FoldContext.small()))); } private Configuration randomLocaleConfig() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java index eed2c7379e9e1..6626ac50d60b5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -78,8 +79,8 @@ protected Expression build(Source source, List args) { Literal pattern = (Literal) args.get(1); if (args.size() > 2) { Literal caseInsensitive = (Literal) args.get(2); - assertThat(caseInsensitive.fold(), equalTo(false)); + assertThat(caseInsensitive.fold(FoldContext.small()), equalTo(false)); } - return new WildcardLike(source, expression, new WildcardPattern(((BytesRef) pattern.fold()).utf8ToString())); + return new WildcardLike(source, expression, new WildcardPattern(((BytesRef) pattern.fold(FoldContext.small())).utf8ToString())); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java index a8c7b5b5a83fd..15860d35539e0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -154,7 +155,7 @@ public void testEdgeCases() { private Object foldTemporalAmount(Object val) { Neg neg = new Neg(Source.EMPTY, new Literal(Source.EMPTY, val, typeOf(val))); - return neg.fold(); + return neg.fold(FoldContext.small()); } private static DataType typeOf(Object val) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java index b004adca351ab..80f67ec8e5e3a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geo.ShapeTestUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -48,27 +49,27 @@ public InTests(@Name("TestCase") Supplier testCaseSup public void testInWithContainedValue() { In in = new In(EMPTY, TWO, Arrays.asList(ONE, TWO, THREE)); - assertTrue((Boolean) in.fold()); + assertTrue((Boolean) in.fold(FoldContext.small())); } public void testInWithNotContainedValue() { In in = new In(EMPTY, THREE, Arrays.asList(ONE, TWO)); - assertFalse((Boolean) in.fold()); + assertFalse((Boolean) in.fold(FoldContext.small())); } public void testHandleNullOnLeftValue() { In in = new In(EMPTY, NULL, Arrays.asList(ONE, TWO, THREE)); - assertNull(in.fold()); + assertNull(in.fold(FoldContext.small())); in = new In(EMPTY, NULL, Arrays.asList(ONE, NULL, THREE)); - assertNull(in.fold()); + assertNull(in.fold(FoldContext.small())); } public void testHandleNullsOnRightValue() { In in = new In(EMPTY, THREE, Arrays.asList(ONE, NULL, THREE)); - assertTrue((Boolean) in.fold()); + assertTrue((Boolean) in.fold(FoldContext.small())); in = new In(EMPTY, ONE, Arrays.asList(TWO, NULL, THREE)); - assertNull(in.fold()); + assertNull(in.fold(FoldContext.small())); } private static Literal L(Object value) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsTests.java index faf0a0d8f418c..6fa1112f23f45 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import static org.elasticsearch.xpack.esql.EsqlTestUtils.of; @@ -18,37 +19,37 @@ public class InsensitiveEqualsTests extends ESTestCase { public void testFold() { - assertTrue(insensitiveEquals(l("foo"), l("foo")).fold()); - assertTrue(insensitiveEquals(l("Foo"), l("foo")).fold()); - assertTrue(insensitiveEquals(l("Foo"), l("fOO")).fold()); - assertTrue(insensitiveEquals(l("foo*"), l("foo*")).fold()); - assertTrue(insensitiveEquals(l("foo*"), l("FOO*")).fold()); - assertTrue(insensitiveEquals(l("foo?bar"), l("foo?bar")).fold()); - assertTrue(insensitiveEquals(l("foo?bar"), l("FOO?BAR")).fold()); - assertFalse(insensitiveEquals(l("Foo"), l("fo*")).fold()); - assertFalse(insensitiveEquals(l("Fox"), l("fo?")).fold()); - assertFalse(insensitiveEquals(l("Foo"), l("*OO")).fold()); - assertFalse(insensitiveEquals(l("BarFooBaz"), l("*O*")).fold()); - assertFalse(insensitiveEquals(l("BarFooBaz"), l("bar*baz")).fold()); - assertFalse(insensitiveEquals(l("foo"), l("*")).fold()); + assertTrue(insensitiveEquals(l("foo"), l("foo")).fold(FoldContext.small())); + assertTrue(insensitiveEquals(l("Foo"), l("foo")).fold(FoldContext.small())); + assertTrue(insensitiveEquals(l("Foo"), l("fOO")).fold(FoldContext.small())); + assertTrue(insensitiveEquals(l("foo*"), l("foo*")).fold(FoldContext.small())); + assertTrue(insensitiveEquals(l("foo*"), l("FOO*")).fold(FoldContext.small())); + assertTrue(insensitiveEquals(l("foo?bar"), l("foo?bar")).fold(FoldContext.small())); + assertTrue(insensitiveEquals(l("foo?bar"), l("FOO?BAR")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("Foo"), l("fo*")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("Fox"), l("fo?")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("Foo"), l("*OO")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("BarFooBaz"), l("*O*")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("BarFooBaz"), l("bar*baz")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("foo"), l("*")).fold(FoldContext.small())); - assertFalse(insensitiveEquals(l("foo*bar"), l("foo\\*bar")).fold()); - assertFalse(insensitiveEquals(l("foo?"), l("foo\\?")).fold()); - assertFalse(insensitiveEquals(l("foo?bar"), l("foo\\?bar")).fold()); - assertFalse(insensitiveEquals(l(randomAlphaOfLength(10)), l("*")).fold()); - assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("???")).fold()); + assertFalse(insensitiveEquals(l("foo*bar"), l("foo\\*bar")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("foo?"), l("foo\\?")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("foo?bar"), l("foo\\?bar")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l(randomAlphaOfLength(10)), l("*")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("???")).fold(FoldContext.small())); - assertFalse(insensitiveEquals(l("foo"), l("bar")).fold()); - assertFalse(insensitiveEquals(l("foo"), l("ba*")).fold()); - assertFalse(insensitiveEquals(l("foo"), l("*a*")).fold()); - assertFalse(insensitiveEquals(l(""), l("bar")).fold()); - assertFalse(insensitiveEquals(l("foo"), l("")).fold()); - assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("??")).fold()); - assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("????")).fold()); + assertFalse(insensitiveEquals(l("foo"), l("bar")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("foo"), l("ba*")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("foo"), l("*a*")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l(""), l("bar")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("foo"), l("")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("??")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("????")).fold(FoldContext.small())); - assertNull(insensitiveEquals(l("foo"), Literal.NULL).fold()); - assertNull(insensitiveEquals(Literal.NULL, l("foo")).fold()); - assertNull(insensitiveEquals(Literal.NULL, Literal.NULL).fold()); + assertNull(insensitiveEquals(l("foo"), Literal.NULL).fold(FoldContext.small())); + assertNull(insensitiveEquals(Literal.NULL, l("foo")).fold(FoldContext.small())); + assertNull(insensitiveEquals(Literal.NULL, Literal.NULL).fold(FoldContext.small())); } public void testProcess() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index 0c03556241d28..11cd123c731e8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; @@ -70,6 +71,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.statsForExistingField; import static org.elasticsearch.xpack.esql.EsqlTestUtils.statsForMissingField; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.hamcrest.Matchers.contains; @@ -93,7 +95,7 @@ public static void init() { mapping = loadMapping("mapping-basic.json"); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); IndexResolution getIndexResult = IndexResolution.valid(test); - logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)); + logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext()); analyzer = new Analyzer( new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, EsqlTestUtils.emptyPolicyResolution()), @@ -161,7 +163,7 @@ public void testMissingFieldInProject() { assertThat(Expressions.names(eval.fields()), contains("last_name")); var alias = as(eval.fields().get(0), Alias.class); var literal = as(alias.child(), Literal.class); - assertThat(literal.fold(), is(nullValue())); + assertThat(literal.value(), is(nullValue())); assertThat(literal.dataType(), is(DataType.KEYWORD)); var limit = as(eval.child(), Limit.class); @@ -304,7 +306,7 @@ public void testMissingFieldInEval() { var alias = as(eval.fields().get(0), Alias.class); var literal = as(alias.child(), Literal.class); - assertThat(literal.fold(), is(nullValue())); + assertThat(literal.value(), is(nullValue())); assertThat(literal.dataType(), is(DataType.INTEGER)); var limit = as(eval.child(), Limit.class); @@ -402,7 +404,7 @@ public void testSparseDocument() throws Exception { EsIndex index = new EsIndex("large", large, Map.of("large", IndexMode.STANDARD)); IndexResolution getIndexResult = IndexResolution.valid(index); - var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)); + var logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext()); var analyzer = new Analyzer( new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, EsqlTestUtils.emptyPolicyResolution()), @@ -411,7 +413,7 @@ public void testSparseDocument() throws Exception { var analyzed = analyzer.analyze(parser.createStatement(query)); var optimized = logicalOptimizer.optimize(analyzed); - var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, searchStats); + var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, FoldContext.small(), searchStats); var plan = new LocalLogicalPlanOptimizer(localContext).localOptimize(optimized); var project = as(plan, Project.class); @@ -423,7 +425,7 @@ public void testSparseDocument() throws Exception { var eval = as(project.child(), Eval.class); var field = eval.fields().get(0); assertThat(Expressions.name(field), is("field005")); - assertThat(Alias.unwrap(field).fold(), Matchers.nullValue()); + assertThat(Alias.unwrap(field).fold(FoldContext.small()), Matchers.nullValue()); } // InferIsNotNull @@ -561,7 +563,7 @@ private LogicalPlan plan(String query) { } private LogicalPlan localPlan(LogicalPlan plan, SearchStats searchStats) { - var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, searchStats); + var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, FoldContext.small(), searchStats); // System.out.println(plan); var localPlan = new LocalLogicalPlanOptimizer(localContext).localOptimize(plan); // System.out.println(localPlan); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 6dee34323443d..1536ed7f99fec 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.xpack.esql.analysis.Verifier; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -90,6 +91,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType; import static org.hamcrest.Matchers.contains; @@ -407,7 +409,7 @@ public void testMultiCountAllWithFilter() { @SuppressWarnings("unchecked") public void testSingleCountWithStatsFilter() { // an optimizer that filters out the ExtractAggregateCommonFilter rule - var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config)) { + var logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext()) { @Override protected List> batches() { var oldBatches = super.batches(); @@ -486,7 +488,7 @@ public void testQueryStringFunction() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var expected = QueryBuilders.queryStringQuery("last_name: Smith"); assertThat(query.query().toString(), is(expected.toString())); } @@ -515,7 +517,7 @@ public void testQueryStringFunctionConjunctionWhereOperands() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 37, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); @@ -550,7 +552,7 @@ public void testQueryStringFunctionWithFunctionsPushedToLucene() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 37, "cidr_match(ip, \"127.0.0.1/32\")"); var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); @@ -585,7 +587,7 @@ public void testQueryStringFunctionMultipleWhereClauses() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(3, 8, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); @@ -618,7 +620,7 @@ public void testQueryStringFunctionMultipleQstrClauses() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var queryStringLeft = QueryBuilders.queryStringQuery("last_name: Smith"); var queryStringRight = QueryBuilders.queryStringQuery("emp_no: [10010 TO *]"); @@ -647,7 +649,7 @@ public void testMatchFunction() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var expected = QueryBuilders.matchQuery("last_name", "Smith").lenient(true); assertThat(query.query().toString(), is(expected.toString())); } @@ -676,7 +678,7 @@ public void testMatchFunctionConjunctionWhereOperands() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 38, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); @@ -711,7 +713,7 @@ public void testMatchFunctionWithFunctionsPushedToLucene() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 32, "cidr_match(ip, \"127.0.0.1/32\")"); var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); @@ -745,7 +747,7 @@ public void testMatchFunctionMultipleWhereClauses() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(3, 8, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); @@ -777,7 +779,7 @@ public void testMatchFunctionMultipleMatchClauses() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var queryStringLeft = QueryBuilders.matchQuery("last_name", "Smith").lenient(true); var queryStringRight = QueryBuilders.matchQuery("first_name", "John").lenient(true); @@ -806,7 +808,7 @@ public void testKqlFunction() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var expected = kqlQueryBuilder("last_name: Smith"); assertThat(query.query().toString(), is(expected.toString())); } @@ -835,7 +837,7 @@ public void testKqlFunctionConjunctionWhereOperands() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 36, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); @@ -870,7 +872,7 @@ public void testKqlFunctionWithFunctionsPushedToLucene() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 36, "cidr_match(ip, \"127.0.0.1/32\")"); var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); @@ -905,7 +907,7 @@ public void testKqlFunctionMultipleWhereClauses() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(3, 8, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); @@ -938,7 +940,7 @@ public void testKqlFunctionMultipleKqlClauses() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var kqlQueryLeft = kqlQueryBuilder("last_name: Smith"); var kqlQueryRight = kqlQueryBuilder("emp_no > 10010"); @@ -1004,7 +1006,7 @@ public void testIsNotNullPushdownFilter() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var expected = QueryBuilders.existsQuery("emp_no"); assertThat(query.query().toString(), is(expected.toString())); } @@ -1028,7 +1030,7 @@ public void testIsNullPushdownFilter() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var expected = QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("emp_no")); assertThat(query.query().toString(), is(expected.toString())); } @@ -1599,7 +1601,7 @@ public void testTermFunction() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var expected = QueryBuilders.termQuery("last_name", "Smith"); assertThat(query.query().toString(), is(expected.toString())); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 4d175dea05071..2aed259e7ad0b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.Nullability; @@ -151,6 +152,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.localSource; import static org.elasticsearch.xpack.esql.EsqlTestUtils.referenceAttribute; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.Analyzer.NO_FIELDS; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze; @@ -188,6 +190,7 @@ public class LogicalPlanOptimizerTests extends ESTestCase { private static EsqlParser parser; private static Analyzer analyzer; + private static LogicalOptimizerContext logicalOptimizerCtx; private static LogicalPlanOptimizer logicalOptimizer; private static Map mapping; private static Map mappingAirports; @@ -203,7 +206,7 @@ public class LogicalPlanOptimizerTests extends ESTestCase { private static Analyzer metricsAnalyzer; private static class SubstitutionOnlyOptimizer extends LogicalPlanOptimizer { - static SubstitutionOnlyOptimizer INSTANCE = new SubstitutionOnlyOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)); + static SubstitutionOnlyOptimizer INSTANCE = new SubstitutionOnlyOptimizer(unboundLogicalOptimizerContext()); SubstitutionOnlyOptimizer(LogicalOptimizerContext optimizerContext) { super(optimizerContext); @@ -218,7 +221,8 @@ protected List> batches() { @BeforeClass public static void init() { parser = new EsqlParser(); - logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)); + logicalOptimizerCtx = unboundLogicalOptimizerContext(); + logicalOptimizer = new LogicalPlanOptimizer(logicalOptimizerCtx); enrichResolution = new EnrichResolution(); AnalyzerTestUtils.loadEnrichPolicyResolution(enrichResolution, "languages_idx", "id", "languages_idx", "mapping-languages.json"); @@ -325,7 +329,7 @@ public void testEmptyProjectInStatWithEval() { assertThat(exprs.size(), equalTo(1)); var alias = as(exprs.get(0), Alias.class); assertThat(alias.name(), equalTo("x")); - assertThat(alias.child().fold(), equalTo(1)); + assertThat(alias.child().fold(FoldContext.small()), equalTo(1)); } /** @@ -361,11 +365,11 @@ public void testEmptyProjectInStatWithGroupAndEval() { assertThat(exprs.size(), equalTo(1)); var alias = as(exprs.get(0), Alias.class); assertThat(alias.name(), equalTo("x")); - assertThat(alias.child().fold(), equalTo(1)); + assertThat(alias.child().fold(FoldContext.small()), equalTo(1)); var filterCondition = as(filter.condition(), GreaterThan.class); assertThat(Expressions.name(filterCondition.left()), equalTo("languages")); - assertThat(filterCondition.right().fold(), equalTo(1)); + assertThat(filterCondition.right().fold(FoldContext.small()), equalTo(1)); } public void testCombineProjections() { @@ -625,7 +629,7 @@ public void testReplaceStatsFilteredAggWithEvalSingleAggWithExpression() { assertThat(alias.name(), is("sum(salary) + 1 where false")); var add = as(alias.child(), Add.class); var literal = as(add.right(), Literal.class); - assertThat(literal.fold(), is(1)); + assertThat(literal.value(), is(1)); var limit = as(eval.child(), Limit.class); var source = as(limit.child(), LocalRelation.class); @@ -658,7 +662,7 @@ public void testReplaceStatsFilteredAggWithEvalMixedFilterAndNoFilter() { var alias = as(eval.fields().getFirst(), Alias.class); assertTrue(alias.child().foldable()); - assertThat(alias.child().fold(), nullValue()); + assertThat(alias.child().fold(FoldContext.small()), nullValue()); assertThat(alias.child().dataType(), is(LONG)); alias = as(eval.fields().getLast(), Alias.class); @@ -695,7 +699,7 @@ public void testReplaceStatsFilteredAggWithEvalFilterFalseAndNull() { var alias = as(eval.fields().getFirst(), Alias.class); assertTrue(alias.child().foldable()); - assertThat(alias.child().fold(), nullValue()); + assertThat(alias.child().fold(FoldContext.small()), nullValue()); assertThat(alias.child().dataType(), is(LONG)); alias = as(eval.fields().get(1), Alias.class); @@ -703,7 +707,7 @@ public void testReplaceStatsFilteredAggWithEvalFilterFalseAndNull() { alias = as(eval.fields().getLast(), Alias.class); assertTrue(alias.child().foldable()); - assertThat(alias.child().fold(), nullValue()); + assertThat(alias.child().fold(FoldContext.small()), nullValue()); assertThat(alias.child().dataType(), is(LONG)); var limit = as(eval.child(), Limit.class); @@ -752,7 +756,7 @@ public void testReplaceStatsFilteredAggWithEvalCountDistinctInExpression() { assertThat(alias.name(), is("count_distinct(salary + 2) + 3 where false")); var add = as(alias.child(), Add.class); var literal = as(add.right(), Literal.class); - assertThat(literal.fold(), is(3)); + assertThat(literal.value(), is(3)); var limit = as(eval.child(), Limit.class); var source = as(limit.child(), LocalRelation.class); @@ -788,13 +792,13 @@ public void testReplaceStatsFilteredAggWithEvalSameAggWithAndWithoutFilter() { var alias = as(eval.fields().getFirst(), Alias.class); assertThat(Expressions.name(alias), containsString("max_a")); assertTrue(alias.child().foldable()); - assertThat(alias.child().fold(), nullValue()); + assertThat(alias.child().fold(FoldContext.small()), nullValue()); assertThat(alias.child().dataType(), is(INTEGER)); alias = as(eval.fields().getLast(), Alias.class); assertThat(Expressions.name(alias), containsString("min_a")); assertTrue(alias.child().foldable()); - assertThat(alias.child().fold(), nullValue()); + assertThat(alias.child().fold(FoldContext.small()), nullValue()); assertThat(alias.child().dataType(), is(INTEGER)); var limit = as(eval.child(), Limit.class); @@ -933,7 +937,7 @@ public void testExtractStatsCommonFilterUsingJustOneAlias() { var gt = as(filter.condition(), GreaterThan.class); assertThat(Expressions.name(gt.left()), is("emp_no")); assertTrue(gt.right().foldable()); - assertThat(gt.right().fold(), is(1)); + assertThat(gt.right().fold(FoldContext.small()), is(1)); var source = as(filter.child(), EsRelation.class); } @@ -1053,7 +1057,7 @@ public void testExtractStatsCommonFilterInConjunction() { var gt = as(filter.condition(), GreaterThan.class); // name is "emp_no > 1 + 1" assertThat(Expressions.name(gt.left()), is("emp_no")); assertTrue(gt.right().foldable()); - assertThat(gt.right().fold(), is(2)); + assertThat(gt.right().fold(FoldContext.small()), is(2)); var source = as(filter.child(), EsRelation.class); } @@ -1083,12 +1087,12 @@ public void testExtractStatsCommonFilterInConjunctionWithMultipleCommonConjuncti var lt = as(and.left(), LessThan.class); assertThat(Expressions.name(lt.left()), is("emp_no")); assertTrue(lt.right().foldable()); - assertThat(lt.right().fold(), is(10)); + assertThat(lt.right().fold(FoldContext.small()), is(10)); var equals = as(and.right(), Equals.class); assertThat(Expressions.name(equals.left()), is("last_name")); assertTrue(equals.right().foldable()); - assertThat(equals.right().fold(), is(BytesRefs.toBytesRef("Doe"))); + assertThat(equals.right().fold(FoldContext.small()), is(BytesRefs.toBytesRef("Doe"))); var source = as(filter.child(), EsRelation.class); } @@ -1303,7 +1307,7 @@ public void testCombineLimits() { var anotherLimit = new Limit(EMPTY, L(limitValues[secondLimit]), oneLimit); assertEquals( new Limit(EMPTY, L(Math.min(limitValues[0], limitValues[1])), emptySource()), - new PushDownAndCombineLimits().rule(anotherLimit) + new PushDownAndCombineLimits().rule(anotherLimit, logicalOptimizerCtx) ); } @@ -1322,7 +1326,7 @@ public void testPushdownLimitsPastLeftJoin() { var limit = new Limit(EMPTY, L(10), join); - var optimizedPlan = new PushDownAndCombineLimits().rule(limit); + var optimizedPlan = new PushDownAndCombineLimits().rule(limit, logicalOptimizerCtx); assertEquals(join.replaceChildren(limit.replaceChild(join.left()), join.right()), optimizedPlan); } @@ -1340,10 +1344,7 @@ public void testMultipleCombineLimits() { var value = i == limitWithMinimum ? minimum : randomIntBetween(100, 1000); plan = new Limit(EMPTY, L(value), plan); } - assertEquals( - new Limit(EMPTY, L(minimum), relation), - new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)).optimize(plan) - ); + assertEquals(new Limit(EMPTY, L(minimum), relation), logicalOptimizer.optimize(plan)); } @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/115311") @@ -1864,7 +1865,7 @@ public void testCopyDefaultLimitPastMvExpand() { assertThat(mvExpand.limit(), equalTo(1000)); var keep = as(mvExpand.child(), EsqlProject.class); var limitPastMvExpand = as(keep.child(), Limit.class); - assertThat(limitPastMvExpand.limit().fold(), equalTo(1000)); + assertThat(limitPastMvExpand.limit().fold(FoldContext.small()), equalTo(1000)); as(limitPastMvExpand.child(), EsRelation.class); } @@ -1887,7 +1888,7 @@ public void testDontPushDownLimitPastMvExpand() { assertThat(mvExpand.limit(), equalTo(10)); var project = as(mvExpand.child(), EsqlProject.class); var limit = as(project.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(1)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1)); as(limit.child(), EsRelation.class); } @@ -1921,7 +1922,7 @@ public void testMultipleMvExpandWithSortAndLimit() { var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(5)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(5)); assertThat(orderNames(topN), contains("salary")); var mvExp = as(topN.child(), MvExpand.class); assertThat(mvExp.limit(), equalTo(5)); @@ -1931,7 +1932,7 @@ public void testMultipleMvExpandWithSortAndLimit() { mvExp = as(filter.child(), MvExpand.class); assertThat(mvExp.limit(), equalTo(10)); topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(10)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10)); filter = as(topN.child(), Filter.class); as(filter.child(), EsRelation.class); } @@ -1955,11 +1956,11 @@ public void testPushDownLimitThroughMultipleSort_AfterMvExpand() { var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(5)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(5)); assertThat(orderNames(topN), contains("salary", "first_name")); var mvExp = as(topN.child(), MvExpand.class); topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); assertThat(orderNames(topN), contains("emp_no")); as(topN.child(), EsRelation.class); } @@ -1985,14 +1986,14 @@ public void testPushDownLimitThroughMultipleSort_AfterMvExpand2() { var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(5)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(5)); assertThat(orderNames(topN), contains("first_name")); topN = as(topN.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(5)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(5)); assertThat(orderNames(topN), contains("salary")); var mvExp = as(topN.child(), MvExpand.class); topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); assertThat(orderNames(topN), contains("emp_no")); as(topN.child(), EsRelation.class); } @@ -2021,11 +2022,11 @@ public void testDontPushDownLimitPastAggregate_AndMvExpand() { var limit = as(plan, Limit.class); var filter = as(limit.child(), Filter.class); - assertThat(limit.limit().fold(), equalTo(5)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(5)); var agg = as(filter.child(), Aggregate.class); var mvExp = as(agg.child(), MvExpand.class); var topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(50)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(50)); assertThat(orderNames(topN), contains("emp_no")); as(topN.child(), EsRelation.class); } @@ -2052,13 +2053,13 @@ public void testPushDown_TheRightLimit_PastMvExpand() { | limit 5"""); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(5)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(5)); var filter = as(limit.child(), Filter.class); var agg = as(filter.child(), Aggregate.class); var mvExp = as(agg.child(), MvExpand.class); assertThat(mvExp.limit(), equalTo(50)); limit = as(mvExp.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(50)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(50)); as(limit.child(), EsRelation.class); } @@ -2083,12 +2084,12 @@ public void testPushDownLimit_PastEvalAndMvExpand() { var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(5)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(5)); assertThat(orderNames(topN), contains("salary")); var eval = as(topN.child(), Eval.class); var mvExp = as(eval.child(), MvExpand.class); topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); assertThat(orderNames(topN), contains("first_name")); as(topN.child(), EsRelation.class); } @@ -2114,7 +2115,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedField_ResultT var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(1000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(1000)); assertThat(orderNames(topN), contains("salary", "first_name")); var filter = as(topN.child(), Filter.class); assertThat(filter.condition(), instanceOf(And.class)); @@ -2143,7 +2144,7 @@ public void testFilterWithSortBeforeMvExpand() { var mvExp = as(plan, MvExpand.class); assertThat(mvExp.limit(), equalTo(10)); var topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(10)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10)); assertThat(orderNames(topN), contains("emp_no")); var filter = as(topN.child(), Filter.class); as(filter.child(), EsRelation.class); @@ -2168,7 +2169,7 @@ public void testMultiMvExpand_SortDownBelow() { | sort first_name"""); var topN = as(plan, TopN.class); - assertThat(topN.limit().fold(), equalTo(1000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(1000)); assertThat(orderNames(topN), contains("first_name")); var mvExpand = as(topN.child(), MvExpand.class); var filter = as(mvExpand.child(), Filter.class); @@ -2200,11 +2201,11 @@ public void testLimitThenSortBeforeMvExpand() { assertThat(mvExpand.limit(), equalTo(10000)); var project = as(mvExpand.child(), EsqlProject.class); var topN = as(project.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(7300)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(7300)); assertThat(orderNames(topN), contains("a")); mvExpand = as(topN.child(), MvExpand.class); var limit = as(mvExpand.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(7300)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(7300)); as(limit.child(), LocalRelation.class); } @@ -2224,7 +2225,7 @@ public void testRemoveUnusedSortBeforeMvExpand_DefaultLimit10000() { var topN = as(plan, TopN.class); assertThat(orderNames(topN), contains("first_name")); - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); var mvExpand = as(topN.child(), MvExpand.class); var topN2 = as(mvExpand.child(), TopN.class); // TODO is it correct? Double-check AddDefaultTopN rule as(topN2.child(), EsRelation.class); @@ -2252,7 +2253,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedField() { var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(15)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(15)); assertThat(orderNames(topN), contains("salary", "first_name")); var filter = as(topN.child(), Filter.class); assertThat(filter.condition(), instanceOf(And.class)); @@ -2260,7 +2261,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedField() { topN = as(mvExp.child(), TopN.class); // the filter acts on first_name (the one used in mv_expand), so the limit 15 is not pushed down past mv_expand // instead the default limit is added - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); assertThat(orderNames(topN), contains("emp_no")); as(topN.child(), EsRelation.class); } @@ -2287,7 +2288,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilter_NOT_OnExpandedField() var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(15)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(15)); assertThat(orderNames(topN), contains("salary", "first_name")); var filter = as(topN.child(), Filter.class); assertThat(filter.condition(), instanceOf(And.class)); @@ -2295,7 +2296,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilter_NOT_OnExpandedField() topN = as(mvExp.child(), TopN.class); // the filters after mv_expand do not act on the expanded field values, as such the limit 15 is the one being pushed down // otherwise that limit wouldn't have pushed down and the default limit was instead being added by default before mv_expanded - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); assertThat(orderNames(topN), contains("emp_no")); as(topN.child(), EsRelation.class); } @@ -2323,14 +2324,14 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedFieldAlias() var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(15)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(15)); assertThat(orderNames(topN), contains("salary", "first_name")); var filter = as(topN.child(), Filter.class); assertThat(filter.condition(), instanceOf(And.class)); var mvExp = as(filter.child(), MvExpand.class); topN = as(mvExp.child(), TopN.class); // the filter uses an alias ("x") to the expanded field ("first_name"), so the default limit is used and not the one provided - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); assertThat(orderNames(topN), contains("gender")); as(topN.child(), EsRelation.class); } @@ -2369,7 +2370,7 @@ public void testSortMvExpandLimit() { var expand = as(plan, MvExpand.class); assertThat(expand.limit(), equalTo(20)); var topN = as(expand.child(), TopN.class); - assertThat(topN.limit().fold(), is(20)); + assertThat(topN.limit().fold(FoldContext.small()), is(20)); var row = as(topN.child(), EsRelation.class); } @@ -2390,7 +2391,7 @@ public void testWhereMvExpand() { var expand = as(plan, MvExpand.class); assertThat(expand.limit(), equalTo(1000)); var limit2 = as(expand.child(), Limit.class); - assertThat(limit2.limit().fold(), is(1000)); + assertThat(limit2.limit().fold(FoldContext.small()), is(1000)); var row = as(limit2.child(), LocalRelation.class); } @@ -2583,7 +2584,7 @@ public void testSimplifyLikeNoWildcard() { assertTrue(filter.condition() instanceof Equals); Equals equals = as(filter.condition(), Equals.class); - assertEquals(BytesRefs.toBytesRef("foo"), equals.right().fold()); + assertEquals(BytesRefs.toBytesRef("foo"), equals.right().fold(FoldContext.small())); assertTrue(filter.child() instanceof EsRelation); } @@ -2609,7 +2610,7 @@ public void testSimplifyRLikeNoWildcard() { assertTrue(filter.condition() instanceof Equals); Equals equals = as(filter.condition(), Equals.class); - assertEquals(BytesRefs.toBytesRef("foo"), equals.right().fold()); + assertEquals(BytesRefs.toBytesRef("foo"), equals.right().fold(FoldContext.small())); assertTrue(filter.child() instanceof EsRelation); } @@ -2773,7 +2774,7 @@ public void testEnrich() { """); var enrich = as(plan, Enrich.class); assertTrue(enrich.policyName().resolved()); - assertThat(enrich.policyName().fold(), is(BytesRefs.toBytesRef("languages_idx"))); + assertThat(enrich.policyName().fold(FoldContext.small()), is(BytesRefs.toBytesRef("languages_idx"))); var eval = as(enrich.child(), Eval.class); var limit = as(eval.child(), Limit.class); as(limit.child(), EsRelation.class); @@ -2819,7 +2820,7 @@ public void testEnrichNotNullFilter() { var filter = as(limit.child(), Filter.class); var enrich = as(filter.child(), Enrich.class); assertTrue(enrich.policyName().resolved()); - assertThat(enrich.policyName().fold(), is(BytesRefs.toBytesRef("languages_idx"))); + assertThat(enrich.policyName().fold(FoldContext.small()), is(BytesRefs.toBytesRef("languages_idx"))); var eval = as(enrich.child(), Eval.class); as(eval.child(), EsRelation.class); } @@ -2940,7 +2941,7 @@ public void testMedianReplacement() { var a = as(aggs.get(0), Alias.class); var per = as(a.child(), Percentile.class); var literal = as(per.percentile(), Literal.class); - assertThat((int) QuantileStates.MEDIAN, is(literal.fold())); + assertThat((int) QuantileStates.MEDIAN, is(literal.value())); assertThat(Expressions.names(agg.groupings()), contains("last_name")); } @@ -2949,7 +2950,7 @@ public void testSplittingInWithFoldableValue() { FieldAttribute fa = getFieldAttribute("foo"); In in = new In(EMPTY, ONE, List.of(TWO, THREE, fa, L(null))); Or expected = new Or(EMPTY, new In(EMPTY, ONE, List.of(TWO, THREE)), new In(EMPTY, ONE, List.of(fa, L(null)))); - assertThat(new SplitInWithFoldableValue().rule(in), equalTo(expected)); + assertThat(new SplitInWithFoldableValue().rule(in, logicalOptimizerCtx), equalTo(expected)); } public void testReplaceFilterWithExact() { @@ -3706,7 +3707,7 @@ private void aggFieldName(Expression exp, Class var alias = as(exp, Alias.class); var af = as(alias.child(), aggType); var field = af.field(); - var name = field.foldable() ? BytesRefs.toString(field.fold()) : Expressions.name(field); + var name = field.foldable() ? BytesRefs.toString(field.fold(FoldContext.small())) : Expressions.name(field); assertThat(name, is(fieldName)); } @@ -4118,7 +4119,7 @@ public void testNestedExpressionsWithGroupingKeyInAggs() { var value = Alias.unwrap(fields.get(0)); var math = as(value, Mod.class); assertThat(Expressions.name(math.left()), is("emp_no")); - assertThat(math.right().fold(), is(2)); + assertThat(math.right().fold(FoldContext.small()), is(2)); // languages + emp_no % 2 var add = as(Alias.unwrap(fields.get(1).canonical()), Add.class); if (add.left() instanceof Mod mod) { @@ -4127,7 +4128,7 @@ public void testNestedExpressionsWithGroupingKeyInAggs() { assertThat(Expressions.name(add.left()), is("languages")); var mod = as(add.right().canonical(), Mod.class); assertThat(Expressions.name(mod.left()), is("emp_no")); - assertThat(mod.right().fold(), is(2)); + assertThat(mod.right().fold(FoldContext.small()), is(2)); } /** @@ -4156,7 +4157,7 @@ public void testNestedExpressionsWithMultiGrouping() { var value = Alias.unwrap(fields.get(0).canonical()); var math = as(value, Mod.class); assertThat(Expressions.name(math.left()), is("emp_no")); - assertThat(math.right().fold(), is(2)); + assertThat(math.right().fold(FoldContext.small()), is(2)); // languages + salary var add = as(Alias.unwrap(fields.get(1).canonical()), Add.class); assertThat(Expressions.name(add.left()), anyOf(is("languages"), is("salary"))); @@ -4173,7 +4174,7 @@ public void testNestedExpressionsWithMultiGrouping() { assertThat(Expressions.name(add3.right()), anyOf(is("salary"), is("languages"))); // emp_no % 2 assertThat(Expressions.name(mod.left()), is("emp_no")); - assertThat(mod.right().fold(), is(2)); + assertThat(mod.right().fold(FoldContext.small()), is(2)); } /** @@ -4611,8 +4612,8 @@ public void testCountOfLiteral() { var mvCoalesce = as(mul.left(), Coalesce.class); assertThat(mvCoalesce.children().size(), equalTo(2)); var mvCount = as(mvCoalesce.children().get(0), MvCount.class); - assertThat(mvCount.fold(), equalTo(2)); - assertThat(mvCoalesce.children().get(1).fold(), equalTo(0)); + assertThat(mvCount.fold(FoldContext.small()), equalTo(2)); + assertThat(mvCoalesce.children().get(1).fold(FoldContext.small()), equalTo(0)); var count = as(mul.right(), ReferenceAttribute.class); assertThat(count.name(), equalTo("$$COUNT$s$0")); @@ -4623,8 +4624,8 @@ public void testCountOfLiteral() { var mvCoalesce_expr = as(mul_expr.left(), Coalesce.class); assertThat(mvCoalesce_expr.children().size(), equalTo(2)); var mvCount_expr = as(mvCoalesce_expr.children().get(0), MvCount.class); - assertThat(mvCount_expr.fold(), equalTo(1)); - assertThat(mvCoalesce_expr.children().get(1).fold(), equalTo(0)); + assertThat(mvCount_expr.fold(FoldContext.small()), equalTo(1)); + assertThat(mvCoalesce_expr.children().get(1).fold(FoldContext.small()), equalTo(0)); var count_expr = as(mul_expr.right(), ReferenceAttribute.class); assertThat(count_expr.name(), equalTo("$$COUNT$s$0")); @@ -4636,7 +4637,7 @@ public void testCountOfLiteral() { assertThat(mvCoalesce_null.children().size(), equalTo(2)); var mvCount_null = as(mvCoalesce_null.children().get(0), MvCount.class); assertThat(mvCount_null.field(), equalTo(NULL)); - assertThat(mvCoalesce_null.children().get(1).fold(), equalTo(0)); + assertThat(mvCoalesce_null.children().get(1).fold(FoldContext.small()), equalTo(0)); var count_null = as(mul_null.right(), ReferenceAttribute.class); assertThat(count_null.name(), equalTo("$$COUNT$s$0")); } @@ -4675,7 +4676,7 @@ public void testSumOfLiteral() { assertThat(s.name(), equalTo("s")); var mul = as(s.child(), Mul.class); var mvSum = as(mul.left(), MvSum.class); - assertThat(mvSum.fold(), equalTo(3)); + assertThat(mvSum.fold(FoldContext.small()), equalTo(3)); var count = as(mul.right(), ReferenceAttribute.class); assertThat(count.name(), equalTo("$$COUNT$s$0")); @@ -4684,7 +4685,7 @@ public void testSumOfLiteral() { assertThat(s_expr.name(), equalTo("s_expr")); var mul_expr = as(s_expr.child(), Mul.class); var mvSum_expr = as(mul_expr.left(), MvSum.class); - assertThat(mvSum_expr.fold(), equalTo(3.14)); + assertThat(mvSum_expr.fold(FoldContext.small()), equalTo(3.14)); var count_expr = as(mul_expr.right(), ReferenceAttribute.class); assertThat(count_expr.name(), equalTo("$$COUNT$s$0")); @@ -4833,7 +4834,7 @@ private static void assertAggOfConstExprs(AggOfLiteralTestCase testCase, List x.foldable() ? new Literal(x.source(), x.fold(), x.dataType()) : x); + be -> LITERALS_ON_THE_RIGHT.rule(be, logicalOptimizerCtx) + ).transformUp(x -> x.foldable() ? new Literal(x.source(), x.fold(FoldContext.small()), x.dataType()) : x); List resolvedFields = fieldAttributeExp.collectFirstChildren(x -> x instanceof FieldAttribute); for (Expression field : resolvedFields) { @@ -5679,8 +5680,7 @@ public void testSimplifyComparisonArithmeticWithFloatsAndDirectionChange() { } private void assertNullLiteral(Expression expression) { - assertEquals(Literal.class, expression.getClass()); - assertNull(expression.fold()); + assertNull(as(expression, Literal.class).value()); } @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108519") @@ -5776,7 +5776,7 @@ public void testReplaceStringCasingWithInsensitiveEqualsEquals() { var filter = as(limit.child(), Filter.class); var insensitive = as(filter.condition(), InsensitiveEquals.class); as(insensitive.left(), FieldAttribute.class); - var bRef = as(insensitive.right().fold(), BytesRef.class); + var bRef = as(insensitive.right().fold(FoldContext.small()), BytesRef.class); assertThat(bRef.utf8ToString(), is(value)); as(filter.child(), EsRelation.class); } @@ -5792,7 +5792,7 @@ public void testReplaceStringCasingWithInsensitiveEqualsNotEquals() { var not = as(filter.condition(), Not.class); var insensitive = as(not.field(), InsensitiveEquals.class); as(insensitive.left(), FieldAttribute.class); - var bRef = as(insensitive.right().fold(), BytesRef.class); + var bRef = as(insensitive.right().fold(FoldContext.small()), BytesRef.class); assertThat(bRef.utf8ToString(), is(value)); as(filter.child(), EsRelation.class); } @@ -5805,7 +5805,7 @@ public void testReplaceStringCasingWithInsensitiveEqualsUnwrap() { var insensitive = as(filter.condition(), InsensitiveEquals.class); var field = as(insensitive.left(), FieldAttribute.class); assertThat(field.fieldName(), is("first_name")); - var bRef = as(insensitive.right().fold(), BytesRef.class); + var bRef = as(insensitive.right().fold(FoldContext.small()), BytesRef.class); assertThat(bRef.utf8ToString(), is("VALÜ")); as(filter.child(), EsRelation.class); } @@ -5856,7 +5856,7 @@ public void testLookupSimple() { var left = as(join.left(), EsqlProject.class); assertThat(left.output().toString(), containsString("int{r}")); var limit = as(left.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); assertThat(join.config().matchFields().stream().map(Object::toString).toList(), matchesList().item(startsWith("int{r}"))); @@ -5925,7 +5925,7 @@ public void testLookupStats() { } var plan = optimizedPlan(query); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); var agg = as(limit.child(), Aggregate.class); assertMap( @@ -6017,7 +6017,7 @@ public void testLookupJoinPushDownFilterOnJoinKeyWithRename() { assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); var project = as(join.left(), Project.class); var limit = as(project.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); var filter = as(limit.child(), Filter.class); // assert that the rename has been undone var op = as(filter.condition(), GreaterThan.class); @@ -6061,7 +6061,7 @@ public void testLookupJoinPushDownFilterOnLeftSideField() { var project = as(join.left(), Project.class); var limit = as(project.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); var filter = as(limit.child(), Filter.class); var op = as(filter.condition(), GreaterThan.class); var field = as(op.left(), FieldAttribute.class); @@ -6100,7 +6100,7 @@ public void testLookupJoinPushDownDisabledForLookupField() { var plan = optimizedPlan(query); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); var filter = as(limit.child(), Filter.class); var op = as(filter.condition(), Equals.class); @@ -6144,7 +6144,7 @@ public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightFiel var plan = optimizedPlan(query); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); // filter kept in place, working on the right side var filter = as(limit.child(), Filter.class); EsqlBinaryComparison op = as(filter.condition(), Equals.class); @@ -6195,7 +6195,7 @@ public void testLookupJoinPushDownDisabledForDisjunctionBetweenLeftAndRightField var plan = optimizedPlan(query); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); var filter = as(limit.child(), Filter.class); var or = as(filter.condition(), Or.class); @@ -6289,7 +6289,7 @@ public void testTranslateMixedAggsWithMathWithoutGrouping() { as(addEval.child(), EsRelation.class); assertThat(Expressions.attribute(mul.left()).id(), equalTo(finalAggs.aggregates().get(1).id())); - assertThat(mul.right().fold(), equalTo(1.1)); + assertThat(mul.right().fold(FoldContext.small()), equalTo(1.1)); assertThat(finalAggs.aggregateType(), equalTo(Aggregate.AggregateType.STANDARD)); Max maxRate = as(Alias.unwrap(finalAggs.aggregates().get(0)), Max.class); @@ -6304,7 +6304,7 @@ public void testTranslateMixedAggsWithMathWithoutGrouping() { ToPartial toPartialMaxCost = as(Alias.unwrap(aggsByTsid.aggregates().get(1)), ToPartial.class); assertThat(Expressions.attribute(toPartialMaxCost.field()).id(), equalTo(addEval.fields().get(0).id())); assertThat(Expressions.attribute(add.left()).name(), equalTo("network.cost")); - assertThat(add.right().fold(), equalTo(0.2)); + assertThat(add.right().fold(FoldContext.small()), equalTo(0.2)); } public void testTranslateMetricsGroupedByOneDimension() { @@ -6533,7 +6533,7 @@ METRICS k8s avg(round(1.05 * rate(network.total_bytes_in))) BY bucket(@timestamp assertThat(Expressions.attribute(finalAgg.groupings().get(1)).id(), equalTo(aggsByTsid.aggregates().get(1).id())); assertThat(Expressions.attribute(mul.left()).id(), equalTo(aggsByTsid.aggregates().get(0).id())); - assertThat(mul.right().fold(), equalTo(1.05)); + assertThat(mul.right().fold(FoldContext.small()), equalTo(1.05)); assertThat(aggsByTsid.aggregateType(), equalTo(Aggregate.AggregateType.METRICS)); Rate rate = as(Alias.unwrap(aggsByTsid.aggregates().get(0)), Rate.class); assertThat(Expressions.attribute(rate.field()).name(), equalTo("network.total_bytes_in")); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index ff710a90e8154..66891210a1e47 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -51,6 +51,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; @@ -164,6 +165,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.statsForMissingField; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze; @@ -245,7 +247,7 @@ public PhysicalPlanOptimizerTests(String name, Configuration config) { @Before public void init() { parser = new EsqlParser(); - logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)); + logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext()); physicalPlanOptimizer = new PhysicalPlanOptimizer(new PhysicalOptimizerContext(config)); EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry(); mapper = new Mapper(); @@ -1117,7 +1119,7 @@ public void testLimit() { var fieldExtract = as(project.child(), FieldExtractExec.class); var source = source(fieldExtract.child()); assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES)); - assertThat(source.limit().fold(), is(10)); + assertThat(source.limit().fold(FoldContext.small()), is(10)); } /** @@ -1199,7 +1201,7 @@ public void testPushLimitToSource() { var leaves = extract.collectLeaves(); assertEquals(1, leaves.size()); var source = as(leaves.get(0), EsQueryExec.class); - assertThat(source.limit().fold(), is(10)); + assertThat(source.limit().fold(FoldContext.small()), is(10)); // extra ints for doc id and emp_no_10 assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES * 2)); } @@ -1246,7 +1248,7 @@ public void testPushLimitAndFilterToSource() { var source = source(extract.child()); assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES * 2)); - assertThat(source.limit().fold(), is(10)); + assertThat(source.limit().fold(FoldContext.small()), is(10)); var rq = as(sv(source.query(), "emp_no"), RangeQueryBuilder.class); assertThat(rq.fieldName(), equalTo("emp_no")); assertThat(rq.from(), equalTo(0)); @@ -2902,14 +2904,14 @@ public void testAvgSurrogateFunctionAfterRenameAndLimit() { var eval = as(project.child(), EvalExec.class); var limit = as(eval.child(), LimitExec.class); assertThat(limit.limit(), instanceOf(Literal.class)); - assertThat(limit.limit().fold(), equalTo(10000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(10000)); var aggFinal = as(limit.child(), AggregateExec.class); assertThat(aggFinal.getMode(), equalTo(FINAL)); var aggPartial = as(aggFinal.child(), AggregateExec.class); assertThat(aggPartial.getMode(), equalTo(INITIAL)); limit = as(aggPartial.child(), LimitExec.class); assertThat(limit.limit(), instanceOf(Literal.class)); - assertThat(limit.limit().fold(), equalTo(10)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(10)); var exchange = as(limit.child(), ExchangeExec.class); project = as(exchange.child(), ProjectExec.class); @@ -2918,7 +2920,7 @@ public void testAvgSurrogateFunctionAfterRenameAndLimit() { var fieldExtract = as(project.child(), FieldExtractExec.class); assertThat(Expressions.names(fieldExtract.attributesToExtract()), is(expectedFields)); var source = source(fieldExtract.child()); - assertThat(source.limit().fold(), equalTo(10)); + assertThat(source.limit().fold(FoldContext.small()), equalTo(10)); } /** @@ -4849,15 +4851,15 @@ public void testPushSpatialDistanceMultiEvalToSource() { var alias1 = as(eval.fields().get(0), Alias.class); assertThat(alias1.name(), is("poi")); var poi = as(alias1.child(), Literal.class); - assertThat(poi.fold(), instanceOf(BytesRef.class)); + assertThat(poi.value(), instanceOf(BytesRef.class)); var alias2 = as(eval.fields().get(1), Alias.class); assertThat(alias2.name(), is("distance")); var stDistance = as(alias2.child(), StDistance.class); var location = as(stDistance.left(), FieldAttribute.class); assertThat(location.fieldName(), is("location")); var poiRef = as(stDistance.right(), Literal.class); - assertThat(poiRef.fold(), instanceOf(BytesRef.class)); - assertThat(poiRef.fold().toString(), is(poi.fold().toString())); + assertThat(poiRef.value(), instanceOf(BytesRef.class)); + assertThat(poiRef.value().toString(), is(poi.value().toString())); // Validate the filter condition var and = as(filter.condition(), And.class); @@ -6205,15 +6207,15 @@ public void testPushCompoundTopNDistanceWithCompoundFilterAndCompoundEvalToSourc var alias1 = as(evalExec.fields().get(0), Alias.class); assertThat(alias1.name(), is("poi")); var poi = as(alias1.child(), Literal.class); - assertThat(poi.fold(), instanceOf(BytesRef.class)); + assertThat(poi.value(), instanceOf(BytesRef.class)); var alias2 = as(evalExec.fields().get(1), Alias.class); assertThat(alias2.name(), is("distance")); var stDistance = as(alias2.child(), StDistance.class); var location = as(stDistance.left(), FieldAttribute.class); assertThat(location.fieldName(), is("location")); var poiRef = as(stDistance.right(), Literal.class); - assertThat(poiRef.fold(), instanceOf(BytesRef.class)); - assertThat(poiRef.fold().toString(), is(poi.fold().toString())); + assertThat(poiRef.value(), instanceOf(BytesRef.class)); + assertThat(poiRef.value().toString(), is(poi.value().toString())); extract = as(evalExec.child(), FieldExtractExec.class); assertThat(names(extract.attributesToExtract()), contains("location")); var source = source(extract.child()); @@ -6294,7 +6296,7 @@ public void testPushCompoundTopNDistanceWithDeeplyNestedCompoundEvalToSource() { var alias1 = as(evalExec.fields().get(0), Alias.class); assertThat(alias1.name(), is("poi")); var poi = as(alias1.child(), Literal.class); - assertThat(poi.fold(), instanceOf(BytesRef.class)); + assertThat(poi.value(), instanceOf(BytesRef.class)); var alias4 = as(evalExec.fields().get(3), Alias.class); assertThat(alias4.name(), is("loc2")); as(alias4.child(), FieldAttribute.class); @@ -6307,8 +6309,8 @@ public void testPushCompoundTopNDistanceWithDeeplyNestedCompoundEvalToSource() { var refLocation = as(stDistance.left(), ReferenceAttribute.class); assertThat(refLocation.name(), is("loc3")); var poiRef = as(stDistance.right(), Literal.class); - assertThat(poiRef.fold(), instanceOf(BytesRef.class)); - assertThat(poiRef.fold().toString(), is(poi.fold().toString())); + assertThat(poiRef.value(), instanceOf(BytesRef.class)); + assertThat(poiRef.value().toString(), is(poi.value().toString())); var alias7 = as(evalExec.fields().get(6), Alias.class); assertThat(alias7.name(), is("distance")); as(alias7.child(), ReferenceAttribute.class); @@ -6391,15 +6393,15 @@ public void testPushCompoundTopNDistanceWithCompoundFilterAndNestedCompoundEvalT var alias1 = as(evalExec.fields().get(0), Alias.class); assertThat(alias1.name(), is("poi")); var poi = as(alias1.child(), Literal.class); - assertThat(poi.fold(), instanceOf(BytesRef.class)); + assertThat(poi.value(), instanceOf(BytesRef.class)); var alias2 = as(evalExec.fields().get(1), Alias.class); assertThat(alias2.name(), is("distance")); var stDistance = as(alias2.child(), StDistance.class); var location = as(stDistance.left(), FieldAttribute.class); assertThat(location.fieldName(), is("location")); var poiRef = as(stDistance.right(), Literal.class); - assertThat(poiRef.fold(), instanceOf(BytesRef.class)); - assertThat(poiRef.fold().toString(), is(poi.fold().toString())); + assertThat(poiRef.value(), instanceOf(BytesRef.class)); + assertThat(poiRef.value().toString(), is(poi.value().toString())); extract = as(evalExec.child(), FieldExtractExec.class); assertThat(names(extract.attributesToExtract()), contains("location")); var source = source(extract.child()); @@ -6931,7 +6933,7 @@ public void testManyEnrich() { var fragment = as(exchange.child(), FragmentExec.class); var partialTopN = as(fragment.fragment(), TopN.class); var enrich2 = as(partialTopN.child(), Enrich.class); - assertThat(BytesRefs.toString(enrich2.policyName().fold()), equalTo("departments")); + assertThat(BytesRefs.toString(enrich2.policyName().fold(FoldContext.small())), equalTo("departments")); assertThat(enrich2.mode(), equalTo(Enrich.Mode.ANY)); var eval = as(enrich2.child(), Eval.class); as(eval.child(), EsRelation.class); @@ -6957,7 +6959,7 @@ public void testManyEnrich() { var fragment = as(exchange.child(), FragmentExec.class); var partialTopN = as(fragment.fragment(), TopN.class); var enrich2 = as(partialTopN.child(), Enrich.class); - assertThat(BytesRefs.toString(enrich2.policyName().fold()), equalTo("departments")); + assertThat(BytesRefs.toString(enrich2.policyName().fold(FoldContext.small())), equalTo("departments")); assertThat(enrich2.mode(), equalTo(Enrich.Mode.ANY)); var eval = as(enrich2.child(), Eval.class); as(eval.child(), EsRelation.class); @@ -7549,7 +7551,7 @@ private LocalExecutionPlanner.LocalExecutionPlan physicalOperationsFromPhysicalP // The TopN needs an estimated row size for the planner to work var plans = PlannerUtils.breakPlanBetweenCoordinatorAndDataNode(EstimatesRowSize.estimateRowSize(0, plan), config); plan = useDataNodePlan ? plans.v2() : plans.v1(); - plan = PlannerUtils.localPlan(List.of(), config, plan); + plan = PlannerUtils.localPlan(List.of(), config, FoldContext.small(), plan); LocalExecutionPlanner planner = new LocalExecutionPlanner( "test", "", @@ -7562,10 +7564,10 @@ private LocalExecutionPlanner.LocalExecutionPlan physicalOperationsFromPhysicalP new ExchangeSinkHandler(null, 10, () -> 10), null, null, - new EsPhysicalOperationProviders(List.of(), null) + new EsPhysicalOperationProviders(FoldContext.small(), List.of(), null) ); - return planner.plan(plan); + return planner.plan(FoldContext.small(), plan); } private List> findFieldNamesInLookupJoinDescription(LocalExecutionPlanner.LocalExecutionPlan physicalOperations) { @@ -7662,7 +7664,7 @@ public void testReductionPlanForTopN() { PhysicalPlan reduction = PlannerUtils.reductionPlan(plans.v2()); TopNExec reductionTopN = as(reduction, TopNExec.class); assertThat(reductionTopN.estimatedRowSize(), equalTo(allFieldRowSize)); - assertThat(reductionTopN.limit().fold(), equalTo(limit)); + assertThat(reductionTopN.limit().fold(FoldContext.small()), equalTo(limit)); } public void testReductionPlanForAggs() { @@ -7818,7 +7820,7 @@ private PhysicalPlan optimizedPlan(PhysicalPlan plan, SearchStats searchStats) { // individually hence why here the plan is kept as is var l = p.transformUp(FragmentExec.class, fragment -> { - var localPlan = PlannerUtils.localPlan(config, fragment, searchStats); + var localPlan = PlannerUtils.localPlan(config, FoldContext.small(), fragment, searchStats); return EstimatesRowSize.estimateRowSize(fragment.estimatedRowSize(), localPlan); }); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java index 9fe479dbb8625..e6a7d110f8c09 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.analysis.Analyzer; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; @@ -26,7 +27,7 @@ public class TestPlannerOptimizer { private final Configuration config; public TestPlannerOptimizer(Configuration config, Analyzer analyzer) { - this(config, analyzer, new LogicalPlanOptimizer(new LogicalOptimizerContext(config))); + this(config, analyzer, new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()))); } public TestPlannerOptimizer(Configuration config, Analyzer analyzer, LogicalPlanOptimizer logicalOptimizer) { @@ -61,8 +62,13 @@ private PhysicalPlan optimizedPlan(PhysicalPlan plan, SearchStats searchStats) { // this is of no use in the unit tests, which checks the plan as a whole instead of each // individually hence why here the plan is kept as is - var logicalTestOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(config, searchStats)); - var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(config, searchStats), true); + var logicalTestOptimizer = new LocalLogicalPlanOptimizer( + new LocalLogicalOptimizerContext(config, FoldContext.small(), searchStats) + ); + var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer( + new LocalPhysicalOptimizerContext(config, FoldContext.small(), searchStats), + true + ); var l = PlannerUtils.localPlan(physicalPlan, logicalTestOptimizer, physicalTestOptimizer); // handle local reduction alignment diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/LogicalOptimizerContextTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/LogicalOptimizerContextTests.java new file mode 100644 index 0000000000000..5d2fec0fc8181 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/LogicalOptimizerContextTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; +import org.elasticsearch.xpack.esql.ConfigurationTestUtils; +import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; +import org.elasticsearch.xpack.esql.session.Configuration; + +import static org.hamcrest.Matchers.equalTo; + +public class LogicalOptimizerContextTests extends ESTestCase { + public void testToString() { + // Random looking numbers for FoldContext are indeed random. Just so we have consistent numbers to assert on in toString. + LogicalOptimizerContext ctx = new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG, new FoldContext(102)); + ctx.foldCtx().trackAllocation(Source.EMPTY, 99); + assertThat( + ctx.toString(), + equalTo("LogicalOptimizerContext[configuration=" + EsqlTestUtils.TEST_CFG + ", foldCtx=FoldContext[3/102]]") + ); + } + + public void testEqualsAndHashCode() { + EqualsHashCodeTestUtils.checkEqualsAndHashCode(randomLogicalOptimizerContext(), this::copy, this::mutate); + } + + private LogicalOptimizerContext randomLogicalOptimizerContext() { + return new LogicalOptimizerContext(ConfigurationTestUtils.randomConfiguration(), randomFoldContext()); + } + + private LogicalOptimizerContext copy(LogicalOptimizerContext c) { + return new LogicalOptimizerContext(c.configuration(), c.foldCtx()); + } + + private LogicalOptimizerContext mutate(LogicalOptimizerContext c) { + Configuration configuration = c.configuration(); + FoldContext foldCtx = c.foldCtx(); + if (randomBoolean()) { + configuration = randomValueOtherThan(configuration, ConfigurationTestUtils::randomConfiguration); + } else { + foldCtx = randomValueOtherThan(foldCtx, this::randomFoldContext); + } + return new LogicalOptimizerContext(configuration, foldCtx); + } + + private FoldContext randomFoldContext() { + FoldContext ctx = new FoldContext(randomNonNegativeLong()); + if (randomBoolean()) { + ctx.trackAllocation(Source.EMPTY, randomLongBetween(0, ctx.initialAllowedBytes())); + } + return ctx; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsEliminationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsEliminationTests.java index 08c8612d8097c..c0c145aee5382 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsEliminationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsEliminationTests.java @@ -22,25 +22,26 @@ import static java.util.Arrays.asList; import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute; import static org.elasticsearch.xpack.esql.EsqlTestUtils.notEqualsOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; public class BooleanFunctionEqualsEliminationTests extends ESTestCase { + private Expression booleanFunctionEqualElimination(BinaryComparison e) { + return new BooleanFunctionEqualsElimination().rule(e, unboundLogicalOptimizerContext()); + } public void testBoolEqualsSimplificationOnExpressions() { - BooleanFunctionEqualsElimination s = new BooleanFunctionEqualsElimination(); Expression exp = new GreaterThan(EMPTY, getFieldAttribute(), new Literal(EMPTY, 0, DataType.INTEGER), null); - assertEquals(exp, s.rule(new Equals(EMPTY, exp, TRUE))); + assertEquals(exp, booleanFunctionEqualElimination(new Equals(EMPTY, exp, TRUE))); // TODO: Replace use of QL Not with ESQL Not - assertEquals(new Not(EMPTY, exp), s.rule(new Equals(EMPTY, exp, FALSE))); + assertEquals(new Not(EMPTY, exp), booleanFunctionEqualElimination(new Equals(EMPTY, exp, FALSE))); } public void testBoolEqualsSimplificationOnFields() { - BooleanFunctionEqualsElimination s = new BooleanFunctionEqualsElimination(); - FieldAttribute field = getFieldAttribute(); List comparisons = asList( @@ -55,7 +56,7 @@ public void testBoolEqualsSimplificationOnFields() { ); for (BinaryComparison comparison : comparisons) { - assertEquals(comparison, s.rule(comparison)); + assertEquals(comparison, booleanFunctionEqualElimination(comparison)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplificationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplificationTests.java index 3b1f8cfc83af3..5b4bf806518de 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplificationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplificationTests.java @@ -9,9 +9,11 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; @@ -20,33 +22,31 @@ public class BooleanSimplificationTests extends ESTestCase { private static final Expression DUMMY_EXPRESSION = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 0); - public void testBoolSimplifyOr() { - BooleanSimplification simplification = new BooleanSimplification(); + private Expression booleanSimplification(ScalarFunction e) { + return new BooleanSimplification().rule(e, unboundLogicalOptimizerContext()); + } - assertEquals(TRUE, simplification.rule(new Or(EMPTY, TRUE, TRUE))); - assertEquals(TRUE, simplification.rule(new Or(EMPTY, TRUE, DUMMY_EXPRESSION))); - assertEquals(TRUE, simplification.rule(new Or(EMPTY, DUMMY_EXPRESSION, TRUE))); + public void testBoolSimplifyOr() { + assertEquals(TRUE, booleanSimplification(new Or(EMPTY, TRUE, TRUE))); + assertEquals(TRUE, booleanSimplification(new Or(EMPTY, TRUE, DUMMY_EXPRESSION))); + assertEquals(TRUE, booleanSimplification(new Or(EMPTY, DUMMY_EXPRESSION, TRUE))); - assertEquals(FALSE, simplification.rule(new Or(EMPTY, FALSE, FALSE))); - assertEquals(DUMMY_EXPRESSION, simplification.rule(new Or(EMPTY, FALSE, DUMMY_EXPRESSION))); - assertEquals(DUMMY_EXPRESSION, simplification.rule(new Or(EMPTY, DUMMY_EXPRESSION, FALSE))); + assertEquals(FALSE, booleanSimplification(new Or(EMPTY, FALSE, FALSE))); + assertEquals(DUMMY_EXPRESSION, booleanSimplification(new Or(EMPTY, FALSE, DUMMY_EXPRESSION))); + assertEquals(DUMMY_EXPRESSION, booleanSimplification(new Or(EMPTY, DUMMY_EXPRESSION, FALSE))); } public void testBoolSimplifyAnd() { - BooleanSimplification simplification = new BooleanSimplification(); + assertEquals(TRUE, booleanSimplification(new And(EMPTY, TRUE, TRUE))); + assertEquals(DUMMY_EXPRESSION, booleanSimplification(new And(EMPTY, TRUE, DUMMY_EXPRESSION))); + assertEquals(DUMMY_EXPRESSION, booleanSimplification(new And(EMPTY, DUMMY_EXPRESSION, TRUE))); - assertEquals(TRUE, simplification.rule(new And(EMPTY, TRUE, TRUE))); - assertEquals(DUMMY_EXPRESSION, simplification.rule(new And(EMPTY, TRUE, DUMMY_EXPRESSION))); - assertEquals(DUMMY_EXPRESSION, simplification.rule(new And(EMPTY, DUMMY_EXPRESSION, TRUE))); - - assertEquals(FALSE, simplification.rule(new And(EMPTY, FALSE, FALSE))); - assertEquals(FALSE, simplification.rule(new And(EMPTY, FALSE, DUMMY_EXPRESSION))); - assertEquals(FALSE, simplification.rule(new And(EMPTY, DUMMY_EXPRESSION, FALSE))); + assertEquals(FALSE, booleanSimplification(new And(EMPTY, FALSE, FALSE))); + assertEquals(FALSE, booleanSimplification(new And(EMPTY, FALSE, DUMMY_EXPRESSION))); + assertEquals(FALSE, booleanSimplification(new And(EMPTY, DUMMY_EXPRESSION, FALSE))); } public void testBoolCommonFactorExtraction() { - BooleanSimplification simplification = new BooleanSimplification(); - Expression a1 = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 1); Expression a2 = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 1); Expression b = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 2); @@ -55,7 +55,7 @@ public void testBoolCommonFactorExtraction() { Or actual = new Or(EMPTY, new And(EMPTY, a1, b), new And(EMPTY, a2, c)); And expected = new And(EMPTY, a1, new Or(EMPTY, b, c)); - assertEquals(expected, simplification.rule(actual)); + assertEquals(expected, booleanSimplification(actual)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisonsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisonsTests.java index d388369e0b167..a0d23731ae82d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisonsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisonsTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; @@ -37,6 +38,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOrEqualOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.notEqualsOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; @@ -45,19 +47,17 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; public class CombineBinaryComparisonsTests extends ESTestCase { - - private static final Expression DUMMY_EXPRESSION = - new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 0); + private Expression combine(BinaryLogic e) { + return new CombineBinaryComparisons().rule(e, unboundLogicalOptimizerContext()); + } public void testCombineBinaryComparisonsNotComparable() { FieldAttribute fa = getFieldAttribute(); LessThanOrEqual lte = lessThanOrEqualOf(fa, SIX); LessThan lt = lessThanOf(fa, FALSE); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - And and = new And(EMPTY, lte, lt); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(exp, and); } @@ -67,9 +67,7 @@ public void testCombineBinaryComparisonsUpper() { LessThanOrEqual lte = lessThanOrEqualOf(fa, SIX); LessThan lt = lessThanOf(fa, FIVE); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - - Expression exp = rule.rule(new And(EMPTY, lte, lt)); + Expression exp = combine(new And(EMPTY, lte, lt)); assertEquals(LessThan.class, exp.getClass()); LessThan r = (LessThan) exp; assertEquals(FIVE, r.right()); @@ -81,9 +79,7 @@ public void testCombineBinaryComparisonsLower() { GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, SIX); GreaterThan gt = greaterThanOf(fa, FIVE); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - - Expression exp = rule.rule(new And(EMPTY, gte, gt)); + Expression exp = combine(new And(EMPTY, gte, gt)); assertEquals(GreaterThanOrEqual.class, exp.getClass()); GreaterThanOrEqual r = (GreaterThanOrEqual) exp; assertEquals(SIX, r.right()); @@ -95,9 +91,7 @@ public void testCombineBinaryComparisonsInclude() { GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, FIVE); GreaterThan gt = greaterThanOf(fa, FIVE); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - - Expression exp = rule.rule(new And(EMPTY, gte, gt)); + Expression exp = combine(new And(EMPTY, gte, gt)); assertEquals(GreaterThan.class, exp.getClass()); GreaterThan r = (GreaterThan) exp; assertEquals(FIVE, r.right()); @@ -111,9 +105,7 @@ public void testCombineMultipleBinaryComparisons() { LessThanOrEqual lte = lessThanOrEqualOf(fa, L(7)); LessThan lt = lessThanOf(fa, SIX); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - - Expression exp = rule.rule(new And(EMPTY, gte, new And(EMPTY, gt, new And(EMPTY, lt, lte)))); + Expression exp = combine(new And(EMPTY, gte, new And(EMPTY, gt, new And(EMPTY, lt, lte)))); assertEquals(And.class, exp.getClass()); And and = (And) exp; assertEquals(gt, and.left()); @@ -128,10 +120,8 @@ public void testCombineMixedMultipleBinaryComparisons() { LessThanOrEqual lte = lessThanOrEqualOf(fa, L(7)); Expression ne = notEqualsOf(fa, FIVE); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - // TRUE AND a != 5 AND 4 < a <= 7 - Expression exp = rule.rule(new And(EMPTY, gte, new And(EMPTY, TRUE, new And(EMPTY, gt, new And(EMPTY, ne, lte))))); + Expression exp = combine(new And(EMPTY, gte, new And(EMPTY, TRUE, new And(EMPTY, gt, new And(EMPTY, ne, lte))))); assertEquals(And.class, exp.getClass()); And and = ((And) exp); assertEquals(And.class, and.right().getClass()); @@ -150,8 +140,7 @@ public void testCombineComparisonsIntoRange() { GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, ONE); LessThan lt = lessThanOf(fa, FIVE); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(new And(EMPTY, gte, lt)); + Expression exp = combine(new And(EMPTY, gte, lt)); assertEquals(And.class, exp.getClass()); And and = (And) exp; @@ -167,8 +156,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndGt3() { GreaterThan gt = greaterThanOf(fa, THREE); And and = new And(EMPTY, neq, gt); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(gt, exp); } @@ -180,8 +168,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndGte2() { GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, TWO); And and = new And(EMPTY, neq, gte); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(GreaterThan.class, exp.getClass()); GreaterThan gt = (GreaterThan) exp; assertEquals(TWO, gt.right()); @@ -195,8 +182,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndGte1() { GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, ONE); And and = new And(EMPTY, neq, gte); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(And.class, exp.getClass()); // can't optimize } @@ -208,8 +194,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndLte3() { LessThanOrEqual lte = lessThanOrEqualOf(fa, THREE); And and = new And(EMPTY, neq, lte); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(and, exp); // can't optimize } @@ -221,8 +206,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndLte2() { LessThanOrEqual lte = lessThanOrEqualOf(fa, TWO); And and = new And(EMPTY, neq, lte); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(LessThan.class, exp.getClass()); LessThan lt = (LessThan) exp; assertEquals(TWO, lt.right()); @@ -236,8 +220,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndLte1() { LessThanOrEqual lte = lessThanOrEqualOf(fa, ONE); And and = new And(EMPTY, neq, lte); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(lte, exp); } @@ -251,8 +234,7 @@ public void testCombineBinaryComparisonsDisjunctionNotComparable() { Or or = new Or(EMPTY, gt1, gt2); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(or); + Expression exp = combine(or); assertEquals(exp, or); } @@ -266,8 +248,7 @@ public void testCombineBinaryComparisonsDisjunctionLowerBound() { Or or = new Or(EMPTY, gt1, new Or(EMPTY, gt2, gt3)); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(or); + Expression exp = combine(or); assertEquals(GreaterThan.class, exp.getClass()); GreaterThan gt = (GreaterThan) exp; @@ -284,8 +265,7 @@ public void testCombineBinaryComparisonsDisjunctionIncludeLowerBounds() { Or or = new Or(EMPTY, new Or(EMPTY, gt1, gt2), gte3); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(or); + Expression exp = combine(or); assertEquals(GreaterThan.class, exp.getClass()); GreaterThan gt = (GreaterThan) exp; @@ -302,8 +282,7 @@ public void testCombineBinaryComparisonsDisjunctionUpperBound() { Or or = new Or(EMPTY, new Or(EMPTY, lt1, lt2), lt3); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(or); + Expression exp = combine(or); assertEquals(LessThan.class, exp.getClass()); LessThan lt = (LessThan) exp; @@ -320,8 +299,7 @@ public void testCombineBinaryComparisonsDisjunctionIncludeUpperBounds() { Or or = new Or(EMPTY, lt2, new Or(EMPTY, lte2, lt1)); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(or); + Expression exp = combine(or); assertEquals(LessThanOrEqual.class, exp.getClass()); LessThanOrEqual lte = (LessThanOrEqual) exp; @@ -340,8 +318,7 @@ public void testCombineBinaryComparisonsDisjunctionOfLowerAndUpperBounds() { Or or = new Or(EMPTY, new Or(EMPTY, lt2, gt3), new Or(EMPTY, lt1, gt4)); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(or); + Expression exp = combine(or); assertEquals(Or.class, exp.getClass()); Or ro = (Or) exp; @@ -367,7 +344,7 @@ public void testBooleanSimplificationCommonExpressionSubstraction() { And right = new And(EMPTY, a2, common); Or or = new Or(EMPTY, left, right); - Expression exp = new BooleanSimplification().rule(or); + Expression exp = new BooleanSimplification().rule(or, unboundLogicalOptimizerContext()); assertEquals(new And(EMPTY, common, new Or(EMPTY, a1, a2)), exp); } @@ -391,8 +368,7 @@ public void testBinaryComparisonAndOutOfRangeNotEqualsDifferentFields() { ); for (And and : testCases) { - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals("Rule should not have transformed [" + and.nodeString() + "]", and, exp); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctionsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctionsTests.java index 043d18dac9fd4..bb5f2fd3505e9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctionsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctionsTests.java @@ -38,16 +38,24 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral; import static org.elasticsearch.xpack.esql.EsqlTestUtils.relation; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.hamcrest.Matchers.contains; public class CombineDisjunctionsTests extends ESTestCase { + private Expression combineDisjunctions(Or e) { + return new CombineDisjunctions().rule(e, unboundLogicalOptimizerContext()); + } + + private LogicalPlan combineDisjunctions(LogicalPlan l) { + return new CombineDisjunctions().apply(l, unboundLogicalOptimizerContext()); + } public void testTwoEqualsWithOr() { FieldAttribute fa = getFieldAttribute(); Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO)); - Expression e = new CombineDisjunctions().rule(or); + Expression e = combineDisjunctions(or); assertEquals(In.class, e.getClass()); In in = (In) e; assertEquals(fa, in.value()); @@ -58,7 +66,7 @@ public void testTwoEqualsWithSameValue() { FieldAttribute fa = getFieldAttribute(); Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, ONE)); - Expression e = new CombineDisjunctions().rule(or); + Expression e = combineDisjunctions(or); assertEquals(Equals.class, e.getClass()); Equals eq = (Equals) e; assertEquals(fa, eq.left()); @@ -69,7 +77,7 @@ public void testOneEqualsOneIn() { FieldAttribute fa = getFieldAttribute(); Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, List.of(TWO))); - Expression e = new CombineDisjunctions().rule(or); + Expression e = combineDisjunctions(or); assertEquals(In.class, e.getClass()); In in = (In) e; assertEquals(fa, in.value()); @@ -80,7 +88,7 @@ public void testOneEqualsOneInWithSameValue() { FieldAttribute fa = getFieldAttribute(); Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, asList(ONE, TWO))); - Expression e = new CombineDisjunctions().rule(or); + Expression e = combineDisjunctions(or); assertEquals(In.class, e.getClass()); In in = (In) e; assertEquals(fa, in.value()); @@ -92,7 +100,7 @@ public void testSingleValueInToEquals() { Equals equals = equalsOf(fa, ONE); Or or = new Or(EMPTY, equals, new In(EMPTY, fa, List.of(ONE))); - Expression e = new CombineDisjunctions().rule(or); + Expression e = combineDisjunctions(or); assertEquals(equals, e); } @@ -101,7 +109,7 @@ public void testEqualsBehindAnd() { And and = new And(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO)); Filter dummy = new Filter(EMPTY, relation(), and); - LogicalPlan transformed = new CombineDisjunctions().apply(dummy); + LogicalPlan transformed = combineDisjunctions(dummy); assertSame(dummy, transformed); assertEquals(and, ((Filter) transformed).condition()); } @@ -111,7 +119,7 @@ public void testTwoEqualsDifferentFields() { FieldAttribute fieldTwo = getFieldAttribute("TWO"); Or or = new Or(EMPTY, equalsOf(fieldOne, ONE), equalsOf(fieldTwo, TWO)); - Expression e = new CombineDisjunctions().rule(or); + Expression e = combineDisjunctions(or); assertEquals(or, e); } @@ -120,7 +128,7 @@ public void testMultipleIn() { Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), new In(EMPTY, fa, List.of(TWO))); Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE))); - Expression e = new CombineDisjunctions().rule(secondOr); + Expression e = combineDisjunctions(secondOr); assertEquals(In.class, e.getClass()); In in = (In) e; assertEquals(fa, in.value()); @@ -132,7 +140,7 @@ public void testOrWithNonCombinableExpressions() { Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), lessThanOf(fa, TWO)); Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE))); - Expression e = new CombineDisjunctions().rule(secondOr); + Expression e = combineDisjunctions(secondOr); assertEquals(Or.class, e.getClass()); Or or = (Or) e; assertEquals(or.left(), firstOr.right()); @@ -160,7 +168,7 @@ public void testCombineCIDRMatch() { cidrs.add(new CIDRMatch(EMPTY, faa, ipa2)); cidrs.add(new CIDRMatch(EMPTY, fab, ipb2)); Or oldOr = (Or) Predicates.combineOr(cidrs); - Expression e = new CombineDisjunctions().rule(oldOr); + Expression e = combineDisjunctions(oldOr); assertEquals(Or.class, e.getClass()); Or newOr = (Or) e; assertEquals(CIDRMatch.class, newOr.left().getClass()); @@ -211,7 +219,7 @@ public void testCombineCIDRMatchEqualsIns() { Or oldOr = (Or) Predicates.combineOr(all); - Expression e = new CombineDisjunctions().rule(oldOr); + Expression e = combineDisjunctions(oldOr); assertEquals(Or.class, e.getClass()); Or newOr = (Or) e; assertEquals(Or.class, newOr.left().getClass()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java index 01af91271e1ba..8a8585b8d0ab5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator; @@ -45,68 +46,77 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.notEqualsOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.of; import static org.elasticsearch.xpack.esql.EsqlTestUtils.rangeOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; public class ConstantFoldingTests extends ESTestCase { + private Expression constantFolding(Expression e) { + return new ConstantFolding().rule(e, unboundLogicalOptimizerContext()); + } public void testConstantFolding() { Expression exp = new Add(EMPTY, TWO, THREE); assertTrue(exp.foldable()); - Expression result = new ConstantFolding().rule(exp); - assertTrue(result instanceof Literal); - assertEquals(5, ((Literal) result).value()); + Expression result = constantFolding(exp); + assertEquals(5, as(result, Literal.class).value()); // check now with an alias - result = new ConstantFolding().rule(new Alias(EMPTY, "a", exp)); + result = constantFolding(new Alias(EMPTY, "a", exp)); assertEquals("a", Expressions.name(result)); assertEquals(Alias.class, result.getClass()); } public void testConstantFoldingBinaryComparison() { - assertEquals(FALSE, new ConstantFolding().rule(greaterThanOf(TWO, THREE)).canonical()); - assertEquals(FALSE, new ConstantFolding().rule(greaterThanOrEqualOf(TWO, THREE)).canonical()); - assertEquals(FALSE, new ConstantFolding().rule(equalsOf(TWO, THREE)).canonical()); - assertEquals(TRUE, new ConstantFolding().rule(notEqualsOf(TWO, THREE)).canonical()); - assertEquals(TRUE, new ConstantFolding().rule(lessThanOrEqualOf(TWO, THREE)).canonical()); - assertEquals(TRUE, new ConstantFolding().rule(lessThanOf(TWO, THREE)).canonical()); + assertEquals(FALSE, constantFolding(greaterThanOf(TWO, THREE)).canonical()); + assertEquals(FALSE, constantFolding(greaterThanOrEqualOf(TWO, THREE)).canonical()); + assertEquals(FALSE, constantFolding(equalsOf(TWO, THREE)).canonical()); + assertEquals(TRUE, constantFolding(notEqualsOf(TWO, THREE)).canonical()); + assertEquals(TRUE, constantFolding(lessThanOrEqualOf(TWO, THREE)).canonical()); + assertEquals(TRUE, constantFolding(lessThanOf(TWO, THREE)).canonical()); } public void testConstantFoldingBinaryLogic() { - assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, greaterThanOf(TWO, THREE), TRUE)).canonical()); - assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, greaterThanOrEqualOf(TWO, THREE), TRUE)).canonical()); + assertEquals(FALSE, constantFolding(new And(EMPTY, greaterThanOf(TWO, THREE), TRUE)).canonical()); + assertEquals(TRUE, constantFolding(new Or(EMPTY, greaterThanOrEqualOf(TWO, THREE), TRUE)).canonical()); } public void testConstantFoldingBinaryLogic_WithNullHandling() { - assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, NULL, TRUE)).canonical().nullable()); - assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, TRUE, NULL)).canonical().nullable()); - assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, NULL, FALSE)).canonical()); - assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, FALSE, NULL)).canonical()); - assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, NULL, NULL)).canonical().nullable()); - - assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, TRUE)).canonical()); - assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, TRUE, NULL)).canonical()); - assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, FALSE)).canonical().nullable()); - assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, FALSE, NULL)).canonical().nullable()); - assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, NULL)).canonical().nullable()); + assertEquals(Nullability.TRUE, constantFolding(new And(EMPTY, NULL, TRUE)).canonical().nullable()); + assertEquals(Nullability.TRUE, constantFolding(new And(EMPTY, TRUE, NULL)).canonical().nullable()); + assertEquals(FALSE, constantFolding(new And(EMPTY, NULL, FALSE)).canonical()); + assertEquals(FALSE, constantFolding(new And(EMPTY, FALSE, NULL)).canonical()); + assertEquals(Nullability.TRUE, constantFolding(new And(EMPTY, NULL, NULL)).canonical().nullable()); + + assertEquals(TRUE, constantFolding(new Or(EMPTY, NULL, TRUE)).canonical()); + assertEquals(TRUE, constantFolding(new Or(EMPTY, TRUE, NULL)).canonical()); + assertEquals(Nullability.TRUE, constantFolding(new Or(EMPTY, NULL, FALSE)).canonical().nullable()); + assertEquals(Nullability.TRUE, constantFolding(new Or(EMPTY, FALSE, NULL)).canonical().nullable()); + assertEquals(Nullability.TRUE, constantFolding(new Or(EMPTY, NULL, NULL)).canonical().nullable()); } public void testConstantFoldingRange() { - assertEquals(true, new ConstantFolding().rule(rangeOf(FIVE, FIVE, true, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold()); - assertEquals(false, new ConstantFolding().rule(rangeOf(FIVE, FIVE, false, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold()); + assertEquals( + true, + constantFolding(rangeOf(FIVE, FIVE, true, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold(FoldContext.small()) + ); + assertEquals( + false, + constantFolding(rangeOf(FIVE, FIVE, false, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold(FoldContext.small()) + ); } public void testConstantNot() { - assertEquals(FALSE, new ConstantFolding().rule(new Not(EMPTY, TRUE))); - assertEquals(TRUE, new ConstantFolding().rule(new Not(EMPTY, FALSE))); + assertEquals(FALSE, constantFolding(new Not(EMPTY, TRUE))); + assertEquals(TRUE, constantFolding(new Not(EMPTY, FALSE))); } public void testConstantFoldingLikes() { - assertEquals(TRUE, new ConstantFolding().rule(new WildcardLike(EMPTY, of("test_emp"), new WildcardPattern("test*"))).canonical()); - assertEquals(TRUE, new ConstantFolding().rule(new RLike(EMPTY, of("test_emp"), new RLikePattern("test.emp"))).canonical()); + assertEquals(TRUE, constantFolding(new WildcardLike(EMPTY, of("test_emp"), new WildcardPattern("test*"))).canonical()); + assertEquals(TRUE, constantFolding(new RLike(EMPTY, of("test_emp"), new RLikePattern("test.emp"))).canonical()); } public void testArithmeticFolding() { @@ -125,7 +135,7 @@ public void testFoldRange() { Expression value = new Literal(EMPTY, 12, DataType.INTEGER); Range range = new Range(EMPTY, value, lowerBound, randomBoolean(), upperBound, randomBoolean(), randomZone()); - Expression folded = new ConstantFolding().rule(range); + Expression folded = constantFolding(range); assertTrue((Boolean) as(folded, Literal.class).value()); } @@ -156,16 +166,15 @@ public void testFoldRangeWithInvalidBoundaries() { // Just applying this to the range directly won't perform a transformDown. LogicalPlan filter = new Filter(EMPTY, emptySource(), range); - Filter foldedOnce = as(new ConstantFolding().apply(filter), Filter.class); + Filter foldedOnce = as(new ConstantFolding().apply(filter, unboundLogicalOptimizerContext()), Filter.class); // We need to run the rule twice, because during the first run only the boundaries can be folded - the range doesn't know it's // foldable, yet. - Filter foldedTwice = as(new ConstantFolding().apply(foldedOnce), Filter.class); + Filter foldedTwice = as(new ConstantFolding().apply(foldedOnce, unboundLogicalOptimizerContext()), Filter.class); assertFalse((Boolean) as(foldedTwice.condition(), Literal.class).value()); } - private static Object foldOperator(BinaryOperator b) { - return ((Literal) new ConstantFolding().rule(b)).value(); + private Object foldOperator(BinaryOperator b) { + return ((Literal) constantFolding(b)).value(); } - } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java index ae31576184938..252b25a214bb8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; @@ -67,9 +68,11 @@ import java.util.List; import static org.elasticsearch.xpack.esql.EsqlTestUtils.L; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute; import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; @@ -85,28 +88,27 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; public class FoldNullTests extends ESTestCase { + private Expression foldNull(Expression e) { + return new FoldNull().rule(e, unboundLogicalOptimizerContext()); + } public void testBasicNullFolding() { - FoldNull rule = new FoldNull(); - assertNullLiteral(rule.rule(new Add(EMPTY, L(randomInt()), Literal.NULL))); - assertNullLiteral(rule.rule(new Round(EMPTY, Literal.NULL, null))); - assertNullLiteral(rule.rule(new Pow(EMPTY, Literal.NULL, Literal.NULL))); - assertNullLiteral(rule.rule(new DateFormat(EMPTY, Literal.NULL, Literal.NULL, null))); - assertNullLiteral(rule.rule(new DateParse(EMPTY, Literal.NULL, Literal.NULL))); - assertNullLiteral(rule.rule(new DateTrunc(EMPTY, Literal.NULL, Literal.NULL))); - assertNullLiteral(rule.rule(new Substring(EMPTY, Literal.NULL, Literal.NULL, Literal.NULL))); + assertNullLiteral(foldNull(new Add(EMPTY, L(randomInt()), Literal.NULL))); + assertNullLiteral(foldNull(new Round(EMPTY, Literal.NULL, null))); + assertNullLiteral(foldNull(new Pow(EMPTY, Literal.NULL, Literal.NULL))); + assertNullLiteral(foldNull(new DateFormat(EMPTY, Literal.NULL, Literal.NULL, null))); + assertNullLiteral(foldNull(new DateParse(EMPTY, Literal.NULL, Literal.NULL))); + assertNullLiteral(foldNull(new DateTrunc(EMPTY, Literal.NULL, Literal.NULL))); + assertNullLiteral(foldNull(new Substring(EMPTY, Literal.NULL, Literal.NULL, Literal.NULL))); } public void testNullFoldingIsNotNull() { - FoldNull foldNull = new FoldNull(); - assertEquals(true, foldNull.rule(new IsNotNull(EMPTY, TRUE)).fold()); - assertEquals(false, foldNull.rule(new IsNotNull(EMPTY, NULL)).fold()); + assertEquals(true, foldNull(new IsNotNull(EMPTY, TRUE)).fold(FoldContext.small())); + assertEquals(false, foldNull(new IsNotNull(EMPTY, NULL)).fold(FoldContext.small())); } @SuppressWarnings("unchecked") public void testNullFoldingDoesNotApplyOnAbstractMultivalueFunction() throws Exception { - FoldNull rule = new FoldNull(); - List> items = List.of( MvDedupe.class, MvFirst.class, @@ -119,119 +121,112 @@ public void testNullFoldingDoesNotApplyOnAbstractMultivalueFunction() throws Exc for (Class clazz : items) { Constructor ctor = clazz.getConstructor(Source.class, Expression.class); AbstractMultivalueFunction conditionalFunction = ctor.newInstance(EMPTY, getFieldAttribute("a")); - assertEquals(conditionalFunction, rule.rule(conditionalFunction)); + assertEquals(conditionalFunction, foldNull(conditionalFunction)); conditionalFunction = ctor.newInstance(EMPTY, NULL); - assertEquals(NULL, rule.rule(conditionalFunction)); + assertEquals(NULL, foldNull(conditionalFunction)); } // avg and count ar different just because they know the return type in advance (all the others infer the type from the input) MvAvg avg = new MvAvg(EMPTY, getFieldAttribute("a")); - assertEquals(avg, rule.rule(avg)); + assertEquals(avg, foldNull(avg)); avg = new MvAvg(EMPTY, NULL); - assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(avg)); + assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(avg)); MvCount count = new MvCount(EMPTY, getFieldAttribute("a")); - assertEquals(count, rule.rule(count)); + assertEquals(count, foldNull(count)); count = new MvCount(EMPTY, NULL); - assertEquals(new Literal(EMPTY, null, INTEGER), rule.rule(count)); + assertEquals(new Literal(EMPTY, null, INTEGER), foldNull(count)); } public void testNullFoldingIsNull() { - FoldNull foldNull = new FoldNull(); - assertEquals(true, foldNull.rule(new IsNull(EMPTY, NULL)).fold()); - assertEquals(false, foldNull.rule(new IsNull(EMPTY, TRUE)).fold()); + assertEquals(true, foldNull(new IsNull(EMPTY, NULL)).fold(FoldContext.small())); + assertEquals(false, foldNull(new IsNull(EMPTY, TRUE)).fold(FoldContext.small())); } public void testGenericNullableExpression() { FoldNull rule = new FoldNull(); // arithmetic - assertNullLiteral(rule.rule(new Add(EMPTY, getFieldAttribute("a"), NULL))); + assertNullLiteral(foldNull(new Add(EMPTY, getFieldAttribute("a"), NULL))); // comparison - assertNullLiteral(rule.rule(greaterThanOf(getFieldAttribute("a"), NULL))); + assertNullLiteral(foldNull(greaterThanOf(getFieldAttribute("a"), NULL))); // regex - assertNullLiteral(rule.rule(new RLike(EMPTY, NULL, new RLikePattern("123")))); + assertNullLiteral(foldNull(new RLike(EMPTY, NULL, new RLikePattern("123")))); // date functions - assertNullLiteral(rule.rule(new DateExtract(EMPTY, NULL, NULL, configuration("")))); + assertNullLiteral(foldNull(new DateExtract(EMPTY, NULL, NULL, configuration("")))); // math functions - assertNullLiteral(rule.rule(new Cos(EMPTY, NULL))); + assertNullLiteral(foldNull(new Cos(EMPTY, NULL))); // string functions - assertNullLiteral(rule.rule(new LTrim(EMPTY, NULL))); + assertNullLiteral(foldNull(new LTrim(EMPTY, NULL))); // spatial - assertNullLiteral(rule.rule(new SpatialCentroid(EMPTY, NULL))); + assertNullLiteral(foldNull(new SpatialCentroid(EMPTY, NULL))); // ip - assertNullLiteral(rule.rule(new CIDRMatch(EMPTY, NULL, List.of(NULL)))); + assertNullLiteral(foldNull(new CIDRMatch(EMPTY, NULL, List.of(NULL)))); // conversion - assertNullLiteral(rule.rule(new ToString(EMPTY, NULL))); + assertNullLiteral(foldNull(new ToString(EMPTY, NULL))); } public void testNullFoldingDoesNotApplyOnLogicalExpressions() { - FoldNull rule = new FoldNull(); - Or or = new Or(EMPTY, NULL, TRUE); - assertEquals(or, rule.rule(or)); + assertEquals(or, foldNull(or)); or = new Or(EMPTY, NULL, NULL); - assertEquals(or, rule.rule(or)); + assertEquals(or, foldNull(or)); And and = new And(EMPTY, NULL, TRUE); - assertEquals(and, rule.rule(and)); + assertEquals(and, foldNull(and)); and = new And(EMPTY, NULL, NULL); - assertEquals(and, rule.rule(and)); + assertEquals(and, foldNull(and)); } @SuppressWarnings("unchecked") public void testNullFoldingDoesNotApplyOnAggregate() throws Exception { - FoldNull rule = new FoldNull(); - List> items = List.of(Max.class, Min.class); for (Class clazz : items) { Constructor ctor = clazz.getConstructor(Source.class, Expression.class); AggregateFunction conditionalFunction = ctor.newInstance(EMPTY, getFieldAttribute("a")); - assertEquals(conditionalFunction, rule.rule(conditionalFunction)); + assertEquals(conditionalFunction, foldNull(conditionalFunction)); conditionalFunction = ctor.newInstance(EMPTY, NULL); - assertEquals(NULL, rule.rule(conditionalFunction)); + assertEquals(NULL, foldNull(conditionalFunction)); } Avg avg = new Avg(EMPTY, getFieldAttribute("a")); - assertEquals(avg, rule.rule(avg)); + assertEquals(avg, foldNull(avg)); avg = new Avg(EMPTY, NULL); - assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(avg)); + assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(avg)); Count count = new Count(EMPTY, getFieldAttribute("a")); - assertEquals(count, rule.rule(count)); + assertEquals(count, foldNull(count)); count = new Count(EMPTY, NULL); - assertEquals(count, rule.rule(count)); + assertEquals(count, foldNull(count)); CountDistinct countd = new CountDistinct(EMPTY, getFieldAttribute("a"), getFieldAttribute("a")); - assertEquals(countd, rule.rule(countd)); + assertEquals(countd, foldNull(countd)); countd = new CountDistinct(EMPTY, NULL, NULL); - assertEquals(new Literal(EMPTY, null, LONG), rule.rule(countd)); + assertEquals(new Literal(EMPTY, null, LONG), foldNull(countd)); Median median = new Median(EMPTY, getFieldAttribute("a")); - assertEquals(median, rule.rule(median)); + assertEquals(median, foldNull(median)); median = new Median(EMPTY, NULL); - assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(median)); + assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(median)); MedianAbsoluteDeviation medianad = new MedianAbsoluteDeviation(EMPTY, getFieldAttribute("a")); - assertEquals(medianad, rule.rule(medianad)); + assertEquals(medianad, foldNull(medianad)); medianad = new MedianAbsoluteDeviation(EMPTY, NULL); - assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(medianad)); + assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(medianad)); Percentile percentile = new Percentile(EMPTY, getFieldAttribute("a"), getFieldAttribute("a")); - assertEquals(percentile, rule.rule(percentile)); + assertEquals(percentile, foldNull(percentile)); percentile = new Percentile(EMPTY, NULL, NULL); - assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(percentile)); + assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(percentile)); Sum sum = new Sum(EMPTY, getFieldAttribute("a")); - assertEquals(sum, rule.rule(sum)); + assertEquals(sum, foldNull(sum)); sum = new Sum(EMPTY, NULL); - assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(sum)); + assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(sum)); } public void testNullFoldableDoesNotApplyToIsNullAndNotNull() { - FoldNull rule = new FoldNull(); - DataType numericType = randomFrom(INTEGER, LONG, DOUBLE); DataType genericType = randomFrom(INTEGER, LONG, DOUBLE, UNSIGNED_LONG, KEYWORD, TEXT, GEO_POINT, GEO_SHAPE, VERSION, IP); List items = List.of( @@ -260,29 +255,26 @@ public void testNullFoldableDoesNotApplyToIsNullAndNotNull() { ); for (Expression item : items) { Expression isNull = new IsNull(EMPTY, item); - Expression transformed = rule.rule(isNull); + Expression transformed = foldNull(isNull); assertEquals(isNull, transformed); IsNotNull isNotNull = new IsNotNull(EMPTY, item); - transformed = rule.rule(isNotNull); + transformed = foldNull(isNotNull); assertEquals(isNotNull, transformed); } } public void testNullBucketGetsFolded() { - FoldNull foldNull = new FoldNull(); - assertEquals(NULL, foldNull.rule(new Bucket(EMPTY, NULL, NULL, NULL, NULL))); + assertEquals(NULL, foldNull(new Bucket(EMPTY, NULL, NULL, NULL, NULL))); } public void testNullCategorizeGroupingNotFolded() { - FoldNull foldNull = new FoldNull(); Categorize categorize = new Categorize(EMPTY, NULL); - assertEquals(categorize, foldNull.rule(categorize)); + assertEquals(categorize, foldNull(categorize)); } private void assertNullLiteral(Expression expression) { - assertEquals(Literal.class, expression.getClass()); - assertNull(expression.fold()); + assertNull(as(expression, Literal.class).value()); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRightTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRightTests.java index 17e69e81444c5..1664e9f4653bb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRightTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRightTests.java @@ -15,6 +15,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.FIVE; import static org.elasticsearch.xpack.esql.EsqlTestUtils.equalsOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; @@ -22,7 +23,7 @@ public class LiteralsOnTheRightTests extends ESTestCase { public void testLiteralsOnTheRight() { Alias a = new Alias(EMPTY, "a", new Literal(EMPTY, 10, INTEGER)); - Expression result = new LiteralsOnTheRight().rule(equalsOf(FIVE, a)); + Expression result = new LiteralsOnTheRight().rule(equalsOf(FIVE, a), unboundLogicalOptimizerContext()); assertTrue(result instanceof Equals); Equals eq = (Equals) result; assertEquals(a, eq.left()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEqualsTests.java index 55091653e75d4..a6c0d838b2c21 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEqualsTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; import org.elasticsearch.xpack.esql.core.expression.predicate.Range; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; @@ -37,11 +38,15 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOrEqualOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.notEqualsOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.rangeOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; public class PropagateEqualsTests extends ESTestCase { + private Expression propagateEquals(BinaryLogic e) { + return new PropagateEquals().rule(e, unboundLogicalOptimizerContext()); + } // a == 1 AND a == 2 -> FALSE public void testDualEqualsConjunction() { @@ -49,8 +54,7 @@ public void testDualEqualsConjunction() { Equals eq1 = equalsOf(fa, ONE); Equals eq2 = equalsOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq1, eq2)); + Expression exp = propagateEquals(new And(EMPTY, eq1, eq2)); assertEquals(FALSE, exp); } @@ -60,8 +64,7 @@ public void testEliminateRangeByEqualsOutsideInterval() { Equals eq1 = equalsOf(fa, new Literal(EMPTY, 10, DataType.INTEGER)); Range r = rangeOf(fa, ONE, false, new Literal(EMPTY, 10, DataType.INTEGER), false); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq1, r)); + Expression exp = propagateEquals(new And(EMPTY, eq1, r)); assertEquals(FALSE, exp); } @@ -71,8 +74,7 @@ public void testPropagateEquals_VarNeq3AndVarEq3() { NotEquals neq = notEqualsOf(fa, THREE); Equals eq = equalsOf(fa, THREE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, neq, eq)); + Expression exp = propagateEquals(new And(EMPTY, neq, eq)); assertEquals(FALSE, exp); } @@ -82,8 +84,7 @@ public void testPropagateEquals_VarNeq4AndVarEq3() { NotEquals neq = notEqualsOf(fa, FOUR); Equals eq = equalsOf(fa, THREE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, neq, eq)); + Expression exp = propagateEquals(new And(EMPTY, neq, eq)); assertEquals(Equals.class, exp.getClass()); assertEquals(eq, exp); } @@ -94,8 +95,7 @@ public void testPropagateEquals_VarEq2AndVarLt2() { Equals eq = equalsOf(fa, TWO); LessThan lt = lessThanOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq, lt)); + Expression exp = propagateEquals(new And(EMPTY, eq, lt)); assertEquals(FALSE, exp); } @@ -105,8 +105,7 @@ public void testPropagateEquals_VarEq2AndVarLte2() { Equals eq = equalsOf(fa, TWO); LessThanOrEqual lt = lessThanOrEqualOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq, lt)); + Expression exp = propagateEquals(new And(EMPTY, eq, lt)); assertEquals(eq, exp); } @@ -116,8 +115,7 @@ public void testPropagateEquals_VarEq2AndVarLte1() { Equals eq = equalsOf(fa, TWO); LessThanOrEqual lt = lessThanOrEqualOf(fa, ONE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq, lt)); + Expression exp = propagateEquals(new And(EMPTY, eq, lt)); assertEquals(FALSE, exp); } @@ -127,8 +125,7 @@ public void testPropagateEquals_VarEq2AndVarGt2() { Equals eq = equalsOf(fa, TWO); GreaterThan gt = greaterThanOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq, gt)); + Expression exp = propagateEquals(new And(EMPTY, eq, gt)); assertEquals(FALSE, exp); } @@ -138,8 +135,7 @@ public void testPropagateEquals_VarEq2AndVarGte2() { Equals eq = equalsOf(fa, TWO); GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq, gte)); + Expression exp = propagateEquals(new And(EMPTY, eq, gte)); assertEquals(eq, exp); } @@ -149,8 +145,7 @@ public void testPropagateEquals_VarEq2AndVarLt3() { Equals eq = equalsOf(fa, TWO); GreaterThan gt = greaterThanOf(fa, THREE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq, gt)); + Expression exp = propagateEquals(new And(EMPTY, eq, gt)); assertEquals(FALSE, exp); } @@ -162,9 +157,8 @@ public void testPropagateEquals_VarEq2AndVarLt3AndVarGt1AndVarNeq4() { GreaterThan gt = greaterThanOf(fa, ONE); NotEquals neq = notEqualsOf(fa, FOUR); - PropagateEquals rule = new PropagateEquals(); Expression and = Predicates.combineAnd(asList(eq, lt, gt, neq)); - Expression exp = rule.rule((And) and); + Expression exp = propagateEquals((And) and); assertEquals(eq, exp); } @@ -176,9 +170,8 @@ public void testPropagateEquals_VarEq2AndVarRangeGt1Lt3AndVarGt0AndVarNeq4() { GreaterThan gt = greaterThanOf(fa, new Literal(EMPTY, 0, DataType.INTEGER)); NotEquals neq = notEqualsOf(fa, FOUR); - PropagateEquals rule = new PropagateEquals(); Expression and = Predicates.combineAnd(asList(eq, range, gt, neq)); - Expression exp = rule.rule((And) and); + Expression exp = propagateEquals((And) and); assertEquals(eq, exp); } @@ -188,8 +181,7 @@ public void testPropagateEquals_VarEq2OrVarGt1() { Equals eq = equalsOf(fa, TWO); GreaterThan gt = greaterThanOf(fa, ONE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, gt)); + Expression exp = propagateEquals(new Or(EMPTY, eq, gt)); assertEquals(gt, exp); } @@ -199,8 +191,7 @@ public void testPropagateEquals_VarEq2OrVarGte2() { Equals eq = equalsOf(fa, TWO); GreaterThan gt = greaterThanOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, gt)); + Expression exp = propagateEquals(new Or(EMPTY, eq, gt)); assertEquals(GreaterThanOrEqual.class, exp.getClass()); GreaterThanOrEqual gte = (GreaterThanOrEqual) exp; assertEquals(TWO, gte.right()); @@ -212,8 +203,7 @@ public void testPropagateEquals_VarEq2OrVarLt3() { Equals eq = equalsOf(fa, TWO); LessThan lt = lessThanOf(fa, THREE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, lt)); + Expression exp = propagateEquals(new Or(EMPTY, eq, lt)); assertEquals(lt, exp); } @@ -223,8 +213,7 @@ public void testPropagateEquals_VarEq3OrVarLt3() { Equals eq = equalsOf(fa, THREE); LessThan lt = lessThanOf(fa, THREE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, lt)); + Expression exp = propagateEquals(new Or(EMPTY, eq, lt)); assertEquals(LessThanOrEqual.class, exp.getClass()); LessThanOrEqual lte = (LessThanOrEqual) exp; assertEquals(THREE, lte.right()); @@ -236,8 +225,7 @@ public void testPropagateEquals_VarEq2OrVarRangeGt1Lt3() { Equals eq = equalsOf(fa, TWO); Range range = rangeOf(fa, ONE, false, THREE, false); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, range)); + Expression exp = propagateEquals(new Or(EMPTY, eq, range)); assertEquals(range, exp); } @@ -247,8 +235,7 @@ public void testPropagateEquals_VarEq2OrVarRangeGt2Lt3() { Equals eq = equalsOf(fa, TWO); Range range = rangeOf(fa, TWO, false, THREE, false); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, range)); + Expression exp = propagateEquals(new Or(EMPTY, eq, range)); assertEquals(Range.class, exp.getClass()); Range r = (Range) exp; assertEquals(TWO, r.lower()); @@ -263,8 +250,7 @@ public void testPropagateEquals_VarEq3OrVarRangeGt2Lt3() { Equals eq = equalsOf(fa, THREE); Range range = rangeOf(fa, TWO, false, THREE, false); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, range)); + Expression exp = propagateEquals(new Or(EMPTY, eq, range)); assertEquals(Range.class, exp.getClass()); Range r = (Range) exp; assertEquals(TWO, r.lower()); @@ -279,8 +265,7 @@ public void testPropagateEquals_VarEq2OrVarNeq2() { Equals eq = equalsOf(fa, TWO); NotEquals neq = notEqualsOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, neq)); + Expression exp = propagateEquals(new Or(EMPTY, eq, neq)); assertEquals(TRUE, exp); } @@ -290,8 +275,7 @@ public void testPropagateEquals_VarEq2OrVarNeq5() { Equals eq = equalsOf(fa, TWO); NotEquals neq = notEqualsOf(fa, FIVE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, neq)); + Expression exp = propagateEquals(new Or(EMPTY, eq, neq)); assertEquals(NotEquals.class, exp.getClass()); NotEquals ne = (NotEquals) exp; assertEquals(FIVE, ne.right()); @@ -305,8 +289,7 @@ public void testPropagateEquals_VarEq2OrVarRangeGt3Lt4OrVarGt2OrVarNe2() { GreaterThan gt = greaterThanOf(fa, TWO); NotEquals neq = notEqualsOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule((Or) Predicates.combineOr(asList(eq, range, neq, gt))); + Expression exp = propagateEquals((Or) Predicates.combineOr(asList(eq, range, neq, gt))); assertEquals(TRUE, exp); } @@ -317,8 +300,7 @@ public void testPropagateEquals_ignoreDateTimeFields() { Equals eq2 = equalsOf(fa, TWO); And and = new And(EMPTY, eq1, eq2); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(and); + Expression exp = propagateEquals(and); assertEquals(and, exp); } @@ -328,8 +310,7 @@ public void testEliminateRangeByEqualsInInterval() { Equals eq1 = equalsOf(fa, ONE); Range r = rangeOf(fa, ONE, true, new Literal(EMPTY, 10, DataType.INTEGER), false); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq1, r)); + Expression exp = propagateEquals(new And(EMPTY, eq1, r)); assertEquals(eq1, exp); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullableTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullableTests.java index d1d6a7fbaa208..d35890e5b56bb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullableTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullableTests.java @@ -31,11 +31,20 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.relation; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; public class PropagateNullableTests extends ESTestCase { + private Expression propagateNullable(And e) { + return new PropagateNullable().rule(e, unboundLogicalOptimizerContext()); + } + + private LogicalPlan propagateNullable(LogicalPlan p) { + return new PropagateNullable().apply(p, unboundLogicalOptimizerContext()); + } + private Literal nullOf(DataType dataType) { return new Literal(Source.EMPTY, null, dataType); } @@ -45,7 +54,7 @@ public void testIsNullAndNotNull() { FieldAttribute fa = getFieldAttribute(); And and = new And(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, fa)); - assertEquals(FALSE, new PropagateNullable().rule(and)); + assertEquals(FALSE, propagateNullable(and)); } // a IS NULL AND b IS NOT NULL AND c IS NULL AND d IS NOT NULL AND e IS NULL AND a IS NOT NULL => false @@ -58,7 +67,7 @@ public void testIsNullAndNotNullMultiField() { And and = new And(EMPTY, andOne, new And(EMPTY, andThree, andTwo)); - assertEquals(FALSE, new PropagateNullable().rule(and)); + assertEquals(FALSE, propagateNullable(and)); } // a IS NULL AND a > 1 => a IS NULL AND false @@ -67,7 +76,7 @@ public void testIsNullAndComparison() { IsNull isNull = new IsNull(EMPTY, fa); And and = new And(EMPTY, isNull, greaterThanOf(fa, ONE)); - assertEquals(new And(EMPTY, isNull, nullOf(BOOLEAN)), new PropagateNullable().rule(and)); + assertEquals(new And(EMPTY, isNull, nullOf(BOOLEAN)), propagateNullable(and)); } // a IS NULL AND b < 1 AND c < 1 AND a < 1 => a IS NULL AND b < 1 AND c < 1 => a IS NULL AND b < 1 AND c < 1 @@ -79,7 +88,7 @@ public void testIsNullAndMultipleComparison() { And and = new And(EMPTY, isNull, nestedAnd); And top = new And(EMPTY, and, lessThanOf(fa, ONE)); - Expression optimized = new PropagateNullable().rule(top); + Expression optimized = propagateNullable(top); Expression expected = new And(EMPTY, and, nullOf(BOOLEAN)); assertEquals(Predicates.splitAnd(expected), Predicates.splitAnd(optimized)); } @@ -97,7 +106,7 @@ public void testIsNullAndDeeplyNestedExpression() { Expression kept = new And(EMPTY, isNull, lessThanOf(getFieldAttribute("b"), THREE)); And and = new And(EMPTY, nullified, kept); - Expression optimized = new PropagateNullable().rule(and); + Expression optimized = propagateNullable(and); Expression expected = new And(EMPTY, new And(EMPTY, nullOf(BOOLEAN), nullOf(BOOLEAN)), kept); assertEquals(Predicates.splitAnd(expected), Predicates.splitAnd(optimized)); @@ -110,13 +119,13 @@ public void testIsNullInDisjunction() { Or or = new Or(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, fa)); Filter dummy = new Filter(EMPTY, relation(), or); - LogicalPlan transformed = new PropagateNullable().apply(dummy); + LogicalPlan transformed = propagateNullable(dummy); assertSame(dummy, transformed); assertEquals(or, ((Filter) transformed).condition()); or = new Or(EMPTY, new IsNull(EMPTY, fa), greaterThanOf(fa, ONE)); dummy = new Filter(EMPTY, relation(), or); - transformed = new PropagateNullable().apply(dummy); + transformed = propagateNullable(dummy); assertSame(dummy, transformed); assertEquals(or, ((Filter) transformed).condition()); } @@ -129,7 +138,7 @@ public void testIsNullDisjunction() { Or or = new Or(EMPTY, isNull, greaterThanOf(fa, THREE)); And and = new And(EMPTY, new Add(EMPTY, fa, ONE), or); - assertEquals(and, new PropagateNullable().rule(and)); + assertEquals(and, propagateNullable(and)); } public void testDoNotOptimizeIsNullAndMultipleComparisonWithConstants() { @@ -141,7 +150,7 @@ public void testDoNotOptimizeIsNullAndMultipleComparisonWithConstants() { And aIsNull_AND_bLT1_AND_cLT1 = new And(EMPTY, aIsNull, bLT1_AND_cLT1); And aIsNull_AND_bLT1_AND_cLT1_AND_aLT1 = new And(EMPTY, aIsNull_AND_bLT1_AND_cLT1, lessThanOf(a, ONE)); - Expression optimized = new PropagateNullable().rule(aIsNull_AND_bLT1_AND_cLT1_AND_aLT1); + Expression optimized = propagateNullable(aIsNull_AND_bLT1_AND_cLT1_AND_aLT1); Literal nullLiteral = new Literal(EMPTY, null, BOOLEAN); assertEquals(asList(aIsNull, nullLiteral, nullLiteral, nullLiteral), Predicates.splitAnd(optimized)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java index c7206c6971bde..b9ffc39e5e130 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java @@ -10,8 +10,10 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; +import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; @@ -20,16 +22,20 @@ import static java.util.Arrays.asList; import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; public class ReplaceRegexMatchTests extends ESTestCase { + private Expression replaceRegexMatch(RegexMatch e) { + return new ReplaceRegexMatch().rule(e, unboundLogicalOptimizerContext()); + } public void testMatchAllWildcardLikeToExist() { for (String s : asList("*", "**", "***")) { WildcardPattern pattern = new WildcardPattern(s); FieldAttribute fa = getFieldAttribute(); WildcardLike l = new WildcardLike(EMPTY, fa, pattern); - Expression e = new ReplaceRegexMatch().rule(l); + Expression e = replaceRegexMatch(l); assertEquals(IsNotNull.class, e.getClass()); IsNotNull inn = (IsNotNull) e; assertEquals(fa, inn.field()); @@ -40,7 +46,7 @@ public void testMatchAllRLikeToExist() { RLikePattern pattern = new RLikePattern(".*"); FieldAttribute fa = getFieldAttribute(); RLike l = new RLike(EMPTY, fa, pattern); - Expression e = new ReplaceRegexMatch().rule(l); + Expression e = replaceRegexMatch(l); assertEquals(IsNotNull.class, e.getClass()); IsNotNull inn = (IsNotNull) e; assertEquals(fa, inn.field()); @@ -51,11 +57,11 @@ public void testExactMatchWildcardLike() { WildcardPattern pattern = new WildcardPattern(s); FieldAttribute fa = getFieldAttribute(); WildcardLike l = new WildcardLike(EMPTY, fa, pattern); - Expression e = new ReplaceRegexMatch().rule(l); + Expression e = replaceRegexMatch(l); assertEquals(Equals.class, e.getClass()); Equals eq = (Equals) e; assertEquals(fa, eq.left()); - assertEquals(s.replace("\\", StringUtils.EMPTY), eq.right().fold()); + assertEquals(s.replace("\\", StringUtils.EMPTY), eq.right().fold(FoldContext.small())); } } @@ -63,11 +69,11 @@ public void testExactMatchRLike() { RLikePattern pattern = new RLikePattern("abc"); FieldAttribute fa = getFieldAttribute(); RLike l = new RLike(EMPTY, fa, pattern); - Expression e = new ReplaceRegexMatch().rule(l); + Expression e = replaceRegexMatch(l); assertEquals(Equals.class, e.getClass()); Equals eq = (Equals) e; assertEquals(fa, eq.left()); - assertEquals("abc", eq.right().fold()); + assertEquals("abc", eq.right().fold(FoldContext.small())); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java index 2429bcb1a1b04..90c8ae1032325 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.Nullability; @@ -417,7 +418,7 @@ private static void assertNoPushdownSort(TestPhysicalPlanBuilder builder, String private static PhysicalPlan pushTopNToSource(TopNExec topNExec) { var configuration = EsqlTestUtils.configuration("from test"); - var ctx = new LocalPhysicalOptimizerContext(configuration, SearchStats.EMPTY); + var ctx = new LocalPhysicalOptimizerContext(configuration, FoldContext.small(), SearchStats.EMPTY); var pushTopNToSource = new PushTopNToSource(); return pushTopNToSource.rule(topNExec, ctx); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java index 710637c05a900..85d4017b166fa 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar; @@ -617,16 +618,14 @@ public void testSimplifyInWithSingleElementList() { Equals eq = (Equals) e; assertThat(eq.left(), instanceOf(UnresolvedAttribute.class)); assertThat(((UnresolvedAttribute) eq.left()).name(), equalTo("a")); - assertThat(eq.right(), instanceOf(Literal.class)); - assertThat(eq.right().fold(), equalTo(1)); + assertThat(as(eq.right(), Literal.class).value(), equalTo(1)); e = whereExpression("1 IN (a)"); assertThat(e, instanceOf(Equals.class)); eq = (Equals) e; assertThat(eq.right(), instanceOf(UnresolvedAttribute.class)); assertThat(((UnresolvedAttribute) eq.right()).name(), equalTo("a")); - assertThat(eq.left(), instanceOf(Literal.class)); - assertThat(eq.left().fold(), equalTo(1)); + assertThat(eq.left().fold(FoldContext.small()), equalTo(1)); e = whereExpression("1 NOT IN (a)"); assertThat(e, instanceOf(Not.class)); @@ -635,9 +634,7 @@ public void testSimplifyInWithSingleElementList() { eq = (Equals) e; assertThat(eq.right(), instanceOf(UnresolvedAttribute.class)); assertThat(((UnresolvedAttribute) eq.right()).name(), equalTo("a")); - assertThat(eq.left(), instanceOf(Literal.class)); - assertThat(eq.left().fold(), equalTo(1)); - + assertThat(eq.left().fold(FoldContext.small()), equalTo(1)); } private Expression whereExpression(String e) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index b83892ea47049..49f03e9b8bc2f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; @@ -1129,51 +1130,51 @@ public void testInputParams() { assertThat(field.name(), is("x")); assertThat(field, instanceOf(Alias.class)); Alias alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); field = row.fields().get(1); assertThat(field.name(), is("y")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is("2")); + assertThat(alias.child().fold(FoldContext.small()), is("2")); field = row.fields().get(2); assertThat(field.name(), is("a")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is("2 days")); + assertThat(alias.child().fold(FoldContext.small()), is("2 days")); field = row.fields().get(3); assertThat(field.name(), is("b")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is("4 hours")); + assertThat(alias.child().fold(FoldContext.small()), is("4 hours")); field = row.fields().get(4); assertThat(field.name(), is("c")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold().getClass(), is(String.class)); - assertThat(alias.child().fold().toString(), is("1.2.3")); + assertThat(alias.child().fold(FoldContext.small()).getClass(), is(String.class)); + assertThat(alias.child().fold(FoldContext.small()).toString(), is("1.2.3")); field = row.fields().get(5); assertThat(field.name(), is("d")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold().getClass(), is(String.class)); - assertThat(alias.child().fold().toString(), is("127.0.0.1")); + assertThat(alias.child().fold(FoldContext.small()).getClass(), is(String.class)); + assertThat(alias.child().fold(FoldContext.small()).toString(), is("127.0.0.1")); field = row.fields().get(6); assertThat(field.name(), is("e")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is(9)); + assertThat(alias.child().fold(FoldContext.small()), is(9)); field = row.fields().get(7); assertThat(field.name(), is("f")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is(11)); + assertThat(alias.child().fold(FoldContext.small()), is(11)); } public void testMissingInputParams() { @@ -1190,13 +1191,13 @@ public void testNamedParams() { assertThat(field.name(), is("x")); assertThat(field, instanceOf(Alias.class)); Alias alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); field = row.fields().get(1); assertThat(field.name(), is("y")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); } public void testInvalidNamedParams() { @@ -1237,13 +1238,13 @@ public void testPositionalParams() { assertThat(field.name(), is("x")); assertThat(field, instanceOf(Alias.class)); Alias alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); field = row.fields().get(1); assertThat(field.name(), is("y")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); } public void testInvalidPositionalParams() { @@ -2057,7 +2058,7 @@ private void assertStringAsLookupIndexPattern(String string, String statement) { var plan = statement(statement); var lookup = as(plan, Lookup.class); var tableName = as(lookup.tableName(), Literal.class); - assertThat(tableName.fold(), equalTo(string)); + assertThat(tableName.fold(FoldContext.small()), equalTo(string)); } public void testIdPatternUnquoted() throws Exception { @@ -2125,7 +2126,7 @@ public void testLookup() { var plan = statement(query); var lookup = as(plan, Lookup.class); var tableName = as(lookup.tableName(), Literal.class); - assertThat(tableName.fold(), equalTo("t")); + assertThat(tableName.fold(FoldContext.small()), equalTo("t")); assertThat(lookup.matchFields(), hasSize(1)); var matchField = as(lookup.matchFields().get(0), UnresolvedAttribute.class); assertThat(matchField.name(), equalTo("j")); @@ -2306,7 +2307,7 @@ public void testMatchOperatorConstantQueryString() { var match = (Match) filter.condition(); var matchField = (UnresolvedAttribute) match.field(); assertThat(matchField.name(), equalTo("field")); - assertThat(match.query().fold(), equalTo("value")); + assertThat(match.query().fold(FoldContext.small()), equalTo("value")); } public void testInvalidMatchOperator() { @@ -2341,7 +2342,7 @@ public void testMatchFunctionFieldCasting() { var toInteger = (ToInteger) function.children().get(0); var matchField = (UnresolvedAttribute) toInteger.field(); assertThat(matchField.name(), equalTo("field")); - assertThat(function.children().get(1).fold(), equalTo("value")); + assertThat(function.children().get(1).fold(FoldContext.small()), equalTo("value")); } public void testMatchOperatorFieldCasting() { @@ -2351,6 +2352,6 @@ public void testMatchOperatorFieldCasting() { var toInteger = (ToInteger) match.field(); var matchField = (UnresolvedAttribute) toInteger.field(); assertThat(matchField.name(), equalTo("field")); - assertThat(match.query().fold(), equalTo("value")); + assertThat(match.query().fold(FoldContext.small()), equalTo("value")); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java index a254207865ad5..2f47a672a68d0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; @@ -42,7 +43,7 @@ public void testTransformWithExpressionTopLevel() throws Exception { assertEquals(Limit.class, transformed.getClass()); Limit l = (Limit) transformed; - assertEquals(24, l.limit().fold()); + assertEquals(24, l.limit().fold(FoldContext.small())); } public void testTransformWithExpressionTree() throws Exception { @@ -53,7 +54,7 @@ public void testTransformWithExpressionTree() throws Exception { assertEquals(OrderBy.class, transformed.getClass()); OrderBy order = (OrderBy) transformed; assertEquals(Limit.class, order.child().getClass()); - assertEquals(24, ((Limit) order.child()).limit().fold()); + assertEquals(24, ((Limit) order.child()).limit().fold(FoldContext.small())); } public void testTransformWithExpressionTopLevelInCollection() throws Exception { @@ -83,12 +84,12 @@ public void testForEachWithExpressionTopLevel() throws Exception { List list = new ArrayList<>(); project.forEachExpression(Literal.class, l -> { - if (l.fold().equals(42)) { - list.add(l.fold()); + if (l.value().equals(42)) { + list.add(l.value()); } }); - assertEquals(singletonList(one.child().fold()), list); + assertEquals(singletonList(one.child().fold(FoldContext.small())), list); } public void testForEachWithExpressionTree() throws Exception { @@ -97,12 +98,12 @@ public void testForEachWithExpressionTree() throws Exception { List list = new ArrayList<>(); o.forEachExpressionDown(Literal.class, l -> { - if (l.fold().equals(42)) { - list.add(l.fold()); + if (l.value().equals(42)) { + list.add(l.value()); } }); - assertEquals(singletonList(limit.limit().fold()), list); + assertEquals(singletonList(limit.limit().fold(FoldContext.small())), list); } public void testForEachWithExpressionTopLevelInCollection() throws Exception { @@ -129,12 +130,12 @@ public void testForEachWithExpressionTreeInCollection() throws Exception { List list = new ArrayList<>(); project.forEachExpression(Literal.class, l -> { - if (l.fold().equals(42)) { - list.add(l.fold()); + if (l.value().equals(42)) { + list.add(l.value()); } }); - assertEquals(singletonList(one.child().fold()), list); + assertEquals(singletonList(one.child().fold(FoldContext.small())), list); } public void testPlanExpressions() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java index 5a7547d011c0f..e2eb05b0c14d3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.TestBlockFactory; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; @@ -145,7 +146,7 @@ public void testEvaluatorSuppliers() { lb.append(LONG); Layout layout = lb.build(); - var supplier = EvalMapper.toEvaluator(expression, layout); + var supplier = EvalMapper.toEvaluator(FoldContext.small(), expression, layout); EvalOperator.ExpressionEvaluator evaluator1 = supplier.get(driverContext()); EvalOperator.ExpressionEvaluator evaluator2 = supplier.get(driverContext()); assertNotNull(evaluator1); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java index 55f32d07fc2cb..4191f42f08237 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; -import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizer; @@ -52,6 +51,7 @@ import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization; import static org.elasticsearch.xpack.esql.core.util.Queries.Clause.FILTER; @@ -78,7 +78,7 @@ public static void init() { Map mapping = loadMapping("mapping-basic.json"); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); IndexResolution getIndexResult = IndexResolution.valid(test); - logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)); + logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext()); physicalPlanOptimizer = new PhysicalPlanOptimizer(new PhysicalOptimizerContext(EsqlTestUtils.TEST_CFG)); mapper = new Mapper(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java index 5d8da21c6faad..a1648c67d9bd4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.xpack.esql.TestBlockFactory; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -83,6 +84,7 @@ public void closeIndex() throws IOException { public void testLuceneSourceOperatorHugeRowSize() throws IOException { int estimatedRowSize = randomEstimatedRowSize(estimatedRowSizeIsHuge); LocalExecutionPlanner.LocalExecutionPlan plan = planner().plan( + FoldContext.small(), new EsQueryExec(Source.EMPTY, index(), IndexMode.STANDARD, List.of(), null, null, null, estimatedRowSize) ); assertThat(plan.driverFactories.size(), lessThanOrEqualTo(pragmas.taskConcurrency())); @@ -98,6 +100,7 @@ public void testLuceneTopNSourceOperator() throws IOException { EsQueryExec.FieldSort sort = new EsQueryExec.FieldSort(sortField, Order.OrderDirection.ASC, Order.NullsPosition.LAST); Literal limit = new Literal(Source.EMPTY, 10, DataType.INTEGER); LocalExecutionPlanner.LocalExecutionPlan plan = planner().plan( + FoldContext.small(), new EsQueryExec(Source.EMPTY, index(), IndexMode.STANDARD, List.of(), null, limit, List.of(sort), estimatedRowSize) ); assertThat(plan.driverFactories.size(), lessThanOrEqualTo(pragmas.taskConcurrency())); @@ -113,6 +116,7 @@ public void testLuceneTopNSourceOperatorDistanceSort() throws IOException { EsQueryExec.GeoDistanceSort sort = new EsQueryExec.GeoDistanceSort(sortField, Order.OrderDirection.ASC, 1, -1); Literal limit = new Literal(Source.EMPTY, 10, DataType.INTEGER); LocalExecutionPlanner.LocalExecutionPlan plan = planner().plan( + FoldContext.small(), new EsQueryExec(Source.EMPTY, index(), IndexMode.STANDARD, List.of(), null, limit, List.of(sort), estimatedRowSize) ); assertThat(plan.driverFactories.size(), lessThanOrEqualTo(pragmas.taskConcurrency())); @@ -187,7 +191,7 @@ private EsPhysicalOperationProviders esPhysicalOperationProviders() throws IOExc ); } releasables.add(searcher); - return new EsPhysicalOperationProviders(shardContexts, null); + return new EsPhysicalOperationProviders(FoldContext.small(), shardContexts, null); } private IndexReader reader() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index 01dd4db123ee2..628737aa36c6c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.esql.TestBlockFactory; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; @@ -71,13 +72,13 @@ public class TestPhysicalOperationProviders extends AbstractPhysicalOperationProviders { private final List indexPages; - private TestPhysicalOperationProviders(List indexPages, AnalysisRegistry analysisRegistry) { - super(analysisRegistry); + private TestPhysicalOperationProviders(FoldContext foldContext, List indexPages, AnalysisRegistry analysisRegistry) { + super(foldContext, analysisRegistry); this.indexPages = indexPages; } - public static TestPhysicalOperationProviders create(List indexPages) throws IOException { - return new TestPhysicalOperationProviders(indexPages, createAnalysisRegistry()); + public static TestPhysicalOperationProviders create(FoldContext foldContext, List indexPages) throws IOException { + return new TestPhysicalOperationProviders(foldContext, indexPages, createAnalysisRegistry()); } public record IndexPage(String index, Page page, List columnNames) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java index f2a619f0dbd89..f3b1d84e507a5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; -import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -39,10 +38,10 @@ import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomTables; -import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_CFG; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyPolicyResolution; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.hamcrest.Matchers.equalTo; @@ -187,7 +186,7 @@ static LogicalPlan parse(String query) { Map mapping = loadMapping("mapping-basic.json"); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); IndexResolution getIndexResult = IndexResolution.valid(test); - var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(TEST_CFG)); + var logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext()); var analyzer = new Analyzer( new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, emptyPolicyResolution()), TEST_VERIFIER diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java index 2cc733c2ea2e3..fac3495697da8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.index.EsIndex; @@ -289,7 +290,7 @@ static LogicalPlan parse(String query) { Map mapping = loadMapping("mapping-basic.json"); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); IndexResolution getIndexResult = IndexResolution.valid(test); - var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(TEST_CFG)); + var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(TEST_CFG, FoldContext.small())); var analyzer = new Analyzer( new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, emptyPolicyResolution()), TEST_VERIFIER diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java index 539cd0314a4d1..a3c5cd9168b4f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver; import org.elasticsearch.xpack.esql.execution.PlanExecutor; import org.elasticsearch.xpack.esql.session.EsqlSession; @@ -119,6 +120,7 @@ public void testFailedMetric() { request, randomAlphaOfLength(10), EsqlTestUtils.TEST_CFG, + FoldContext.small(), enrichResolver, new EsqlExecutionInfo(randomBoolean()), groupIndicesByCluster, @@ -149,6 +151,7 @@ public void onFailure(Exception e) { request, randomAlphaOfLength(10), EsqlTestUtils.TEST_CFG, + FoldContext.small(), enrichResolver, new EsqlExecutionInfo(randomBoolean()), groupIndicesByCluster, From d6133235a339eb99a146a6ff3ad6958fe0d63cf6 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 13 Jan 2025 16:12:34 +0100 Subject: [PATCH 12/44] Add index name validation rule for empty index names (#119960) * Add index name validation rule for empty index names * Created negative tests for ES|QL index names with only exclusion character * Added test for `*-` --- .../cluster/metadata/MetadataCreateIndexService.java | 3 +++ .../cluster/metadata/MetadataCreateIndexServiceTests.java | 3 +++ .../xpack/esql/parser/AbstractStatementParserTests.java | 8 ++++++-- .../xpack/esql/parser/StatementParserTests.java | 3 +++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 7fc8a8693dcdc..fb19a3f04fcec 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -237,6 +237,9 @@ public SystemIndices getSystemIndices() { * Validate the name for an index or alias against some static rules. */ public static void validateIndexOrAliasName(String index, BiFunction exceptionCtor) { + if (index == null || index.isEmpty()) { + throw exceptionCtor.apply(index, "must not be empty"); + } if (Strings.validFileName(index) == false) { throw exceptionCtor.apply(index, "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 3623683532c59..3ed74392f746e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -561,6 +561,9 @@ public void testValidateIndexName() throws Exception { validateIndexName(checkerService, "..", "must not be '.' or '..'"); validateIndexName(checkerService, "foo:bar", "must not contain ':'"); + + validateIndexName(checkerService, "", "must not be empty"); + validateIndexName(checkerService, null, "must not be empty"); })); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java index e6fef186721a0..99a04b6ed8f10 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java @@ -151,8 +151,12 @@ void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, expectInvalidIndexNameErrorWithLineNumber(query, "\"" + indexString + "\"", lineNumber, indexString); } - void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, String lineNumber, String error) { - expectError(LoggerMessageFormat.format(null, query, indexString), lineNumber + "Invalid index name [" + error); + void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, String lineNumber, String name) { + expectError(LoggerMessageFormat.format(null, query, indexString), lineNumber + "Invalid index name [" + name); + } + + void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, String lineNumber, String name, String error) { + expectError(LoggerMessageFormat.format(null, query, indexString), lineNumber + "Invalid index name [" + name + "], " + error); } void expectDateMathErrorWithLineNumber(String query, String arg, String lineNumber, String error) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 49f03e9b8bc2f..a4712ae77b5d8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -578,6 +578,9 @@ public void testInvalidCharacterInIndexPattern() { expectInvalidIndexNameErrorWithLineNumber(command, "indexpattern, --indexpattern", lineNumber, "-indexpattern"); expectInvalidIndexNameErrorWithLineNumber(command, "indexpattern, \"--indexpattern\"", lineNumber, "-indexpattern"); expectInvalidIndexNameErrorWithLineNumber(command, "\"indexpattern, --indexpattern\"", commands.get(command), "-indexpattern"); + expectInvalidIndexNameErrorWithLineNumber(command, "\"- , -\"", commands.get(command), "", "must not be empty"); + expectInvalidIndexNameErrorWithLineNumber(command, "\"indexpattern,-\"", commands.get(command), "", "must not be empty"); + clustersAndIndices(command, "indexpattern", "*-"); clustersAndIndices(command, "indexpattern", "-indexpattern"); } From a8d2680cc203b79a583b18ec9df4c49b7c219dd2 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Mon, 13 Jan 2025 16:13:21 +0100 Subject: [PATCH 13/44] Unmute LocalPhysicalPlanOptimizerTests (#119884) * Unmute tests * Mute tests --- muted-tests.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index d9fcebb7f2916..cecff79b002a2 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -227,15 +227,6 @@ tests: - class: org.elasticsearch.search.profile.dfs.DfsProfilerIT method: testProfileDfs issue: https://github.com/elastic/elasticsearch/issues/119711 -- class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests - method: testSingleMatchFunctionFilterPushdownWithStringValues {default} - issue: https://github.com/elastic/elasticsearch/issues/119720 -- class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests - method: testSingleMatchFunctionPushdownWithCasting {default} - issue: https://github.com/elastic/elasticsearch/issues/119722 -- class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests - method: testSingleMatchOperatorFilterPushdownWithStringValues {default} - issue: https://github.com/elastic/elasticsearch/issues/119721 - class: org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilterIT method: testBulkOperations {p0=false} issue: https://github.com/elastic/elasticsearch/issues/119901 From 53773cbf2404e8b426e0c78f062b3463e604f642 Mon Sep 17 00:00:00 2001 From: Svilen Mihaylov Date: Mon, 13 Jan 2025 10:34:29 -0500 Subject: [PATCH 14/44] Fix for issue 119723 (#120001) Adjust random parameter generator to exclude newly valid value (-1) Closes #119723 --- .../search/fetch/subphase/highlight/HighlightBuilderTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index 0f73c367ff2ef..1bcc89b68c141 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -574,11 +574,10 @@ public void testPreTagsWithoutPostTags() throws IOException { assertEquals("pre_tags are set but post_tags are not set", e.getCause().getCause().getMessage()); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/119723") public void testInvalidMaxAnalyzedOffset() throws IOException { XContentParseException e = expectParseThrows( XContentParseException.class, - "{ \"max_analyzed_offset\" : " + randomIntBetween(-100, -1) + "}" + "{ \"max_analyzed_offset\" : " + randomIntBetween(-100, -2) + "}" ); assertThat(e.getMessage(), containsString("[highlight] failed to parse field [" + MAX_ANALYZED_OFFSET_FIELD.toString() + "]")); assertThat(e.getCause().getMessage(), containsString("[max_analyzed_offset] must be a positive integer, or -1")); From de0e7c9d80488e491b1d32927d87aa4de9111de5 Mon Sep 17 00:00:00 2001 From: Luke Whiting Date: Mon, 13 Jan 2025 16:01:50 +0000 Subject: [PATCH 15/44] Match dot prefix of migrated DS backing index with the source index (#120042) * Match dot prefix of migrated index with the source index * Update docs/changelog/120042.yaml --- docs/changelog/120042.yaml | 5 +++ ...indexDatastreamIndexTransportActionIT.java | 44 ++++++++++++++++++- ...ReindexDataStreamIndexTransportAction.java | 8 +++- 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/120042.yaml diff --git a/docs/changelog/120042.yaml b/docs/changelog/120042.yaml new file mode 100644 index 0000000000000..0093068ae9894 --- /dev/null +++ b/docs/changelog/120042.yaml @@ -0,0 +1,5 @@ +pr: 120042 +summary: Match dot prefix of migrated DS backing index with the source index +area: Data streams +type: bug +issues: [] diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java index 0902f6ce6468a..46af8ab2fb4c2 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java @@ -96,7 +96,7 @@ public void testDestIndexDeletedIfExists() throws Exception { assertHitCount(prepareSearch(destIndex).setSize(0), 0); } - public void testDestIndexNameSet() throws Exception { + public void testDestIndexNameSet_noDotPrefix() throws Exception { assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); @@ -110,6 +110,20 @@ public void testDestIndexNameSet() throws Exception { assertEquals(expectedDestIndexName, response.getDestIndex()); } + public void testDestIndexNameSet_withDotPrefix() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var sourceIndex = "." + randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + + // call reindex + var response = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet(); + + var expectedDestIndexName = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + assertEquals(expectedDestIndexName, response.getDestIndex()); + } + public void testDestIndexContainsDocs() throws Exception { assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); @@ -446,4 +460,32 @@ private static String getIndexUUID(String index) { .get(index) .get(IndexMetadata.SETTING_INDEX_UUID); } + + public void testGenerateDestIndexName_noDotPrefix() { + String sourceIndex = "sourceindex"; + String expectedDestIndex = "migrated-sourceindex"; + String actualDestIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + assertEquals(expectedDestIndex, actualDestIndex); + } + + public void testGenerateDestIndexName_withDotPrefix() { + String sourceIndex = ".sourceindex"; + String expectedDestIndex = ".migrated-sourceindex"; + String actualDestIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + assertEquals(expectedDestIndex, actualDestIndex); + } + + public void testGenerateDestIndexName_withHyphen() { + String sourceIndex = "source-index"; + String expectedDestIndex = "migrated-source-index"; + String actualDestIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + assertEquals(expectedDestIndex, actualDestIndex); + } + + public void testGenerateDestIndexName_withUnderscore() { + String sourceIndex = "source_index"; + String expectedDestIndex = "migrated-source_index"; + String actualDestIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + assertEquals(expectedDestIndex, actualDestIndex); + } } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java index d86885ce0fbe4..ff350429dae01 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java @@ -221,8 +221,12 @@ private static void copySettingOrUnset(Settings settingsBefore, Settings.Builder } } - public static String generateDestIndexName(String sourceIndex) { - return "migrated-" + sourceIndex; + static String generateDestIndexName(String sourceIndex) { + String prefix = "migrated-"; + if (sourceIndex.startsWith(".")) { + return "." + prefix + sourceIndex.substring(1); + } + return prefix + sourceIndex; } private static ActionListener failIfNotAcknowledged( From d79edcbec8058b957c851541fa74e8bb3e3fa46a Mon Sep 17 00:00:00 2001 From: Dan Rubinstein Date: Mon, 13 Jan 2025 11:38:26 -0500 Subject: [PATCH 16/44] Add enterprise license check for Inference API actions (#119893) * Add enterprise license check for Inference API actions * Update docs/changelog/119893.yaml * Adding missing plugin to ModelRegistryIT and removing license check from get inference services API * Fix tests * Fix basic license test * Removing unused feature flag from InferenceUpgradeTestCase --------- Co-authored-by: Elastic Machine --- docs/changelog/119893.yaml | 5 ++ .../inference/InferenceBasicLicenseIT.java | 84 +++++++++++++++++++ .../InferenceLicenseBaseRestTest.java | 43 ++++++++++ .../inference/InferenceTrialLicenseIT.java | 84 +++++++++++++++++++ .../application/InferenceUpgradeTestCase.java | 21 ++++- .../ShardBulkInferenceActionFilterIT.java | 3 +- .../integration/ModelRegistryIT.java | 3 +- .../xpack/inference/InferencePlugin.java | 8 ++ .../action/BaseTransportInferenceAction.java | 12 +++ .../action/TransportInferenceAction.java | 3 + .../TransportPutInferenceModelAction.java | 12 +++ ...sportUnifiedCompletionInferenceAction.java | 3 + .../TransportUpdateInferenceModelAction.java | 12 +++ .../SemanticTextClusterMetadataTests.java | 3 +- .../BaseTransportInferenceActionTestCase.java | 21 ++++- .../action/TransportInferenceActionTests.java | 3 + ...TransportUnifiedCompletionActionTests.java | 3 + ...emanticTextNonDynamicFieldMapperTests.java | 3 +- .../TextSimilarityRankMultiNodeTests.java | 3 +- .../TextSimilarityRankTests.java | 3 +- 20 files changed, 323 insertions(+), 9 deletions(-) create mode 100644 docs/changelog/119893.yaml create mode 100644 x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBasicLicenseIT.java create mode 100644 x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceLicenseBaseRestTest.java create mode 100644 x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceTrialLicenseIT.java diff --git a/docs/changelog/119893.yaml b/docs/changelog/119893.yaml new file mode 100644 index 0000000000000..35a46ce0940d3 --- /dev/null +++ b/docs/changelog/119893.yaml @@ -0,0 +1,5 @@ +pr: 119893 +summary: Add enterprise license check for Inference API actions +area: Machine Learning +type: enhancement +issues: [] diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBasicLicenseIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBasicLicenseIT.java new file mode 100644 index 0000000000000..4400ad8bbb538 --- /dev/null +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBasicLicenseIT.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.junit.ClassRule; + +import static org.elasticsearch.xpack.inference.InferenceBaseRestTest.mockSparseServiceModelConfig; + +public class InferenceBasicLicenseIT extends InferenceLicenseBaseRestTest { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .setting("xpack.license.self_generated.type", "basic") + .setting("xpack.security.enabled", "true") + .user("x_pack_rest_user", "x-pack-test-password") + .plugin("inference-service-test") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + public void testPutModel_RestrictedWithBasicLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + var modelConfig = mockSparseServiceModelConfig(null, true); + sendRestrictedRequest("PUT", endpoint, modelConfig); + } + + public void testUpdateModel_RestrictedWithBasicLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s/_update?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + var requestBody = """ + { + "task_settings": { + "num_threads": 2 + } + } + """; + sendRestrictedRequest("PUT", endpoint, requestBody); + } + + public void testPerformInference_RestrictedWithBasicLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + var requestBody = """ + { + "input": ["washing", "machine"] + } + """; + sendRestrictedRequest("POST", endpoint, requestBody); + } + + public void testGetServices_NonRestrictedWithBasicLicense() throws Exception { + var endpoint = "_inference/_services"; + sendNonRestrictedRequest("GET", endpoint, null, 200, false); + } + + public void testGetModels_NonRestrictedWithBasicLicense() throws Exception { + var endpoint = "_inference/_all"; + sendNonRestrictedRequest("GET", endpoint, null, 200, false); + } + + public void testDeleteModel_NonRestrictedWithBasicLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + sendNonRestrictedRequest("DELETE", endpoint, null, 404, true); + } +} diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceLicenseBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceLicenseBaseRestTest.java new file mode 100644 index 0000000000000..43183bae73252 --- /dev/null +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceLicenseBaseRestTest.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.containsString; + +public class InferenceLicenseBaseRestTest extends ESRestTestCase { + protected void sendRestrictedRequest(String method, String endpoint, String body) throws IOException { + var request = new Request(method, endpoint); + request.setJsonEntity(body); + + var exception = assertThrows(ResponseException.class, () -> client().performRequest(request)); + assertEquals(403, exception.getResponse().getStatusLine().getStatusCode()); + assertThat(exception.getMessage(), containsString("current license is non-compliant for [inference]")); + } + + protected void sendNonRestrictedRequest(String method, String endpoint, String body, int expectedStatusCode, boolean exceptionExpected) + throws IOException { + var request = new Request(method, endpoint); + request.setJsonEntity(body); + + int actualStatusCode; + if (exceptionExpected) { + var exception = assertThrows(ResponseException.class, () -> client().performRequest(request)); + actualStatusCode = exception.getResponse().getStatusLine().getStatusCode(); + } else { + var response = client().performRequest(request); + actualStatusCode = response.getStatusLine().getStatusCode(); + } + assertEquals(expectedStatusCode, actualStatusCode); + } +} diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceTrialLicenseIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceTrialLicenseIT.java new file mode 100644 index 0000000000000..c7066a827fa7e --- /dev/null +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceTrialLicenseIT.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.junit.ClassRule; + +import static org.elasticsearch.xpack.inference.InferenceBaseRestTest.mockSparseServiceModelConfig; + +public class InferenceTrialLicenseIT extends InferenceLicenseBaseRestTest { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.security.enabled", "true") + .user("x_pack_rest_user", "x-pack-test-password") + .plugin("inference-service-test") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + public void testPutModel_NonRestrictedWithTrialLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + var modelConfig = mockSparseServiceModelConfig(null, true); + sendNonRestrictedRequest("PUT", endpoint, modelConfig, 200, false); + } + + public void testUpdateModel_NonRestrictedWithTrialLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s/_update?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + var requestBody = """ + { + "task_settings": { + "num_threads": 2 + } + } + """; + sendNonRestrictedRequest("PUT", endpoint, requestBody, 404, true); + } + + public void testPerformInference_NonRestrictedWithTrialLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + var requestBody = """ + { + "input": ["washing", "machine"] + } + """; + sendNonRestrictedRequest("POST", endpoint, requestBody, 404, true); + } + + public void testGetServices_NonRestrictedWithBasicLicense() throws Exception { + var endpoint = "_inference/_services"; + sendNonRestrictedRequest("GET", endpoint, null, 200, false); + } + + public void testGetModels_NonRestrictedWithBasicLicense() throws Exception { + var endpoint = "_inference/_all"; + sendNonRestrictedRequest("GET", endpoint, null, 200, false); + } + + public void testDeleteModel_NonRestrictedWithBasicLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + sendNonRestrictedRequest("DELETE", endpoint, null, 404, true); + } +} diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java index d38503a884092..880557c59f11c 100644 --- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java +++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java @@ -12,8 +12,11 @@ import org.elasticsearch.client.Request; import org.elasticsearch.common.Strings; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.http.MockWebServer; -import org.elasticsearch.upgrades.AbstractRollingUpgradeTestCase; +import org.elasticsearch.upgrades.ParameterizedRollingUpgradeTestCase; +import org.junit.ClassRule; import java.io.IOException; import java.util.LinkedList; @@ -22,7 +25,7 @@ import static org.elasticsearch.core.Strings.format; -public class InferenceUpgradeTestCase extends AbstractRollingUpgradeTestCase { +public class InferenceUpgradeTestCase extends ParameterizedRollingUpgradeTestCase { static final String MODELS_RENAMED_TO_ENDPOINTS = "8.15.0"; @@ -30,6 +33,20 @@ public InferenceUpgradeTestCase(@Name("upgradedNodes") int upgradedNodes) { super(upgradedNodes); } + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(NODE_NUM) + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .build(); + + @Override + protected ElasticsearchCluster getUpgradeCluster() { + return cluster; + } + protected static String getUrl(MockWebServer webServer) { return format("http://%s:%s", webServer.getHostName(), webServer.getPort()); } diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java index 90a9dd3355b3c..3f7a51cc4e9f9 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.inference.Utils; import org.elasticsearch.xpack.inference.mock.TestDenseInferenceServiceExtension; import org.elasticsearch.xpack.inference.mock.TestSparseInferenceServiceExtension; @@ -73,7 +74,7 @@ public void setup() throws Exception { @Override protected Collection> nodePlugins() { - return Arrays.asList(Utils.TestInferencePlugin.class); + return Arrays.asList(Utils.TestInferencePlugin.class, LocalStateCompositeXPackPlugin.class); } @Override diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java index be6b3725b0f35..24585318b15b3 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests; import org.elasticsearch.xpack.inference.registry.ModelRegistry; @@ -76,7 +77,7 @@ public void createComponents() { @Override protected Collection> getPlugins() { - return pluginList(ReindexPlugin.class, InferencePlugin.class); + return pluginList(ReindexPlugin.class, InferencePlugin.class, LocalStateCompositeXPackPlugin.class); } public void testStoreModel() throws Exception { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index b16c53a428d73..d303ead4d9188 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -29,6 +29,8 @@ import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.inference.InferenceServiceExtension; import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.license.License; +import org.elasticsearch.license.LicensedFeature; import org.elasticsearch.node.PluginComponentBinding; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.ExtensiblePlugin; @@ -150,6 +152,12 @@ public class InferencePlugin extends Plugin implements ActionPlugin, ExtensibleP Setting.Property.Dynamic ); + public static final LicensedFeature.Momentary INFERENCE_API_FEATURE = LicensedFeature.momentary( + "inference", + "api", + License.OperationMode.ENTERPRISE + ); + public static final String NAME = "inference"; public static final String UTILITY_THREAD_POOL_NAME = "inference_utility"; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java index 2a0e8e1775279..b6c7d26b36f9a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java @@ -23,9 +23,12 @@ import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.inference.action.BaseInferenceActionRequest; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; @@ -38,6 +41,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.xpack.inference.InferencePlugin.INFERENCE_API_FEATURE; import static org.elasticsearch.xpack.inference.telemetry.InferenceStats.modelAttributes; import static org.elasticsearch.xpack.inference.telemetry.InferenceStats.responseAttributes; @@ -48,6 +52,7 @@ public abstract class BaseTransportInferenceAction requestReader ) { super(inferenceActionName, transportService, actionFilters, requestReader, EsExecutors.DIRECT_EXECUTOR_SERVICE); + this.licenseState = licenseState; this.modelRegistry = modelRegistry; this.serviceRegistry = serviceRegistry; this.inferenceStats = inferenceStats; @@ -72,6 +79,11 @@ public BaseTransportInferenceAction( @Override protected void doExecute(Task task, Request request, ActionListener listener) { + if (INFERENCE_API_FEATURE.check(licenseState) == false) { + listener.onFailure(LicenseUtils.newComplianceException(XPackField.INFERENCE)); + return; + } + var timer = InferenceTimer.start(); var getModelListener = ActionListener.wrap((UnparsedModel unparsedModel) -> { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java index 08e6d869a553d..24ef0d7d610d0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.inference.Model; import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; @@ -28,6 +29,7 @@ public class TransportInferenceAction extends BaseTransportInferenceAction listener ) throws Exception { + if (INFERENCE_API_FEATURE.check(licenseState) == false) { + listener.onFailure(LicenseUtils.newComplianceException(XPackField.INFERENCE)); + return; + } + var requestAsMap = requestToMap(request); var resolvedTaskType = ServiceUtils.resolveTaskType(request.getTaskType(), (String) requestAsMap.remove(TaskType.NAME)); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java index f0906231d8f42..1478130f6a6c8 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; @@ -30,6 +31,7 @@ public class TransportUnifiedCompletionInferenceAction extends BaseTransportInfe public TransportUnifiedCompletionInferenceAction( TransportService transportService, ActionFilters actionFilters, + XPackLicenseState licenseState, ModelRegistry modelRegistry, InferenceServiceRegistry serviceRegistry, InferenceStats inferenceStats, @@ -39,6 +41,7 @@ public TransportUnifiedCompletionInferenceAction( UnifiedCompletionAction.NAME, transportService, actionFilters, + licenseState, modelRegistry, serviceRegistry, inferenceStats, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java index 3c47de1ad64d1..4a90f73722438 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java @@ -34,12 +34,15 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.inference.action.UpdateInferenceModelAction; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; import org.elasticsearch.xpack.core.ml.action.UpdateTrainedModelDeploymentAction; @@ -57,6 +60,7 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; +import static org.elasticsearch.xpack.inference.InferencePlugin.INFERENCE_API_FEATURE; import static org.elasticsearch.xpack.inference.services.ServiceUtils.resolveTaskType; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS; @@ -66,6 +70,7 @@ public class TransportUpdateInferenceModelAction extends TransportMasterNodeActi private static final Logger logger = LogManager.getLogger(TransportUpdateInferenceModelAction.class); + private final XPackLicenseState licenseState; private final ModelRegistry modelRegistry; private final InferenceServiceRegistry serviceRegistry; private final Client client; @@ -77,6 +82,7 @@ public TransportUpdateInferenceModelAction( ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + XPackLicenseState licenseState, ModelRegistry modelRegistry, InferenceServiceRegistry serviceRegistry, Client client @@ -92,6 +98,7 @@ public TransportUpdateInferenceModelAction( UpdateInferenceModelAction.Response::new, EsExecutors.DIRECT_EXECUTOR_SERVICE ); + this.licenseState = licenseState; this.modelRegistry = modelRegistry; this.serviceRegistry = serviceRegistry; this.client = client; @@ -104,6 +111,11 @@ protected void masterOperation( ClusterState state, ActionListener masterListener ) { + if (INFERENCE_API_FEATURE.check(licenseState) == false) { + masterListener.onFailure(LicenseUtils.newComplianceException(XPackField.INFERENCE)); + return; + } + var bodyTaskType = request.getContentAsSettings().taskType(); var resolvedTaskType = resolveTaskType(request.getTaskType(), bodyTaskType != null ? bodyTaskType.toString() : null); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/cluster/metadata/SemanticTextClusterMetadataTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/cluster/metadata/SemanticTextClusterMetadataTests.java index bfec2d5ac3484..a595134ecd548 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/cluster/metadata/SemanticTextClusterMetadataTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/cluster/metadata/SemanticTextClusterMetadataTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.inference.InferencePlugin; import org.hamcrest.Matchers; @@ -28,7 +29,7 @@ public class SemanticTextClusterMetadataTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return List.of(InferencePlugin.class); + return List.of(InferencePlugin.class, LocalStateCompositeXPackPlugin.class); } public void testCreateIndexWithSemanticTextField() { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java index 47f3a0e0b57aa..a723e5a9dffdf 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java @@ -18,11 +18,13 @@ import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.BaseInferenceActionRequest; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; import org.elasticsearch.xpack.inference.registry.ModelRegistry; import org.elasticsearch.xpack.inference.telemetry.InferenceStats; @@ -49,6 +51,7 @@ import static org.mockito.Mockito.when; public abstract class BaseTransportInferenceActionTestCase extends ESTestCase { + private MockLicenseState licenseState; private ModelRegistry modelRegistry; private StreamingTaskManager streamingTaskManager; private BaseTransportInferenceAction action; @@ -64,16 +67,28 @@ public void setUp() throws Exception { super.setUp(); TransportService transportService = mock(); ActionFilters actionFilters = mock(); + licenseState = mock(); modelRegistry = mock(); serviceRegistry = mock(); inferenceStats = new InferenceStats(mock(), mock()); streamingTaskManager = mock(); - action = createAction(transportService, actionFilters, modelRegistry, serviceRegistry, inferenceStats, streamingTaskManager); + action = createAction( + transportService, + actionFilters, + licenseState, + modelRegistry, + serviceRegistry, + inferenceStats, + streamingTaskManager + ); + + mockValidLicenseState(); } protected abstract BaseTransportInferenceAction createAction( TransportService transportService, ActionFilters actionFilters, + MockLicenseState licenseState, ModelRegistry modelRegistry, InferenceServiceRegistry serviceRegistry, InferenceStats inferenceStats, @@ -361,4 +376,8 @@ protected void mockModelAndServiceRegistry(InferenceService service) { when(serviceRegistry.getService(any())).thenReturn(Optional.of(service)); } + + protected void mockValidLicenseState(){ + when(licenseState.isAllowed(InferencePlugin.INFERENCE_API_FEATURE)).thenReturn(true); + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java index e54175cb27009..a5efe04c22c04 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; @@ -23,6 +24,7 @@ public class TransportInferenceActionTests extends BaseTransportInferenceActionT protected BaseTransportInferenceAction createAction( TransportService transportService, ActionFilters actionFilters, + MockLicenseState licenseState, ModelRegistry modelRegistry, InferenceServiceRegistry serviceRegistry, InferenceStats inferenceStats, @@ -31,6 +33,7 @@ protected BaseTransportInferenceAction createAction( return new TransportInferenceAction( transportService, actionFilters, + licenseState, modelRegistry, serviceRegistry, inferenceStats, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java index 4c943599ce523..3856a3d111b6e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.inference.InferenceServiceRegistry; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; @@ -36,6 +37,7 @@ public class TransportUnifiedCompletionActionTests extends BaseTransportInferenc protected BaseTransportInferenceAction createAction( TransportService transportService, ActionFilters actionFilters, + MockLicenseState licenseState, ModelRegistry modelRegistry, InferenceServiceRegistry serviceRegistry, InferenceStats inferenceStats, @@ -44,6 +46,7 @@ protected BaseTransportInferenceAction createAc return new TransportUnifiedCompletionInferenceAction( transportService, actionFilters, + licenseState, modelRegistry, serviceRegistry, inferenceStats, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextNonDynamicFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextNonDynamicFieldMapperTests.java index 1f58c4165056d..0025b3a53a69f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextNonDynamicFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextNonDynamicFieldMapperTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.index.mapper.NonDynamicFieldMapperTests; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.inference.Utils; import org.elasticsearch.xpack.inference.mock.TestSparseInferenceServiceExtension; import org.junit.Before; @@ -26,7 +27,7 @@ public void setup() throws Exception { @Override protected Collection> getPlugins() { - return List.of(Utils.TestInferencePlugin.class); + return List.of(Utils.TestInferencePlugin.class, LocalStateCompositeXPackPlugin.class); } @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankMultiNodeTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankMultiNodeTests.java index 6d6403b69ea11..69b1e19fa91de 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankMultiNodeTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankMultiNodeTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.rank.RankBuilder; import org.elasticsearch.search.rank.rerank.AbstractRerankerIT; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.inference.InferencePlugin; import java.util.Collection; @@ -40,7 +41,7 @@ protected RankBuilder getThrowingRankBuilder(int rankWindowSize, String rankFeat @Override protected Collection> pluginsNeeded() { - return List.of(InferencePlugin.class, TextSimilarityTestPlugin.class); + return List.of(InferencePlugin.class, TextSimilarityTestPlugin.class, LocalStateCompositeXPackPlugin.class); } public void testQueryPhaseShardThrowingAllShardsFail() throws Exception { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankTests.java index a042fca44fdb5..a6a4ce2b2ffdf 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.search.rank.rerank.AbstractRerankerIT; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.InferencePlugin; import org.junit.Before; @@ -108,7 +109,7 @@ protected InferenceAction.Request generateRequest(List docFeatures) { @Override protected Collection> getPlugins() { - return List.of(InferencePlugin.class, TextSimilarityTestPlugin.class); + return List.of(InferencePlugin.class, TextSimilarityTestPlugin.class, LocalStateCompositeXPackPlugin.class); } @Before From 3e12016a9795f1307e848d4d295bd64725d35376 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 13 Jan 2025 17:46:08 +0100 Subject: [PATCH 17/44] Improve error handling when parsing retrievers (#120047) PARSER#declareNamedObject does not handle empty objects or objects with extra stuff. Therefore we do it manually. --- .../random/RandomRankRetrieverBuilder.java | 24 ++++++- .../RandomRankRetrieverBuilderTests.java | 63 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java index 7236b0141a86d..503000c31f7e7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.retriever.RetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverParserContext; import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -48,7 +49,12 @@ public class RandomRankRetrieverBuilder extends RetrieverBuilder { }); static { - PARSER.declareNamedObject(constructorArg(), (p, c, n) -> p.namedObject(RetrieverBuilder.class, n, c), RETRIEVER_FIELD); + PARSER.declareField( + constructorArg(), + RandomRankRetrieverBuilder::parseRetrieverBuilder, + RETRIEVER_FIELD, + ObjectParser.ValueType.OBJECT + ); PARSER.declareString(optionalConstructorArg(), FIELD_FIELD); PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD); PARSER.declareInt(optionalConstructorArg(), SEED_FIELD); @@ -63,6 +69,22 @@ public static RandomRankRetrieverBuilder fromXContent(XContentParser parser, Ret return PARSER.apply(parser, context); } + private static RetrieverBuilder parseRetrieverBuilder(XContentParser parser, RetrieverParserContext context) throws IOException { + assert parser.currentToken() == XContentParser.Token.START_OBJECT; + parser.nextToken(); + if (parser.currentToken() == XContentParser.Token.END_OBJECT) { + throw new ParsingException(parser.getTokenLocation(), "empty [" + RETRIEVER_FIELD + "] object"); + } + assert parser.currentToken() == XContentParser.Token.FIELD_NAME; + final RetrieverBuilder builder = parser.namedObject(RetrieverBuilder.class, parser.currentName(), context); + parser.nextToken(); + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + throw new ParsingException(parser.getTokenLocation(), "unexpected field [" + parser.currentName() + "]"); + } + assert parser.currentToken() == XContentParser.Token.END_OBJECT; + return builder; + } + private final RetrieverBuilder retrieverBuilder; private final String field; private final int rankWindowSize; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java index c0ef4e45f101f..deb5fb47ab939 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.usage.SearchUsage; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.json.JsonXContent; @@ -23,6 +24,7 @@ import java.util.List; import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE; +import static org.hamcrest.Matchers.containsString; public class RandomRankRetrieverBuilderTests extends AbstractXContentTestCase { @@ -99,4 +101,65 @@ public void testParserDefaults() throws IOException { } } + public void testParserEmptyRetriever() throws IOException { + String json = """ + { + "retriever": { + }, + "field": "my-field" + }"""; + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + XContentParseException ex = expectThrows( + XContentParseException.class, + () -> RandomRankRetrieverBuilder.PARSER.parse(parser, null) + ); + assertThat(ex.getMessage(), containsString("[random_reranker] failed to parse field [retriever]")); + assertThat(ex.getCause().getMessage(), containsString("empty [retriever] object")); + } + } + + public void testParserWrongRetrieverName() throws IOException { + String json = """ + { + "retriever": { + "test2": { + "value": "my-test-retriever" + } + }, + "field": "my-field" + }"""; + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + XContentParseException ex = expectThrows( + XContentParseException.class, + () -> RandomRankRetrieverBuilder.PARSER.parse(parser, null) + ); + assertThat(ex.getMessage(), containsString("[random_reranker] failed to parse field [retriever]")); + assertThat(ex.getCause().getMessage(), containsString("unknown field [test2]")); + } + } + + public void testExtraContent() throws IOException { + String json = """ + { + "retriever": { + "test": { + "value": "my-test-retriever" + }, + "field2": "my-field" + }, + "field": "my-field" + }"""; + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + XContentParseException ex = expectThrows( + XContentParseException.class, + () -> RandomRankRetrieverBuilder.PARSER.parse(parser, null) + ); + assertThat(ex.getMessage(), containsString("[random_reranker] failed to parse field [retriever]")); + assertThat(ex.getCause().getMessage(), containsString("unexpected field [field2]")); + } + } + } From 7a267ee9e2bd2554246c7efe64ec84010f9ad55b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= Date: Mon, 13 Jan 2025 17:55:39 +0100 Subject: [PATCH 18/44] ESQL: Function tests standardization (#119506) Made a quick review of the ESQL function tests, using `parameterSuppliersFromTypedDataWithDefaultChecks` when possible, and documenting a bit if not. Some tests didn't have the error checks for types/nulls, so it added them. Some functions couldn't be updated, and found some bugs and 500s in the process, which will be fixed in separate PR(s). --- .../AbstractScalarFunctionTestCase.java | 2 +- .../scalar/convert/ToDatePeriodTests.java | 2 +- .../scalar/convert/ToTimeDurationTests.java | 2 +- .../function/scalar/date/DateDiffTests.java | 2 +- .../function/scalar/date/NowTests.java | 6 +- .../function/scalar/math/AcosTests.java | 2 +- .../function/scalar/math/AsinTests.java | 2 +- .../function/scalar/math/CbrtTests.java | 2 +- .../function/scalar/math/ETests.java | 24 +++-- .../function/scalar/math/ExpTests.java | 2 +- .../function/scalar/math/Log10Tests.java | 2 +- .../function/scalar/math/LogTests.java | 6 +- .../function/scalar/math/PiTests.java | 24 +++-- .../function/scalar/math/PowTests.java | 2 +- .../function/scalar/math/SignumTests.java | 2 +- .../function/scalar/math/SqrtTests.java | 2 +- .../function/scalar/math/TauTests.java | 24 +++-- .../multivalue/MvPSeriesWeightedSumTests.java | 12 +-- .../scalar/multivalue/MvSortTests.java | 93 +++++-------------- .../scalar/multivalue/MvSumTests.java | 2 +- .../function/scalar/string/LocateTests.java | 4 +- .../function/scalar/string/RepeatTests.java | 4 +- .../function/scalar/string/ReverseTests.java | 2 +- .../operator/arithmetic/AddTests.java | 39 +++----- .../operator/arithmetic/DivTests.java | 5 +- .../operator/arithmetic/ModTests.java | 5 +- .../operator/arithmetic/MulTests.java | 16 +++- .../operator/arithmetic/SubTests.java | 44 ++++----- .../operator/comparison/EqualsTests.java | 2 +- .../comparison/GreaterThanOrEqualTests.java | 2 +- .../operator/comparison/GreaterThanTests.java | 2 +- .../comparison/LessThanOrEqualTests.java | 2 +- .../operator/comparison/LessThanTests.java | 2 +- .../operator/comparison/NotEqualsTests.java | 3 +- 34 files changed, 160 insertions(+), 187 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java index 64086334b7251..944515e54af75 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java @@ -421,7 +421,7 @@ protected static TestCaseSupplier arithmeticExceptionOverflowCase( String typeNameOverflow = dataType.typeName().toLowerCase(Locale.ROOT) + " overflow"; return new TestCaseSupplier( "<" + typeNameOverflow + ">", - List.of(dataType), + List.of(dataType, dataType), () -> new TestCaseSupplier.TestCase( List.of( new TestCaseSupplier.TypedData(lhsSupplier.get(), dataType, "lhs"), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java index 9abbfbd61c1a8..8060326365d5c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java @@ -71,7 +71,7 @@ public static Iterable parameters() { })); } } - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java index 6486f6efcdd3c..980eed1af9109 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java @@ -70,7 +70,7 @@ public static Iterable parameters() { })); } } - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java index da069e3c37cc4..b4a37b0297571 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java @@ -99,7 +99,7 @@ public static Iterable parameters() { equalTo(0) ); })); - return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } public void testDateDiffFunction() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java index ce6ee1702ee66..c667747a8ba75 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java @@ -32,7 +32,8 @@ public NowTests(@Name("TestCase") Supplier testCaseSu @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedData( + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, List.of( new TestCaseSupplier( "Now Test", @@ -44,7 +45,8 @@ public static Iterable parameters() { equalTo(EsqlTestUtils.TEST_CFG.now().toInstant().toEpochMilli()) ) ) - ) + ), + (valid, position) -> "" ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java index 278c9123e30b1..6531e7bee90ab 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java @@ -56,7 +56,7 @@ public static Iterable parameters() { ) ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java index 04fec5a20b438..410dc61ec5fa6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java @@ -56,7 +56,7 @@ public static Iterable parameters() { ) ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java index bfe35a08b8ba1..d702e28baf9d8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java @@ -72,7 +72,7 @@ public static Iterable parameters() { ); suppliers = anyNullIsNull(true, suppliers); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java index 50ed71262e5df..f3922a355180d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java @@ -29,14 +29,22 @@ public ETests(@Name("TestCase") Supplier testCaseSupp @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("E Test", List.of(DataType.INTEGER), () -> { - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")), - "LiteralsEvaluator[lit=2.718281828459045]", - DataType.DOUBLE, - equalTo(Math.E) - ); - }))); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + List.of( + new TestCaseSupplier( + "E Test", + List.of(), + () -> new TestCaseSupplier.TestCase( + List.of(), + "LiteralsEvaluator[lit=2.718281828459045]", + DataType.DOUBLE, + equalTo(Math.E) + ) + ) + ), + (v, p) -> "" + ); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java index d42f4ffde0609..bc5faf1b2560d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java @@ -75,7 +75,7 @@ public static Iterable parameters() { suppliers = anyNullIsNull(true, suppliers); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java index 44ad4547481d6..7942320656f3f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java @@ -124,7 +124,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java index 671cffe9e7f9e..0ee277dbcadb2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java @@ -187,11 +187,7 @@ public static Iterable parameters() { ) ); - // Add null cases before the rest of the error cases, so messages are correct. - suppliers = anyNullIsNull(true, suppliers); - - // Negative cases - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java index e93bc7b43d98e..79742952dbf59 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java @@ -29,14 +29,22 @@ public PiTests(@Name("TestCase") Supplier testCaseSup @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Pi Test", List.of(DataType.INTEGER), () -> { - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")), - "LiteralsEvaluator[lit=3.141592653589793]", - DataType.DOUBLE, - equalTo(Math.PI) - ); - }))); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + List.of( + new TestCaseSupplier( + "Pi Test", + List.of(), + () -> new TestCaseSupplier.TestCase( + List.of(), + "LiteralsEvaluator[lit=3.141592653589793]", + DataType.DOUBLE, + equalTo(Math.PI) + ) + ) + ), + (v, p) -> "numeric" + ); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java index 9d8b87bab8878..2fc139a5458c3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java @@ -77,7 +77,7 @@ public static Iterable parameters() { ) ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java index 8c612e5e664e0..4bf1351969d79 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java @@ -72,7 +72,7 @@ public static Iterable parameters() { suppliers = anyNullIsNull(true, suppliers); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java index 23f2adc6c02e0..7cba5d6d57d45 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java @@ -109,7 +109,7 @@ public static Iterable parameters() { "Line -1:-1: java.lang.ArithmeticException: Square root of negative" ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java index 1a622b25b3353..40e66333f953e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java @@ -29,14 +29,22 @@ public TauTests(@Name("TestCase") Supplier testCaseSu @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Tau Test", List.of(DataType.INTEGER), () -> { - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")), - "LiteralsEvaluator[lit=6.283185307179586]", - DataType.DOUBLE, - equalTo(Tau.TAU) - ); - }))); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + List.of( + new TestCaseSupplier( + "Tau Test", + List.of(), + () -> new TestCaseSupplier.TestCase( + List.of(), + "LiteralsEvaluator[lit=6.283185307179586]", + DataType.DOUBLE, + equalTo(Tau.TAU) + ) + ) + ), + (v, p) -> "numeric" + ); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java index 156fc4bfe7c36..0c905b28ac931 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java @@ -32,22 +32,20 @@ public MvPSeriesWeightedSumTests(@Name("TestCase") Supplier parameters() { List cases = new ArrayList<>(); + doubles(cases); - cases = randomizeBytesRefsOffset(cases); - cases = anyNullIsNull( - cases, + return parameterSuppliersFromTypedDataWithDefaultChecks( (nullPosition, nullValueDataType, original) -> nullValueDataType == DataType.NULL ? DataType.NULL : original.expectedType(), (nullPosition, nullData, original) -> { if (nullData.isForceLiteral()) { return equalTo("LiteralsEvaluator[lit=null]"); } return nullData.type() == DataType.NULL ? equalTo("LiteralsEvaluator[lit=null]") : original; - } + }, + cases, + (valid, position) -> "double" ); - cases = errorsForCasesWithoutExamples(cases, (valid, position) -> "double"); - - return parameterSuppliersFromTypedData(cases); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java index e5f240c811bd0..5d21c135e467e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java @@ -18,9 +18,11 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matchers; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.function.Supplier; @@ -42,6 +44,12 @@ public static Iterable parameters() { doubles(suppliers); bytesRefs(suppliers); nulls(suppliers); + + suppliers = anyNullIsNull( + suppliers, + (nullPosition, nullValueDataType, original) -> nullPosition == 0 ? nullValueDataType : original.expectedType(), + (nullPosition, nullData, original) -> nullData.isForceLiteral() ? Matchers.equalTo("LiteralsEvaluator[lit=null]") : original + ); return parameterSuppliersFromTypedData(suppliers); } @@ -143,75 +151,22 @@ private static void doubles(List suppliers) { } private static void bytesRefs(List suppliers) { - suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD, DataType.KEYWORD), () -> { - List field = randomList(1, 10, () -> randomLiteral(DataType.KEYWORD).value()); - BytesRef order = new BytesRef("DESC"); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(field, DataType.KEYWORD, "field"), - new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() - ), - "MvSortBytesRef[field=Attribute[channel=0], order=false]", - DataType.KEYWORD, - equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList()) - ); - })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT, DataType.KEYWORD), () -> { - List field = randomList(1, 10, () -> randomLiteral(DataType.TEXT).value()); - BytesRef order = new BytesRef("ASC"); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(field, DataType.TEXT, "field"), - new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() - ), - "MvSortBytesRef[field=Attribute[channel=0], order=true]", - DataType.TEXT, - equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList()) - ); - })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.KEYWORD), () -> { - List field = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); - BytesRef order = new BytesRef("ASC"); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(field, DataType.SEMANTIC_TEXT, "field"), - new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() - ), - "MvSortBytesRef[field=Attribute[channel=0], order=true]", - DataType.SEMANTIC_TEXT, - equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList()) - ); - })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.KEYWORD), () -> { - List field = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); - BytesRef order = new BytesRef("DESC"); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(field, DataType.IP, "field"), - new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() - ), - "MvSortBytesRef[field=Attribute[channel=0], order=false]", - DataType.IP, - equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList()) - ); - })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.VERSION, DataType.KEYWORD), () -> { - List field = randomList(1, 10, () -> randomLiteral(DataType.VERSION).value()); - BytesRef order = new BytesRef("ASC"); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(field, DataType.VERSION, "field"), - new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() - ), - "MvSortBytesRef[field=Attribute[channel=0], order=true]", - DataType.VERSION, - equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList()) - ); - })); + for (DataType type : List.of(DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT, DataType.IP, DataType.VERSION)) { + suppliers.add(new TestCaseSupplier(List.of(type, DataType.KEYWORD), () -> { + List field = randomList(1, 10, () -> (BytesRef) randomLiteral(type).value()); + boolean order = randomBoolean(); + var sortedData = field.stream().sorted(order ? Comparator.naturalOrder() : Comparator.reverseOrder()).toList(); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field, type, "field"), + new TestCaseSupplier.TypedData(new BytesRef(order ? "ASC" : "DESC"), DataType.KEYWORD, "order").forceLiteral() + ), + "MvSortBytesRef[field=Attribute[channel=0], order=" + order + "]", + type, + equalTo(sortedData.size() == 1 ? sortedData.get(0) : sortedData) + ); + })); + } } private static void nulls(List suppliers) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java index 26d113359a6f8..89b148144fc83 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java @@ -65,7 +65,7 @@ public static Iterable parameters() { data.add(asLongUnsigned(UNSIGNED_LONG_MAX)); return data; })); - return parameterSuppliersFromTypedData(anyNullIsNull(false, cases)); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases, (v, p) -> "numeric"); } private static TestCaseSupplier arithmeticExceptionCase(DataType dataType, Supplier dataSupplier) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java index ac060b9454564..e58819beb803c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java @@ -76,8 +76,6 @@ public static Iterable parameters() { } } - suppliers = anyNullIsNull(true, suppliers); - // Here follows some non-randomized examples that we want to cover on every run suppliers.add(supplier("a tiger", "a t", null, 1)); suppliers.add(supplier("a tiger", "a", null, 1)); @@ -128,7 +126,7 @@ public static Iterable parameters() { suppliers.add(supplier("🐱Meow!🐶Woof!", "Meow!🐶Woof!", 0, 2)); suppliers.add(supplier("🐱Meow!🐶Woof!", "eow!🐶Woof!", 0, 3)); - return parameterSuppliersFromTypedData(suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java index 8b4ea066fdccb..5eb654b0d8235 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java @@ -122,13 +122,11 @@ public static Iterable parameters() { .withFoldingException(IllegalArgumentException.class, "Number parameter cannot be negative, found [" + number + "]"); })); - cases = anyNullIsNull(true, cases); - cases = errorsForCasesWithoutExamples(cases, (v, p) -> switch (p) { + return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> switch (p) { case 0 -> "string"; case 1 -> "integer"; default -> ""; }); - return parameterSuppliersFromTypedData(cases); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java index 58d52cc02b548..397fb8064626c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java @@ -39,7 +39,7 @@ public static Iterable parameters() { } } - return parameterSuppliersFromTypedData(suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java index aa4c037e5e961..62cb0bd868ee5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java @@ -35,7 +35,6 @@ import static org.elasticsearch.xpack.esql.core.util.DateUtils.asMillis; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; @@ -247,33 +246,7 @@ public static Iterable parameters() { ); }).toList()); - // Datetime tests are split in two, depending on their permissiveness of null-injection, which cannot happen "automatically" for - // Datetime + Period/Duration, since the expression will take the non-null arg's type. - suppliers = anyNullIsNull( - suppliers, - (nullPosition, nullType, original) -> original.expectedType(), - (nullPosition, nullData, original) -> nullData.isForceLiteral() ? equalTo("LiteralsEvaluator[lit=null]") : original - ); - suppliers = errorsForCasesWithoutExamples(suppliers, AddTests::addErrorMessageString); - - // Cases that should generate warnings - suppliers.add(new TestCaseSupplier("MV", List.of(DataType.INTEGER, DataType.INTEGER), () -> { - // Ensure we don't have an overflow - int rhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1); - int lhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1); - int lhs2 = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(List.of(lhs, lhs2), DataType.INTEGER, "lhs"), - new TestCaseSupplier.TypedData(rhs, DataType.INTEGER, "rhs") - ), - "AddIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", - DataType.INTEGER, - is(nullValue()) - ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") - .withWarning("Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value"); - })); - // exact math arithmetic exceptions + // Exact math arithmetic exceptions suppliers.add( arithmeticExceptionOverflowCase( DataType.INTEGER, @@ -315,6 +288,16 @@ public static Iterable parameters() { ) ); + // Datetime tests are split in two, depending on their permissiveness of null-injection, which cannot happen "automatically" for + // Datetime + Period/Duration, since the expression will take the non-null arg's type. + suppliers = anyNullIsNull( + suppliers, + (nullPosition, nullType, original) -> original.expectedType(), + (nullPosition, nullData, original) -> nullData.isForceLiteral() ? equalTo("LiteralsEvaluator[lit=null]") : original + ); + suppliers = errorsForCasesWithoutExamples(suppliers, AddTests::addErrorMessageString); + + // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial return parameterSuppliersFromTypedData(suppliers); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java index c6b607ded7999..70536cd0036e8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java @@ -89,8 +89,6 @@ public static Iterable parameters() { ) ); - suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), DivTests::divErrorMessageString); - // Divide by zero cases - all of these should warn and return null TestCaseSupplier.NumericTypeTestConfigs typeStuff = new TestCaseSupplier.NumericTypeTestConfigs<>( new TestCaseSupplier.NumericTypeTestConfig<>( @@ -163,6 +161,9 @@ public static Iterable parameters() { ) ); + suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), DivTests::divErrorMessageString); + + // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial return parameterSuppliersFromTypedData(suppliers); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java index 65335cdeeb4e1..2ef1cbe89e6a2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java @@ -76,8 +76,6 @@ public static Iterable parameters() { ) ); - suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), ModTests::modErrorMessageString); - // Divide by zero cases - all of these should warn and return null TestCaseSupplier.NumericTypeTestConfigs typeStuff = new TestCaseSupplier.NumericTypeTestConfigs<>( new TestCaseSupplier.NumericTypeTestConfig<>( @@ -150,6 +148,9 @@ public static Iterable parameters() { ) ); + suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), ModTests::modErrorMessageString); + + // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial return parameterSuppliersFromTypedData(suppliers); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java index a2ca331cb261e..12b685eaeeb8d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.function.Supplier; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned; @@ -124,7 +125,20 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers)); + suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), MulTests::mulErrorMessageString); + + // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial + return parameterSuppliersFromTypedData(suppliers); + } + + private static String mulErrorMessageString(boolean includeOrdinal, List> validPerPosition, List types) { + try { + return typeErrorMessage(includeOrdinal, validPerPosition, types, (a, b) -> "numeric"); + } catch (IllegalStateException e) { + // This means all the positional args were okay, so the expected error is from the combination + return "[*] has arguments with incompatible types [" + types.get(0).typeName() + "] and [" + types.get(1).typeName() + "]"; + + } } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java index 3a7d4e6482d72..600db229e90a3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java @@ -24,6 +24,7 @@ import java.time.ZonedDateTime; import java.time.temporal.TemporalAmount; import java.util.List; +import java.util.Set; import java.util.function.BinaryOperator; import java.util.function.Supplier; import java.util.function.ToLongBiFunction; @@ -33,8 +34,6 @@ import static org.elasticsearch.xpack.esql.core.util.DateUtils.asMillis; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.ZERO_AS_UNSIGNED_LONG; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; public class SubTests extends AbstractScalarFunctionTestCase { @@ -245,7 +244,8 @@ public static Iterable parameters() { "SubUnsignedLongsEvaluator" ) ); - suppliers = anyNullIsNull(suppliers, (nullPosition, nullValueDataType, original) -> { + + suppliers = errorsForCasesWithoutExamples(anyNullIsNull(suppliers, (nullPosition, nullValueDataType, original) -> { if (nullValueDataType == DataType.NULL) { return original.getData().get(nullPosition == 0 ? 1 : 0).type(); } @@ -255,28 +255,30 @@ public static Iterable parameters() { return equalTo("LiteralsEvaluator[lit=null]"); } return original; - }); - - suppliers.add(new TestCaseSupplier("MV", List.of(DataType.INTEGER, DataType.INTEGER), () -> { - // Ensure we don't have an overflow - int rhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1); - int lhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1); - int lhs2 = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(List.of(lhs, lhs2), DataType.INTEGER, "lhs"), - new TestCaseSupplier.TypedData(rhs, DataType.INTEGER, "rhs") - ), - "SubIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", - DataType.INTEGER, - is(nullValue()) - ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") - .withWarning("Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value"); - })); + }), SubTests::subErrorMessageString); + // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial return parameterSuppliersFromTypedData(suppliers); } + private static String subErrorMessageString(boolean includeOrdinal, List> validPerPosition, List types) { + if (types.get(1) == DataType.DATETIME) { + if (types.get(0).isNumeric() || DataType.isMillisOrNanos(types.get(0))) { + return "[-] has arguments with incompatible types [" + types.get(0).typeName() + "] and [datetime]"; + } + if (DataType.isNull(types.get(0))) { + return "[-] arguments are in unsupported order: cannot subtract a [DATETIME] value [datetime] from a [NULL] amount [null]"; + } + } + + try { + return typeErrorMessage(includeOrdinal, validPerPosition, types, (a, b) -> "date_nanos, datetime or numeric"); + } catch (IllegalStateException e) { + // This means all the positional args were okay, so the expected error is from the combination + return "[-] has arguments with incompatible types [" + types.get(0).typeName() + "] and [" + types.get(1).typeName() + "]"; + } + } + @Override protected Expression build(Source source, List args) { return new Sub(source, args.get(0), args.get(1)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java index 8e46f7d28540d..4b9f21187f425 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java @@ -236,7 +236,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java index f00e8a19f9dac..45cfef518d39a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java @@ -158,7 +158,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java index 169f1886af47f..6ab55ddbcf403 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java @@ -172,7 +172,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java index 7eccc2c8cc86b..8d48d5812d9da 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java @@ -158,7 +158,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java index a17be8aa97978..f5f4bae5b2afe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java @@ -172,7 +172,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java index 296b4b37cfa73..95001366f1cad 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java @@ -232,7 +232,8 @@ public static Iterable parameters() { false ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override From cbc39aa11c9a58bf6922f3d6c7a8519efdbc1353 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Mon, 13 Jan 2025 12:05:03 -0500 Subject: [PATCH 19/44] Fix CrossClusterEsqlRCSEnrichUnavailableRemotesIT failing tests Fixes https://github.com/elastic/elasticsearch/issues/119910 (#119977) --- ...rossClusterEsqlRCS1EnrichUnavailableRemotesIT.java | 11 ++++++++--- ...rossClusterEsqlRCS2EnrichUnavailableRemotesIT.java | 10 +++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java index 47c7ac8241fcd..19d742e32aa08 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java @@ -24,11 +24,13 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; public class CrossClusterEsqlRCS1EnrichUnavailableRemotesIT extends AbstractRemoteClusterSecurityTestCase { private static final AtomicBoolean SSL_ENABLED_REF = new AtomicBoolean(); @@ -185,8 +187,7 @@ private void esqlEnrichWithSkipUnavailableTrue() throws Exception { Map failuresMap = (Map) remoteClusterFailures.get(0); Map reason = (Map) failuresMap.get("reason"); - assertThat(reason.get("type").toString(), equalTo("connect_transport_exception")); - assertThat(reason.get("reason").toString(), containsString("Unable to connect to [my_remote_cluster]")); + assertThat(reason.get("type").toString(), oneOf("node_disconnected_exception", "connect_transport_exception")); } finally { fulfillingCluster.start(); closeFulfillingClusterClient(); @@ -202,7 +203,11 @@ private void esqlEnrichWithSkipUnavailableFalse() throws Exception { String query = "FROM to-be-enr*,my_remote_cluster:to-be-enr* | ENRICH " + randomFrom(modes) + ":employees-policy | LIMIT 10"; ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(query))); - assertThat(ex.getMessage(), containsString("connect_transport_exception")); + + assertThat( + ex.getMessage(), + anyOf(containsString("connect_transport_exception"), containsString("node_disconnected_exception")) + ); } finally { fulfillingCluster.start(); closeFulfillingClusterClient(); diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java index da59e3c772736..3f068cf3a04cc 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java @@ -26,11 +26,13 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; public class CrossClusterEsqlRCS2EnrichUnavailableRemotesIT extends AbstractRemoteClusterSecurityTestCase { private static final AtomicReference> API_KEY_MAP_REF = new AtomicReference<>(); @@ -205,8 +207,7 @@ private void esqlEnrichWithSkipUnavailableTrue() throws Exception { Map failuresMap = (Map) remoteClusterFailures.get(0); Map reason = (Map) failuresMap.get("reason"); - assertThat(reason.get("type").toString(), equalTo("connect_transport_exception")); - assertThat(reason.get("reason").toString(), containsString("Unable to connect to [my_remote_cluster]")); + assertThat(reason.get("type").toString(), oneOf("node_disconnected_exception", "connect_transport_exception")); } finally { fulfillingCluster.start(); closeFulfillingClusterClient(); @@ -222,7 +223,10 @@ private void esqlEnrichWithSkipUnavailableFalse() throws Exception { String query = "FROM to-be-enr*,my_remote_cluster:to-be-enr* | ENRICH " + randomFrom(modes) + ":employees-policy | LIMIT 10"; ResponseException ex = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(esqlRequest(query))); - assertThat(ex.getMessage(), containsString("connect_transport_exception")); + assertThat( + ex.getMessage(), + anyOf(containsString("connect_transport_exception"), containsString("node_disconnected_exception")) + ); } finally { fulfillingCluster.start(); closeFulfillingClusterClient(); From f854006cf2fe246cdf312658278d79e328b56242 Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 14 Jan 2025 01:13:42 +0800 Subject: [PATCH 20/44] [Inference API] Remove alibabacloud inference unused code (#119826) * remove unused code * add changelog * Revert "add changelog" This reverts commit 9edf46fb4122090629342292ea4bfaea4cd4f3f7. --------- Co-authored-by: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> --- ...aCloudSearchCompletionServiceSettingsTests.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettingsTests.java index 3a5f56c0b4247..167a3f2688292 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettingsTests.java @@ -8,9 +8,7 @@ package org.elasticsearch.xpack.inference.services.alibabacloudsearch.completion; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.inference.services.ServiceFields; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettings; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettingsTests; import org.hamcrest.MatcherAssert; @@ -29,10 +27,6 @@ public static AlibabaCloudSearchCompletionServiceSettings createRandom() { } public void testFromMap() { - var url = "https://www.abc.com"; - var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); - var dims = 1536; - var maxInputTokens = 512; var model = "model"; var host = "host"; var workspaceName = "default"; @@ -40,14 +34,6 @@ public void testFromMap() { var serviceSettings = AlibabaCloudSearchCompletionServiceSettings.fromMap( new HashMap<>( Map.of( - ServiceFields.URL, - url, - ServiceFields.SIMILARITY, - similarity, - ServiceFields.DIMENSIONS, - dims, - ServiceFields.MAX_INPUT_TOKENS, - maxInputTokens, AlibabaCloudSearchServiceSettings.HOST, host, AlibabaCloudSearchServiceSettings.SERVICE_ID, From 2f944bd4c77fef6ef63e6f517be9a6b02c8e3920 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Mon, 13 Jan 2025 13:19:51 -0500 Subject: [PATCH 21/44] Unmute ShardBulkInferenceActionFilterIT (#119986) --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index cecff79b002a2..2ede31c946207 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -227,9 +227,6 @@ tests: - class: org.elasticsearch.search.profile.dfs.DfsProfilerIT method: testProfileDfs issue: https://github.com/elastic/elasticsearch/issues/119711 -- class: org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilterIT - method: testBulkOperations {p0=false} - issue: https://github.com/elastic/elasticsearch/issues/119901 - class: org.elasticsearch.xpack.inference.InferenceCrudIT method: testGetServicesWithCompletionTaskType issue: https://github.com/elastic/elasticsearch/issues/119959 From 264d1c29d42f7d58f30c44d75e14ba923bd10ac6 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Mon, 13 Jan 2025 13:29:48 -0500 Subject: [PATCH 22/44] Update Text Similarity Reranker to Properly Handle Aliases (#120062) --- docs/changelog/120062.yaml | 6 ++ .../rank/feature/RankFeatureShardPhase.java | 2 +- .../xpack/inference/InferenceFeatures.java | 3 +- .../TextSimilarityRankRetrieverBuilder.java | 4 + .../70_text_similarity_rank_retriever.yml | 79 +++++++++++++++++++ 5 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/120062.yaml diff --git a/docs/changelog/120062.yaml b/docs/changelog/120062.yaml new file mode 100644 index 0000000000000..42e8d97f17444 --- /dev/null +++ b/docs/changelog/120062.yaml @@ -0,0 +1,6 @@ +pr: 120062 +summary: Update Text Similarity Reranker to Properly Handle Aliases +area: Ranking +type: bug +issues: + - 119617 diff --git a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java index e64bbe3c39d79..4374c06da365d 100644 --- a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java +++ b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java @@ -83,7 +83,7 @@ public static void processFetch(SearchContext searchContext) { // FetchSearchResult#shardResult() SearchHits hits = fetchSearchResult.hits(); RankFeatureShardResult featureRankShardResult = (RankFeatureShardResult) rankFeaturePhaseRankShardContext - .buildRankFeatureShardResult(hits, searchContext.shardTarget().getShardId().id()); + .buildRankFeatureShardResult(hits, searchContext.request().shardRequestIndex()); // save the result in the search context // need to add profiling info as well available from fetch if (featureRankShardResult != null) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index cf75c1bfe640e..580e5dd4bf8a1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -52,7 +52,8 @@ public Set getTestFeatures() { SEMANTIC_MATCH_QUERY_REWRITE_INTERCEPTION_SUPPORTED, SEMANTIC_SPARSE_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED, SemanticInferenceMetadataFieldsMapper.EXPLICIT_NULL_FIXES, - SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED + SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED, + TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_ALIAS_HANDLING_FIX ); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java index 1b6fb55659a87..bd4c9c1a8be42 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java @@ -44,6 +44,10 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder "text_similarity_reranker_retriever_composition_supported", true ); + public static final NodeFeature TEXT_SIMILARITY_RERANKER_ALIAS_HANDLING_FIX = new NodeFeature( + "text_similarity_reranker_alias_handling_fix", + true + ); public static final ParseField RETRIEVER_FIELD = new ParseField("retriever"); public static final ParseField INFERENCE_ID_FIELD = new ParseField("inference_id"); diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml index 2980d42d22c3e..a3c55dfddf611 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml @@ -216,3 +216,82 @@ setup: - close_to: { hits.hits.0._explanation.value: { value: 0.4, error: 0.000001 } } - match: {hits.hits.0._explanation.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[text\\].*/" } - match: {hits.hits.0._explanation.details.0.description: "/weight.*science.*/" } + +--- +"text similarity reranker properly handles aliases": + - requires: + cluster_features: "text_similarity_reranker_alias_handling_fix" + reason: Test for alias handling fix + + # Create an empty index that will have an earlier shard index than the index with the desired result when referenced + # via the alias + - do: + indices.create: + index: first-test-index + body: + mappings: + properties: + text: + type: text + topic: + type: keyword + subtopic: + type: keyword + + - do: + indices.create: + index: second-test-index + body: + settings: + number_of_shards: 2 + number_of_replicas: 0 + mappings: + properties: + text: + type: text + topic: + type: keyword + subtopic: + type: keyword + + - do: + indices.put_alias: + index: first-test-index + name: test-alias + + - do: + indices.put_alias: + index: second-test-index + name: test-alias + + - do: + index: + index: second-test-index + id: doc_1 + body: + text: "As seen from Earth, a solar eclipse happens when the Moon is directly between the Earth and the Sun." + topic: [ "science" ] + subtopic: [ "technology" ] + refresh: true + + - do: + search: + index: test-alias + body: + track_total_hits: true + retriever: + text_similarity_reranker: + retriever: + standard: + query: + term: + topic: "science" + rank_window_size: 10 + inference_id: my-rerank-model + inference_text: "How often does the moon hide the sun?" + field: text + size: 10 + + - match: { hits.total.value: 1 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: "doc_1" } From f5ac68df95ccb712053355fdf54645edf6a00b19 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Mon, 13 Jan 2025 22:18:47 +0100 Subject: [PATCH 23/44] ESQL: Document support for semantic_text field mapping (#120052) * Document support for semantic_text field mapping * Address review comments --- docs/reference/esql/esql-limitations.asciidoc | 4 ++-- docs/reference/esql/functions/description/match.asciidoc | 2 +- docs/reference/esql/functions/kibana/definition/match.json | 2 +- docs/reference/esql/functions/kibana/docs/match.md | 3 ++- .../xpack/esql/expression/function/fulltext/Match.java | 3 ++- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/reference/esql/esql-limitations.asciidoc b/docs/reference/esql/esql-limitations.asciidoc index 8ce8064a8161c..adfd38478ab21 100644 --- a/docs/reference/esql/esql-limitations.asciidoc +++ b/docs/reference/esql/esql-limitations.asciidoc @@ -30,11 +30,11 @@ include::processing-commands/limit.asciidoc[tag=limitation] ** You can use `to_datetime` to cast to millisecond dates to use unsupported functions * `double` (`float`, `half_float`, `scaled_float` are represented as `double`) * `ip` -* `keyword` family including `keyword`, `constant_keyword`, and `wildcard` +* `keyword` <> including `keyword`, `constant_keyword`, and `wildcard` * `int` (`short` and `byte` are represented as `int`) * `long` * `null` -* `text` +* `text` <> including `text`, `semantic_text` and `match_only_text` * experimental:[] `unsigned_long` * `version` * Spatial types diff --git a/docs/reference/esql/functions/description/match.asciidoc b/docs/reference/esql/functions/description/match.asciidoc index 931fd5eb2f94a..0724f0f108e3c 100644 --- a/docs/reference/esql/functions/description/match.asciidoc +++ b/docs/reference/esql/functions/description/match.asciidoc @@ -2,4 +2,4 @@ *Description* -Use `MATCH` to perform a <> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. Match can be used on text fields, as well as other field types like boolean, dates, and numeric types. For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. `MATCH` returns true if the provided query matches the row. +Use `MATCH` to perform a <> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. Match can be used on fields from the text family like <> and <>, as well as other field types like keyword, boolean, dates, and numeric types. For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. `MATCH` returns true if the provided query matches the row. diff --git a/docs/reference/esql/functions/kibana/definition/match.json b/docs/reference/esql/functions/kibana/definition/match.json index d61534da81a6d..eb206cb9ddf4d 100644 --- a/docs/reference/esql/functions/kibana/definition/match.json +++ b/docs/reference/esql/functions/kibana/definition/match.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "match", - "description" : "Use `MATCH` to perform a <> on the specified field.\nUsing `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.\n\nMatch can be used on text fields, as well as other field types like boolean, dates, and numeric types.\n\nFor a simplified syntax, you can use the <> `:` operator instead of `MATCH`.\n\n`MATCH` returns true if the provided query matches the row.", + "description" : "Use `MATCH` to perform a <> on the specified field.\nUsing `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.\n\nMatch can be used on fields from the text family like <> and <>,\nas well as other field types like keyword, boolean, dates, and numeric types.\n\nFor a simplified syntax, you can use the <> `:` operator instead of `MATCH`.\n\n`MATCH` returns true if the provided query matches the row.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/docs/match.md b/docs/reference/esql/functions/kibana/docs/match.md index 72258a1682936..80bf84351c188 100644 --- a/docs/reference/esql/functions/kibana/docs/match.md +++ b/docs/reference/esql/functions/kibana/docs/match.md @@ -6,7 +6,8 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ Use `MATCH` to perform a <> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. -Match can be used on text fields, as well as other field types like boolean, dates, and numeric types. +Match can be used on fields from the text family like <> and <>, +as well as other field types like keyword, boolean, dates, and numeric types. For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java index 7552b100119f0..93f23d2f7ad08 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java @@ -103,7 +103,8 @@ public class Match extends FullTextFunction implements PostOptimizationVerificat Use `MATCH` to perform a <> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. - Match can be used on text fields, as well as other field types like boolean, dates, and numeric types. + Match can be used on fields from the text family like <> and <>, + as well as other field types like keyword, boolean, dates, and numeric types. For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. From 5167497f50b1892bd3fc50b15c57dff643eff33a Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 14 Jan 2025 06:54:22 +0800 Subject: [PATCH 24/44] [Inference API] fix spell words: covertToString to convertToString (#119922) --- docs/changelog/119922.yaml | 5 +++++ .../AlibabaCloudSearchEmbeddingsRequestEntity.java | 4 ++-- .../AlibabaCloudSearchSparseRequestEntity.java | 2 +- .../request/cohere/CohereEmbeddingsRequestEntity.java | 4 ++-- .../AlibabaCloudSearchEmbeddingsRequestEntityTests.java | 2 +- .../request/cohere/CohereEmbeddingsRequestEntityTests.java | 5 ++++- 6 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 docs/changelog/119922.yaml diff --git a/docs/changelog/119922.yaml b/docs/changelog/119922.yaml new file mode 100644 index 0000000000000..2fc9d9529c968 --- /dev/null +++ b/docs/changelog/119922.yaml @@ -0,0 +1,5 @@ +pr: 119922 +summary: "[Inference API] fix spell words: covertToString to convertToString" +area: Machine Learning +type: enhancement +issues: [] diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java index 1fc61d3331d20..0804cb332b15d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java @@ -39,7 +39,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(TEXTS_FIELD, input); - String inputType = covertToString(taskSettings.getInputType()); + String inputType = convertToString(taskSettings.getInputType()); if (inputType != null) { builder.field(INPUT_TYPE_FIELD, inputType); } @@ -49,7 +49,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } // default for testing - static String covertToString(InputType inputType) { + static String convertToString(InputType inputType) { if (inputType == null) { return null; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java index 8fae9408b860d..22676c11dc548 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java @@ -34,7 +34,7 @@ public record AlibabaCloudSearchSparseRequestEntity(List input, AlibabaC public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(TEXTS_FIELD, input); - String inputType = AlibabaCloudSearchEmbeddingsRequestEntity.covertToString(taskSettings.getInputType()); + String inputType = AlibabaCloudSearchEmbeddingsRequestEntity.convertToString(taskSettings.getInputType()); if (inputType != null) { builder.field(INPUT_TYPE_FIELD, inputType); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java index 63cc5c3cb7261..a972cbbac959d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java @@ -51,7 +51,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } if (taskSettings.getInputType() != null) { - builder.field(INPUT_TYPE_FIELD, covertToString(taskSettings.getInputType())); + builder.field(INPUT_TYPE_FIELD, convertToString(taskSettings.getInputType())); } if (embeddingType != null) { @@ -67,7 +67,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } // default for testing - static String covertToString(InputType inputType) { + static String convertToString(InputType inputType) { return switch (inputType) { case INGEST -> SEARCH_DOCUMENT; case SEARCH -> SEARCH_QUERY; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java index 6aaab219c331d..39eeef3a95520 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java @@ -50,7 +50,7 @@ public void testXContent_WritesNoOptionalFields_WhenTheyAreNotDefined() throws I public void testConvertToString_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() { var thrownException = expectThrows( AssertionError.class, - () -> AlibabaCloudSearchEmbeddingsRequestEntity.covertToString(InputType.UNSPECIFIED) + () -> AlibabaCloudSearchEmbeddingsRequestEntity.convertToString(InputType.UNSPECIFIED) ); MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]")); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java index 0690bf56893ca..8c8aeba4a0a07 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java @@ -84,7 +84,10 @@ public void testXContent_WritesNoOptionalFields_WhenTheyAreNotDefined() throws I } public void testConvertToString_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() { - var thrownException = expectThrows(AssertionError.class, () -> CohereEmbeddingsRequestEntity.covertToString(InputType.UNSPECIFIED)); + var thrownException = expectThrows( + AssertionError.class, + () -> CohereEmbeddingsRequestEntity.convertToString(InputType.UNSPECIFIED) + ); MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]")); } } From 455fde2849cd2bd6c9a38bc21383417194d64205 Mon Sep 17 00:00:00 2001 From: Dianna Hohensee Date: Mon, 13 Jan 2025 19:58:55 -0500 Subject: [PATCH 25/44] Add documentation around desired balance (#119902) Relates ES-10341 --- .../routing/allocation/AllocationService.java | 9 ++---- .../allocation/allocator/DesiredBalance.java | 21 +++++++++++- .../allocator/DesiredBalanceComputer.java | 13 +++++--- .../allocator/DesiredBalanceReconciler.java | 15 +++++++++ .../DesiredBalanceShardsAllocator.java | 32 +++++++++++++++---- .../allocator/PendingListenersQueue.java | 4 +++ .../allocation/allocator/ShardAssignment.java | 14 +++++++- .../allocator/DesiredBalanceTests.java | 10 +++--- 8 files changed, 94 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 5d1e6741c5e22..2f2fd4ef453f6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -158,9 +158,7 @@ public ClusterState applyStartedShards(ClusterState clusterState, List ass * The placeholder value for {@link DesiredBalance} when the node stands down as master. */ public static final DesiredBalance NOT_MASTER = new DesiredBalance(-2, Map.of()); + /** * The starting value for {@link DesiredBalance} when the node becomes the master. */ @@ -57,6 +65,10 @@ public static boolean hasChanges(DesiredBalance a, DesiredBalance b) { return Objects.equals(a.assignments, b.assignments) == false; } + /** + * Returns the sum of shard movements needed to reach the new desired balance. Doesn't count new shard copies as a move, nor removal or + * unassignment of a shard copy. + */ public static int shardMovements(DesiredBalance old, DesiredBalance updated) { var intersection = Sets.intersection(old.assignments().keySet(), updated.assignments().keySet()); int movements = 0; @@ -70,8 +82,15 @@ public static int shardMovements(DesiredBalance old, DesiredBalance updated) { return movements; } + /** + * Returns the number of shard movements needed to reach the new shard assignment. Doesn't count new shard copies as a move, nor removal + * or unassignment of a shard copy. + */ private static int shardMovements(ShardAssignment old, ShardAssignment updated) { - var movements = Math.min(0, old.assigned() - updated.assigned());// compensate newly started shards + // A shard move should retain the same number of assigned nodes, just swap out one node for another. We will compensate for newly + // started shards -- adding a shard copy is not a move -- by initializing the count with a negative value so that incrementing later + // for a new node zeros out. + var movements = Math.min(0, old.assigned() - updated.assigned()); for (String nodeId : updated.nodeIds()) { if (old.nodeIds().contains(nodeId) == false) { movements++; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java index 3b22221ea7db4..03630c284fa30 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java @@ -415,11 +415,14 @@ boolean hasEnoughIterations(int currentIteration) { } private static Map collectShardAssignments(RoutingNodes routingNodes) { - final var entries = routingNodes.getAssignedShards().entrySet(); - assert entries.stream().flatMap(t -> t.getValue().stream()).allMatch(ShardRouting::started) : routingNodes; - final Map res = Maps.newHashMapWithExpectedSize(entries.size()); - for (var shardAndAssignments : entries) { - res.put(shardAndAssignments.getKey(), ShardAssignment.ofAssignedShards(shardAndAssignments.getValue())); + final var allAssignedShards = routingNodes.getAssignedShards().entrySet(); + assert allAssignedShards.stream().flatMap(t -> t.getValue().stream()).allMatch(ShardRouting::started) : routingNodes; + final Map res = Maps.newHashMapWithExpectedSize(allAssignedShards.size()); + for (var shardIdAndShardRoutings : allAssignedShards) { + res.put( + shardIdAndShardRoutings.getKey(), + ShardAssignment.createFromAssignedShardRoutingsList(shardIdAndShardRoutings.getValue()) + ); } return res; } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java index 83b370c1a7928..909a7a7a99a61 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java @@ -54,6 +54,10 @@ public class DesiredBalanceReconciler { private static final Logger logger = LogManager.getLogger(DesiredBalanceReconciler.class); + /** + * The minimum interval that log messages will be written if the number of undesired shard allocations reaches the percentage of total + * shards set by {@link #UNDESIRED_ALLOCATIONS_LOG_THRESHOLD_SETTING}. + */ public static final Setting UNDESIRED_ALLOCATIONS_LOG_INTERVAL_SETTING = Setting.timeSetting( "cluster.routing.allocation.desired_balance.undesired_allocations.log_interval", TimeValue.timeValueHours(1), @@ -62,6 +66,10 @@ public class DesiredBalanceReconciler { Setting.Property.NodeScope ); + /** + * Warning log messages may be periodically written if the number of shards that are on undesired nodes reaches this percentage setting. + * Works together with {@link #UNDESIRED_ALLOCATIONS_LOG_INTERVAL_SETTING} to log on a periodic basis. + */ public static final Setting UNDESIRED_ALLOCATIONS_LOG_THRESHOLD_SETTING = Setting.doubleSetting( "cluster.routing.allocation.desired_balance.undesired_allocations.threshold", 0.1, @@ -96,6 +104,13 @@ public DesiredBalanceReconciler( this.nodeAllocationStatsAndWeightsCalculator = nodeAllocationStatsAndWeightsCalculator; } + /** + * Applies a desired shard allocation to the routing table by initializing and relocating shards in the cluster state. + * + * @param desiredBalance The new desired cluster shard allocation + * @param allocation Cluster state information with which to make decisions, contains routing table metadata that will be modified to + * reach the given desired balance. + */ public void reconcile(DesiredBalance desiredBalance, RoutingAllocation allocation) { var nodeIds = allocation.routingNodes().getAllNodeIds(); allocationOrdering.retainNodes(nodeIds); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java index 5be26f0b3e8c7..2c73a27ad3418 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.AllocationService.RerouteStrategy; import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; @@ -56,11 +57,30 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator { private final ShardsAllocator delegateAllocator; private final ThreadPool threadPool; + /** + * This is a callback to run {@link AllocationService#executeWithRoutingAllocation(ClusterState, String, RerouteStrategy)}, which + * produces a new ClusterState with the changes made by {@link DesiredBalanceReconciler#reconcile}. The {@link RerouteStrategy} provided + * to the callback calls into {@link #desiredBalanceReconciler} for the changes. The {@link #masterServiceTaskQueue} will publish the + * new cluster state after the cluster state is constructed by the {@link ReconcileDesiredBalanceExecutor}. + */ private final DesiredBalanceReconcilerAction reconciler; private final DesiredBalanceComputer desiredBalanceComputer; + /** + * Reconciliation ({@link DesiredBalanceReconciler#reconcile(DesiredBalance, RoutingAllocation)}) takes the {@link DesiredBalance} + * output of {@link DesiredBalanceComputer#compute} and identifies how shards need to be added, moved or removed to go from the current + * cluster shard allocation to the new desired allocation. + */ private final DesiredBalanceReconciler desiredBalanceReconciler; private final ContinuousComputation desiredBalanceComputation; - private final PendingListenersQueue queue; + /** + * Saves and runs listeners after DesiredBalance computations complete. + */ + private final PendingListenersQueue pendingListenersQueue; + /** + * Each reroute request gets assigned a monotonically increasing sequence number. Many reroute requests may arrive before the balancer + * asynchronously runs a computation. The balancer will use the latest request and save this sequence number to track back to the + * request. + */ private final AtomicLong indexGenerator = new AtomicLong(-1); private final ConcurrentLinkedQueue> pendingDesiredBalanceMoves = new ConcurrentLinkedQueue<>(); private final MasterServiceTaskQueue masterServiceTaskQueue; @@ -199,7 +219,7 @@ public String toString() { return "DesiredBalanceShardsAllocator#allocate"; } }; - this.queue = new PendingListenersQueue(); + this.pendingListenersQueue = new PendingListenersQueue(); this.masterServiceTaskQueue = clusterService.createTaskQueue( "reconcile-desired-balance", Priority.URGENT, @@ -235,7 +255,7 @@ public void allocate(RoutingAllocation allocation, ActionListener listener var index = indexGenerator.incrementAndGet(); logger.debug("Executing allocate for [{}]", index); - queue.add(index, listener); + pendingListenersQueue.add(index, listener); // This can only run on master, so unset not-master if exists if (currentDesiredBalanceRef.compareAndSet(DesiredBalance.NOT_MASTER, DesiredBalance.BECOME_MASTER_INITIAL)) { logger.debug("initialized desired balance for becoming master"); @@ -378,7 +398,7 @@ public DesiredBalanceStats getStats() { private void onNoLongerMaster() { if (indexGenerator.getAndSet(-1) != -1) { currentDesiredBalanceRef.set(DesiredBalance.NOT_MASTER); - queue.completeAllAsNotMaster(); + pendingListenersQueue.completeAllAsNotMaster(); pendingDesiredBalanceMoves.clear(); desiredBalanceReconciler.clear(); desiredBalanceMetrics.zeroAllMetrics(); @@ -428,7 +448,7 @@ private ClusterState applyBalance( batchExecutionContext.initialState(), createReconcileAllocationAction(latest.getTask().desiredBalance) ); - latest.success(() -> queue.complete(latest.getTask().desiredBalance.lastConvergedIndex())); + latest.success(() -> pendingListenersQueue.complete(latest.getTask().desiredBalance.lastConvergedIndex())); return newState; } } @@ -447,7 +467,7 @@ private static void discardSupersededTasks( // only for tests - in production, this happens after reconciliation protected final void completeToLastConvergedIndex() { - queue.complete(currentDesiredBalanceRef.get().lastConvergedIndex()); + pendingListenersQueue.complete(currentDesiredBalanceRef.get().lastConvergedIndex()); } private void recordTime(CounterMetric metric, Runnable action) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java index e1b58cf79ac09..5b14277f2c651 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java @@ -24,6 +24,10 @@ import static org.elasticsearch.cluster.service.ClusterApplierService.CLUSTER_UPDATE_THREAD_NAME; import static org.elasticsearch.cluster.service.MasterService.MASTER_UPDATE_THREAD_NAME; +/** + * Registers listeners with an `index` number ({@link #add(long, ActionListener)}) and then completes them whenever the latest index number + * is greater or equal to a listener's index value ({@link #complete(long)}). + */ public class PendingListenersQueue { private static final Logger logger = LogManager.getLogger(PendingListenersQueue.class); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java index 4fb9137cb4544..2bd1b9bb2bb64 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java @@ -17,6 +17,14 @@ import static java.util.Collections.unmodifiableSet; +/** + * Simple shard assignment summary of shard copies for a particular index shard. + * + * @param nodeIds The node IDs of nodes holding a shard copy. + * @param total The total number of shard copies. + * @param unassigned The number of unassigned shard copies. + * @param ignored The number of ignored shard copies. + */ public record ShardAssignment(Set nodeIds, int total, int unassigned, int ignored) { public ShardAssignment { @@ -28,9 +36,13 @@ public int assigned() { return nodeIds.size(); } - public static ShardAssignment ofAssignedShards(List routings) { + /** + * Helper method to instantiate a new ShardAssignment from a given list of ShardRouting instances. Assumes all shards are assigned. + */ + public static ShardAssignment createFromAssignedShardRoutingsList(List routings) { var nodeIds = new LinkedHashSet(); for (ShardRouting routing : routings) { + assert routing.unassignedInfo() == null : "Expected assigned shard copies only, unassigned info: " + routing.unassignedInfo(); nodeIds.add(routing.currentNodeId()); } return new ShardAssignment(unmodifiableSet(nodeIds), routings.size(), 0, 0); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java index 760900817780a..2c15addfe217b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java @@ -46,19 +46,19 @@ public void testShardMovements() { ); assertThat( - "1 shard movements when existing shard is moved and new shard copy is unassigned", + "1 shard movements when an existing shard copy is moved and new shard copy is unassigned", shardMovements(new ShardAssignment(Set.of("a", "b"), 2, 0, 0), new ShardAssignment(Set.of("a", "c"), 3, 1, 0)), equalTo(1) ); assertThat( - "1 shard movement", + "1 shard movement when an existing shard copy is moved", shardMovements(new ShardAssignment(Set.of("a", "b"), 2, 0, 0), new ShardAssignment(Set.of("a", "c"), 2, 0, 0)), equalTo(1) ); assertThat( - "2 shard movement", + "2 shard movements when both shard copies are move to new nodes", shardMovements(new ShardAssignment(Set.of("a", "b"), 2, 0, 0), new ShardAssignment(Set.of("c", "d"), 2, 0, 0)), equalTo(2) ); @@ -77,10 +77,10 @@ public void testShardMovements() { } private static int shardMovements(ShardAssignment old, ShardAssignment updated) { - return DesiredBalance.shardMovements(of(old), of(updated)); + return DesiredBalance.shardMovements(createDesiredBalanceWith(old), createDesiredBalanceWith(updated)); } - private static DesiredBalance of(ShardAssignment assignment) { + private static DesiredBalance createDesiredBalanceWith(ShardAssignment assignment) { return new DesiredBalance(1, Map.of(new ShardId("index", "_na_", 0), assignment)); } } From 4605ecf44de75ff7a5b6ec060b9540ab6e3394ee Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Tue, 14 Jan 2025 12:39:41 +1000 Subject: [PATCH 26/44] Remove trappy timeouts from `GetMappingsRequest` (#120035) Relates #107984 --- .../datastreams/DataStreamIT.java | 8 ++-- .../index/mapper/size/SizeMappingIT.java | 2 +- .../action/IndicesRequestIT.java | 2 +- .../admin/indices/create/CreateIndexIT.java | 6 +-- .../indices/create/CreateSystemIndicesIT.java | 5 ++- .../action/bulk/BulkIntegrationIT.java | 2 +- .../cluster/SimpleClusterStateIT.java | 7 ++- .../gateway/MetadataNodesIT.java | 4 +- .../elasticsearch/index/HiddenIndexIT.java | 2 +- .../index/mapper/DynamicMappingIT.java | 26 ++++++++--- .../mapper/MultiFieldsIntegrationIT.java | 10 ++--- .../indices/IndicesOptionsIntegrationIT.java | 26 +++++------ .../SystemIndexMappingUpdateServiceIT.java | 5 ++- .../mapping/SimpleGetFieldMappingsIT.java | 2 +- .../indices/mapping/SimpleGetMappingsIT.java | 12 ++--- .../mapping/UpdateMappingIntegrationIT.java | 11 +++-- .../mapping/get/GetMappingsRequest.java | 5 ++- .../get/GetMappingsRequestBuilder.java | 5 ++- .../master/info/ClusterInfoRequest.java | 12 +++++ .../client/internal/IndicesAdminClient.java | 4 +- .../admin/indices/RestGetMappingAction.java | 5 +-- .../mapper/FieldFilterMapperPluginTests.java | 8 ++-- .../mapper/NonDynamicFieldMapperTestCase.java | 4 +- .../xpack/ccr/CcrRepositoryIT.java | 2 +- .../xpack/ccr/IndexFollowingIT.java | 23 ++++++++-- .../sourceonly/SourceOnlySnapshotIT.java | 2 +- .../downsample/TransportDownsampleAction.java | 2 +- .../DownsampleActionSingleNodeTests.java | 7 +-- .../action/CreateIndexFromSourceActionIT.java | 5 ++- ...indexDatastreamIndexTransportActionIT.java | 6 ++- .../ml/integration/JobResultsProviderIT.java | 4 +- ...ransportStartDataFrameAnalyticsAction.java | 5 ++- .../xpack/ml/dataframe/DestinationIndex.java | 2 + .../xpack/ml/dataframe/MappingsMerger.java | 4 +- .../job/persistence/JobResultsProvider.java | 6 ++- .../action/TransportPutRollupJobAction.java | 7 +-- .../action/PutJobStateMachineTests.java | 45 ++++++++++++++++--- .../DocumentAndFieldLevelSecurityTests.java | 8 ++-- .../integration/KibanaUserRoleIntegTests.java | 2 +- .../HistoryTemplateHttpMappingsTests.java | 3 +- .../HistoryTemplateTimeMappingsTests.java | 2 +- .../integration/HistoryIntegrationTests.java | 6 +-- 42 files changed, 208 insertions(+), 106 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 4a3dfac36d4ec..2739eb51376ea 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -596,7 +596,7 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { verifyResolvability(dataStreamName, indicesAdmin().prepareGetFieldMappings(dataStreamName), false); verifyResolvability(dataStreamName, indicesAdmin().preparePutMapping(dataStreamName).setSource(""" {"_doc":{"properties": {"my_field":{"type":"keyword"}}}}""", XContentType.JSON), false); - verifyResolvability(dataStreamName, indicesAdmin().prepareGetMappings(dataStreamName), false); + verifyResolvability(dataStreamName, indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, dataStreamName), false); verifyResolvability( dataStreamName, indicesAdmin().prepareUpdateSettings(dataStreamName).setSettings(Settings.builder().put("index.number_of_replicas", 0)), @@ -643,7 +643,7 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { verifyResolvability(wildcardExpression, indicesAdmin().prepareGetFieldMappings(wildcardExpression), false); verifyResolvability(wildcardExpression, indicesAdmin().preparePutMapping(wildcardExpression).setSource(""" {"_doc":{"properties": {"my_field":{"type":"keyword"}}}}""", XContentType.JSON), false); - verifyResolvability(wildcardExpression, indicesAdmin().prepareGetMappings(wildcardExpression), false); + verifyResolvability(wildcardExpression, indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, wildcardExpression), false); verifyResolvability(wildcardExpression, indicesAdmin().prepareGetSettings(wildcardExpression), false); verifyResolvability( wildcardExpression, @@ -1180,7 +1180,7 @@ public void testUpdateMappingViaDataStream() throws Exception { DataStreamTimestampFieldMapper.NAME, Map.of("enabled", true) ); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("logs-foobar").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "logs-foobar").get(); assertThat(getMappingsResponse.getMappings().size(), equalTo(2)); assertThat(getMappingsResponse.getMappings().get(backingIndex1).getSourceAsMap(), equalTo(expectedMapping)); assertThat(getMappingsResponse.getMappings().get(backingIndex2).getSourceAsMap(), equalTo(expectedMapping)); @@ -1195,7 +1195,7 @@ public void testUpdateMappingViaDataStream() throws Exception { .setSource("{\"properties\":{\"my_field\":{\"type\":\"keyword\"}}}", XContentType.JSON) .get(); // The mappings of all backing indices should be updated: - getMappingsResponse = indicesAdmin().prepareGetMappings("logs-foobar").get(); + getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "logs-foobar").get(); assertThat(getMappingsResponse.getMappings().size(), equalTo(2)); assertThat(getMappingsResponse.getMappings().get(backingIndex1).getSourceAsMap(), equalTo(expectedMapping)); assertThat(getMappingsResponse.getMappings().get(backingIndex2).getSourceAsMap(), equalTo(expectedMapping)); diff --git a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java index c2251910c3122..435849821691e 100644 --- a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java +++ b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java @@ -91,7 +91,7 @@ private void assertSizeMappingEnabled(String index, boolean enabled) throws IOEx "Expected size field mapping to be " + (enabled ? "enabled" : "disabled") + " for %s", index ); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(index).get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get(); Map mappingSource = getMappingsResponse.getMappings().get(index).getSourceAsMap(); assertThat(errMsg, mappingSource, hasKey("_size")); String sizeAsString = mappingSource.get("_size").toString(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java index 420f6427a55e1..68e65b16aa3a2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java @@ -519,7 +519,7 @@ public void testDeleteIndex() { public void testGetMappings() { interceptTransportActions(GetMappingsAction.NAME); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(randomIndicesOrAliases()); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(randomIndicesOrAliases()); internalCluster().coordOnlyNodeClient().admin().indices().getMappings(getMappingsRequest).actionGet(); clearInterceptedActions(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index 4c25b3262d559..b14bf38f3cbcc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -120,7 +120,7 @@ public void testNonNestedMappings() throws Exception { ) ); - GetMappingsResponse response = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); MappingMetadata mappings = response.mappings().get("test"); assertNotNull(mappings); @@ -130,7 +130,7 @@ public void testNonNestedMappings() throws Exception { public void testEmptyNestedMappings() throws Exception { assertAcked(prepareCreate("test").setMapping(XContentFactory.jsonBuilder().startObject().endObject())); - GetMappingsResponse response = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); MappingMetadata mappings = response.mappings().get("test"); assertNotNull(mappings); @@ -150,7 +150,7 @@ public void testEmptyMappings() throws Exception { prepareCreate("test").setMapping(XContentFactory.jsonBuilder().startObject().startObject("_doc").endObject().endObject()) ); - GetMappingsResponse response = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); MappingMetadata mappings = response.mappings().get("test"); assertNotNull(mappings); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java index 867ca89f9e7f3..b9dadf86c3345 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java @@ -357,8 +357,9 @@ private void assertHasAliases(Set aliasNames, String name, String primar * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values. */ private void assertMappingsAndSettings(String expectedMappings, String concreteIndex) { - final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME)) - .actionGet(); + final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings( + new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(INDEX_NAME) + ).actionGet(); final Map mappings = getMappingsResponse.getMappings(); assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index e45555b1dec19..2cd319d148321 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -55,7 +55,7 @@ public void testBulkIndexCreatesMapping() throws Exception { bulkBuilder.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, XContentType.JSON); bulkBuilder.get(); assertBusy(() -> { - GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertTrue(mappingsResponse.getMappings().containsKey("logstash-2014.03.30")); }); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java index 7777dd778a6c1..9eed1f757b5b1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java @@ -258,11 +258,14 @@ public void testLargeClusterStatePublishing() throws Exception { .setTimeout(TimeValue.timeValueMinutes(1)) ); ensureGreen(); // wait for green state, so its both green, and there are no more pending events - MappingMetadata masterMappingMetadata = indicesAdmin().prepareGetMappings("test").get().getMappings().get("test"); + MappingMetadata masterMappingMetadata = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test") + .get() + .getMappings() + .get("test"); for (Client client : clients()) { MappingMetadata mappingMetadata = client.admin() .indices() - .prepareGetMappings("test") + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "test") .setLocal(true) .get() .getMappings() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java index d6ccdf3dc0399..256566045c59a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java @@ -115,7 +115,7 @@ public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception { ) .get(); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(index).get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get(); assertNotNull( ((Map) (getMappingsResponse.getMappings().get(index).getSourceAsMap().get("properties"))).get("integer_field") ); @@ -146,7 +146,7 @@ public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception { ) .get(); - getMappingsResponse = indicesAdmin().prepareGetMappings(index).get(); + getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get(); assertNotNull( ((Map) (getMappingsResponse.getMappings().get(index).getSourceAsMap().get("properties"))).get("float_field") ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java index 802ba04375c48..24af560f608d3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java @@ -97,7 +97,7 @@ public void testGlobalTemplatesDoNotApply() { assertAcked(indicesAdmin().prepareCreate("a_hidden_index").setSettings(Settings.builder().put("index.hidden", true).build())); - GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings("a_hidden_index").get(); + GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "a_hidden_index").get(); assertThat(mappingsResponse.mappings().size(), is(1)); MappingMetadata mappingMetadata = mappingsResponse.mappings().get("a_hidden_index"); assertNotNull(mappingMetadata); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java index a37fb25052ac5..06561bc6d4c97 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java @@ -96,7 +96,11 @@ public void testSimpleDynamicMappingsSuccessful() { client().prepareIndex("index").setId("1").setSource("a.x", 1).get(); client().prepareIndex("index").setId("2").setSource("a.y", 2).get(); - Map mappings = indicesAdmin().prepareGetMappings("index").get().mappings().get("index").sourceAsMap(); + Map mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "index") + .get() + .mappings() + .get("index") + .sourceAsMap(); assertTrue(new WriteField("properties.a", () -> mappings).exists()); assertTrue(new WriteField("properties.a.properties.x", () -> mappings).exists()); } @@ -183,7 +187,7 @@ private Map indexConcurrently(int numberOfFieldsToCreate, Settin for (int i = 0; i < numberOfFieldsToCreate; ++i) { assertTrue(client().prepareGet("index", Integer.toString(i)).get().isExists()); } - GetMappingsResponse mappings = indicesAdmin().prepareGetMappings("index").get(); + GetMappingsResponse mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "index").get(); MappingMetadata indexMappings = mappings.getMappings().get("index"); assertNotNull(indexMappings); Map typeMappingsMap = indexMappings.getSourceAsMap(); @@ -214,7 +218,11 @@ public void testConcurrentDynamicMappingsWithConflictingType() throws Throwable for (int i = 0; i < numberOfDocsToCreate; ++i) { assertTrue(client().prepareGet("index", Integer.toString(i)).get().isExists()); } - Map index = indicesAdmin().prepareGetMappings("index").get().getMappings().get("index").getSourceAsMap(); + Map index = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "index") + .get() + .getMappings() + .get("index") + .getSourceAsMap(); for (int i = 0, j = 1; i < numberOfDocsToCreate; i++, j++) { assertThat(new WriteField("properties.field" + i + ".type", () -> index).get(null), is(oneOf("long", "float"))); assertThat(new WriteField("properties.field" + j + ".type", () -> index).get(null), is(oneOf("long", "float"))); @@ -806,7 +814,11 @@ public void testSubobjectsFalseAtRoot() throws Exception { assertEquals(RestStatus.CREATED, indexResponse.status()); assertBusy(() -> { - Map mappings = indicesAdmin().prepareGetMappings("test").get().mappings().get("test").sourceAsMap(); + Map mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test") + .get() + .mappings() + .get("test") + .sourceAsMap(); @SuppressWarnings("unchecked") Map properties = (Map) mappings.get("properties"); assertEquals(4, properties.size()); @@ -851,7 +863,11 @@ public void testSubobjectsFalse() throws Exception { assertEquals(RestStatus.CREATED, indexResponse.status()); assertBusy(() -> { - Map mappings = indicesAdmin().prepareGetMappings("test").get().mappings().get("test").sourceAsMap(); + Map mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test") + .get() + .mappings() + .get("test") + .sourceAsMap(); Map properties = (Map) mappings.get("properties"); Map foo = (Map) properties.get("foo"); properties = (Map) foo.get("properties"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java index 4a7de4b0ebc23..1a51fc12fed8e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java @@ -37,7 +37,7 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase { public void testMultiFields() throws Exception { assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createTypeSource())); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); Map mappingSource = mappingMetadata.sourceAsMap(); @@ -53,7 +53,7 @@ public void testMultiFields() throws Exception { assertAcked(indicesAdmin().preparePutMapping("my-index").setSource(createPutMappingSource())); - getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); mappingSource = mappingMetadata.sourceAsMap(); @@ -74,7 +74,7 @@ public void testMultiFields() throws Exception { public void testGeoPointMultiField() throws Exception { assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createMappingSource("geo_point"))); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); Map mappingSource = mappingMetadata.sourceAsMap(); @@ -102,7 +102,7 @@ public void testGeoPointMultiField() throws Exception { public void testCompletionMultiField() throws Exception { assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createMappingSource("completion"))); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); Map mappingSource = mappingMetadata.sourceAsMap(); @@ -123,7 +123,7 @@ public void testCompletionMultiField() throws Exception { public void testIpMultiField() throws Exception { assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createMappingSource("ip"))); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); Map mappingSource = mappingMetadata.sourceAsMap(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java index 652f4e02ffbce..c53cf3b56f65a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java @@ -537,26 +537,26 @@ public void testPutMapping() throws Exception { } verify(indicesAdmin().preparePutMapping("foo").setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("foo").get().mappings().get("foo"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foo").get().mappings().get("foo"), notNullValue()); verify(indicesAdmin().preparePutMapping("b*").setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("bar").get().mappings().get("bar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "bar").get().mappings().get("bar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue()); verify(indicesAdmin().preparePutMapping("_all").setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("foo").get().mappings().get("foo"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("foobar").get().mappings().get("foobar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("bar").get().mappings().get("bar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foo").get().mappings().get("foo"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foobar").get().mappings().get("foobar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "bar").get().mappings().get("bar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue()); verify(indicesAdmin().preparePutMapping().setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("foo").get().mappings().get("foo"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("foobar").get().mappings().get("foobar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("bar").get().mappings().get("bar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foo").get().mappings().get("foo"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foobar").get().mappings().get("foobar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "bar").get().mappings().get("bar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue()); verify(indicesAdmin().preparePutMapping("c*").setSource("field", "type=text"), true); assertAcked(indicesAdmin().prepareClose("barbaz").get()); verify(indicesAdmin().preparePutMapping("barbaz").setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue()); } public static final class TestPlugin extends Plugin { @@ -664,7 +664,7 @@ static GetFieldMappingsRequestBuilder getFieldMapping(String... indices) { } static GetMappingsRequestBuilder getMapping(String... indices) { - return indicesAdmin().prepareGetMappings(indices); + return indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, indices); } static GetSettingsRequestBuilder getSettings(String... indices) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java index de565605ff58a..7264585337fc7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java @@ -123,8 +123,9 @@ private void triggerClusterStateUpdates() { * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values. */ private void assertMappingsAndSettings(String expectedMappings) { - final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME)) - .actionGet(); + final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings( + new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(INDEX_NAME) + ).actionGet(); final Map mappings = getMappingsResponse.getMappings(); assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java index 71fcb25c2e0b2..e3092bda185fe 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java @@ -198,7 +198,7 @@ public void testGetFieldMappingsWithBlocks() throws Exception { try { enableIndexBlock("test", SETTING_BLOCKS_METADATA); - assertBlocked(indicesAdmin().prepareGetMappings(), INDEX_METADATA_BLOCK); + assertBlocked(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT), INDEX_METADATA_BLOCK); } finally { disableIndexBlock("test", SETTING_BLOCKS_METADATA); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java index 20e59fab3bd0c..023aa402b7337 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java @@ -42,7 +42,7 @@ protected Collection> nodePlugins() { public void testGetMappingsWhereThereAreNone() { createIndex("index"); - GetMappingsResponse response = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertThat(response.mappings().containsKey("index"), equalTo(true)); assertEquals(MappingMetadata.EMPTY_MAPPINGS, response.mappings().get("index")); } @@ -70,19 +70,19 @@ public void testSimpleGetMappings() throws Exception { assertThat(clusterHealth.isTimedOut(), equalTo(false)); // Get all mappings - GetMappingsResponse response = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertThat(response.mappings().size(), equalTo(2)); assertThat(response.mappings().get("indexa"), notNullValue()); assertThat(response.mappings().get("indexb"), notNullValue()); // Get all mappings, via wildcard support - response = indicesAdmin().prepareGetMappings("*").get(); + response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "*").get(); assertThat(response.mappings().size(), equalTo(2)); assertThat(response.mappings().get("indexa"), notNullValue()); assertThat(response.mappings().get("indexb"), notNullValue()); // Get mappings in indexa - response = indicesAdmin().prepareGetMappings("indexa").get(); + response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "indexa").get(); assertThat(response.mappings().size(), equalTo(1)); assertThat(response.mappings().get("indexa"), notNullValue()); } @@ -94,7 +94,7 @@ public void testGetMappingsWithBlocks() throws IOException { for (String block : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY)) { try { enableIndexBlock("test", block); - GetMappingsResponse response = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertThat(response.mappings().size(), equalTo(1)); assertNotNull(response.mappings().get("test")); } finally { @@ -104,7 +104,7 @@ public void testGetMappingsWithBlocks() throws IOException { try { enableIndexBlock("test", SETTING_BLOCKS_METADATA); - assertBlocked(indicesAdmin().prepareGetMappings(), INDEX_METADATA_BLOCK); + assertBlocked(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT), INDEX_METADATA_BLOCK); } finally { disableIndexBlock("test", SETTING_BLOCKS_METADATA); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java index 6f6e488d46b23..fa2598348a1ce 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java @@ -109,7 +109,7 @@ public void testUpdateMappingWithoutType() { assertThat(putMappingResponse.isAcknowledged(), equalTo(true)); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertThat(getMappingsResponse.mappings().get("test").source().toString(), equalTo(""" {"_doc":{"properties":{"body":{"type":"text"},"date":{"type":"integer"}}}}""")); } @@ -123,7 +123,7 @@ public void testUpdateMappingWithoutTypeMultiObjects() { assertThat(putMappingResponse.isAcknowledged(), equalTo(true)); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertThat(getMappingsResponse.mappings().get("test").source().toString(), equalTo(""" {"_doc":{"properties":{"date":{"type":"integer"}}}}""")); } @@ -215,7 +215,10 @@ public void testUpdateMappingConcurrently() throws Throwable { .get(); assertThat(response.isAcknowledged(), equalTo(true)); - GetMappingsResponse getMappingResponse = client2.admin().indices().prepareGetMappings(indexName).get(); + GetMappingsResponse getMappingResponse = client2.admin() + .indices() + .prepareGetMappings(TEST_REQUEST_TIMEOUT, indexName) + .get(); MappingMetadata mappings = getMappingResponse.getMappings().get(indexName); @SuppressWarnings("unchecked") Map properties = (Map) mappings.getSourceAsMap().get("properties"); @@ -284,7 +287,7 @@ private void assertConcreteMappingsOnAll(final String index, final String... fie * Waits for the given mapping type to exists on the master node. */ private void assertMappingOnMaster(final String index, final String... fieldNames) { - GetMappingsResponse response = indicesAdmin().prepareGetMappings(index).get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get(); MappingMetadata mappings = response.getMappings().get(index); assertThat(mappings, notNullValue()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java index dd4114c947174..84789d8a2acfb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.info.ClusterInfoRequest; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -21,7 +22,9 @@ public class GetMappingsRequest extends ClusterInfoRequest { - public GetMappingsRequest() {} + public GetMappingsRequest(TimeValue masterTimeout) { + super(masterTimeout); + } public GetMappingsRequest(StreamInput in) throws IOException { super(in); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java index 3f54138581398..a12ba4f60c26a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java @@ -11,13 +11,14 @@ import org.elasticsearch.action.support.master.info.ClusterInfoRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; +import org.elasticsearch.core.TimeValue; public class GetMappingsRequestBuilder extends ClusterInfoRequestBuilder< GetMappingsRequest, GetMappingsResponse, GetMappingsRequestBuilder> { - public GetMappingsRequestBuilder(ElasticsearchClient client, String... indices) { - super(client, GetMappingsAction.INSTANCE, new GetMappingsRequest().indices(indices)); + public GetMappingsRequestBuilder(ElasticsearchClient client, TimeValue masterTimeout, String... indices) { + super(client, GetMappingsAction.INSTANCE, new GetMappingsRequest(masterTimeout).indices(indices)); } } diff --git a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java index 943b03588d4ba..634a103e9754a 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import java.io.IOException; @@ -27,11 +28,22 @@ public abstract class ClusterInfoRequest getMappings(GetMappingsRequest request) return execute(GetMappingsAction.INSTANCE, request); } - public GetMappingsRequestBuilder prepareGetMappings(String... indices) { - return new GetMappingsRequestBuilder(this, indices); + public GetMappingsRequestBuilder prepareGetMappings(TimeValue masterTimeout, String... indices) { + return new GetMappingsRequestBuilder(this, masterTimeout, indices); } public void getFieldMappings(GetFieldMappingsRequest request, ActionListener listener) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index 27620fa750ea9..f9e02646041a6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.http.HttpChannel; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -51,11 +50,9 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); - final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); + final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(getMasterNodeTimeout(request)); getMappingsRequest.indices(indices); getMappingsRequest.indicesOptions(IndicesOptions.fromRequest(request, getMappingsRequest.indicesOptions())); - final TimeValue timeout = getMasterNodeTimeout(request); - getMappingsRequest.masterNodeTimeout(timeout); getMappingsRequest.local(request.paramAsBoolean("local", getMappingsRequest.local())); final HttpChannel httpChannel = request.getHttpChannel(); return channel -> new RestCancellableNodeClient(client, httpChannel).admin() diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java index 907a1a15721dc..4b0fac2cf2e0f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java @@ -54,7 +54,7 @@ public void putMappings() { } public void testGetMappings() { - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertExpectedMappings(getMappingsResponse.mappings()); } @@ -71,7 +71,7 @@ public void testGetFieldMappings() { assertFieldMappings(mappings.get("filtered"), FILTERED_FLAT_FIELDS); // double check that submitting the filtered mappings to an unfiltered index leads to the same get field mappings output // as the one coming from a filtered index with same mappings - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("filtered").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "filtered").get(); MappingMetadata filtered = getMappingsResponse.getMappings().get("filtered"); assertAcked(indicesAdmin().prepareCreate("test").setMapping(filtered.getSourceAsMap())); GetFieldMappingsResponse response = indicesAdmin().prepareGetFieldMappings("test").setFields("*").get(); @@ -98,7 +98,7 @@ public void testFieldCapabilities() { assertFieldCaps(filtered, filteredFields); // double check that submitting the filtered mappings to an unfiltered index leads to the same field_caps output // as the one coming from a filtered index with same mappings - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("filtered").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "filtered").get(); MappingMetadata filteredMapping = getMappingsResponse.getMappings().get("filtered"); assertAcked(indicesAdmin().prepareCreate("test").setMapping(filteredMapping.getSourceAsMap())); FieldCapabilitiesResponse test = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("test")).actionGet(); @@ -155,7 +155,7 @@ private void assertExpectedMappings(Map mappings) { private void assertMappingsAreValid(Map sourceAsMap) { // check that the returned filtered mappings are still valid mappings by submitting them and retrieving them back assertAcked(indicesAdmin().prepareCreate("test").setMapping(sourceAsMap)); - GetMappingsResponse testMappingsResponse = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse testMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertEquals(1, testMappingsResponse.getMappings().size()); // the mappings are returned unfiltered for this index, yet they are the same as the previous ones that were returned filtered assertFiltered(testMappingsResponse.getMappings().get("test")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java index 6ea41763cbcc2..b669c717719b0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java @@ -43,7 +43,7 @@ public void testCreateExplicitMappingSucceeds() throws Exception { """, getMapping()); var resp = client().admin().indices().prepareCreate("test").setMapping(mapping).get(); assertTrue(resp.isAcknowledged()); - var mappingsResp = client().admin().indices().prepareGetMappings("test").get(); + var mappingsResp = client().admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); var mappingMetadata = mappingsResp.getMappings().get("test"); var fieldType = XContentMapValues.extractValue("properties.field.type", mappingMetadata.getSourceAsMap()); assertThat(fieldType, equalTo(getTypeName())); @@ -149,7 +149,7 @@ public void testCreateExplicitMappingInIndexTemplateSucceeds() throws Exception var resp = client().prepareIndex("test1").setSource("field", "hello world").get(); assertThat(resp.status(), equalTo(RestStatus.CREATED)); - var mappingsResp = client().admin().indices().prepareGetMappings("test1").get(); + var mappingsResp = client().admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test1").get(); var mappingMetadata = mappingsResp.getMappings().get("test1"); var fieldType = XContentMapValues.extractValue("properties.field.type", mappingMetadata.getSourceAsMap()); assertThat(fieldType, equalTo(getTypeName())); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index 7ce55313aa771..cca8228557aba 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -488,7 +488,7 @@ public void testFollowerMappingIsUpdated() throws IOException { clusterStateRequest.indices(followerIndex); MappingMetadata mappingMetadata = followerClient().admin() .indices() - .prepareGetMappings("index2") + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2") .get() .getMappings() .get("index2"); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index 7556d02ddb1cf..e50627822e315 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -371,7 +371,12 @@ public void testSyncMappings() throws Exception { } assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), firstBatchNumDocs)); - MappingMetadata mappingMetadata = followerClient().admin().indices().prepareGetMappings("index2").get().getMappings().get("index2"); + MappingMetadata mappingMetadata = followerClient().admin() + .indices() + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2") + .get() + .getMappings() + .get("index2"); assertThat(XContentMapValues.extractValue("properties.f.type", mappingMetadata.sourceAsMap()), equalTo("integer")); assertThat(XContentMapValues.extractValue("properties.k", mappingMetadata.sourceAsMap()), nullValue()); @@ -382,7 +387,12 @@ public void testSyncMappings() throws Exception { } assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), firstBatchNumDocs + secondBatchNumDocs)); - mappingMetadata = followerClient().admin().indices().prepareGetMappings("index2").get().getMappings().get("index2"); + mappingMetadata = followerClient().admin() + .indices() + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2") + .get() + .getMappings() + .get("index2"); assertThat(XContentMapValues.extractValue("properties.f.type", mappingMetadata.sourceAsMap()), equalTo("integer")); assertThat(XContentMapValues.extractValue("properties.k.type", mappingMetadata.sourceAsMap()), equalTo("long")); pauseFollow("index2"); @@ -410,7 +420,12 @@ public void testNoMappingDefined() throws Exception { assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), 1)); pauseFollow("index2"); - MappingMetadata mappingMetadata = followerClient().admin().indices().prepareGetMappings("index2").get().getMappings().get("index2"); + MappingMetadata mappingMetadata = followerClient().admin() + .indices() + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2") + .get() + .getMappings() + .get("index2"); assertThat(XContentMapValues.extractValue("properties.f.type", mappingMetadata.sourceAsMap()), equalTo("long")); assertThat(XContentMapValues.extractValue("properties.k", mappingMetadata.sourceAsMap()), nullValue()); } @@ -1188,7 +1203,7 @@ public void testUpdateAnalysisLeaderIndexSettings() throws Exception { assertThat(getSettingsResponse.getSetting("follower", "index.analysis.analyzer.my_analyzer.type"), equalTo("custom")); assertThat(getSettingsResponse.getSetting("follower", "index.analysis.analyzer.my_analyzer.tokenizer"), equalTo("keyword")); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest(TEST_REQUEST_TIMEOUT); getMappingsRequest.indices("follower"); GetMappingsResponse getMappingsResponse = followerClient().admin().indices().getMappings(getMappingsRequest).actionGet(); MappingMetadata mappingMetadata = getMappingsResponse.getMappings().get("follower"); diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java index 1bf52b663b30f..af7e223e1c5b9 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java @@ -197,7 +197,7 @@ public void testSnapshotWithDanglingLocalSegment() throws Exception { } private static void assertMappings(String sourceIdx, boolean requireRouting, boolean useNested) throws IOException { - GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(sourceIdx).get(); + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, sourceIdx).get(); MappingMetadata mapping = getMappingsResponse.getMappings().get(sourceIdx); String nested = useNested ? """ ,"incorrect":{"type":"object"},"nested":{"type":"nested","properties":{"value":{"type":"long"}}}""" : ""; diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 01a26a7a0b7bb..41d815c8206a7 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -285,7 +285,7 @@ protected void masterOperation( // At any point if there is an issue, delete the downsample index // 1. Extract source index mappings - final GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(sourceIndexName); + final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(request.masterNodeTimeout()).indices(sourceIndexName); getMappingsRequest.setParentTask(parentTask); client.admin().indices().getMappings(getMappingsRequest, listener.delegateFailureAndWrap((delegate, getMappingsResponse) -> { final Map sourceIndexMappings = getMappingsResponse.mappings() diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index 33a436a1c7f1b..31b415270915b 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -1179,7 +1179,7 @@ private InternalAggregations aggregate(final String index, AggregationBuilder ag @SuppressWarnings("unchecked") private void assertDownsampleIndex(String sourceIndex, String downsampleIndex, DownsampleConfig config) throws Exception { // Retrieve field information for the metric fields - final GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(sourceIndex).get(); + final GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, sourceIndex).get(); final Map sourceIndexMappings = getMappingsResponse.mappings() .entrySet() .stream() @@ -1221,8 +1221,9 @@ private void assertDownsampleIndex(String sourceIndex, String downsampleIndex, D assertFieldMappings(config, metricFields, mappings); - GetMappingsResponse indexMappings = indicesAdmin().getMappings(new GetMappingsRequest().indices(downsampleIndex, sourceIndex)) - .actionGet(); + GetMappingsResponse indexMappings = indicesAdmin().getMappings( + new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(downsampleIndex, sourceIndex) + ).actionGet(); Map downsampleIndexProperties = (Map) indexMappings.mappings() .get(downsampleIndex) .sourceAsMap() diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java index b460c6abfeee4..f1b14658c6b8e 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java @@ -110,7 +110,8 @@ public void testMappingsCopiedFromSource() { client().execute(CreateIndexFromSourceAction.INSTANCE, new CreateIndexFromSourceAction.Request(sourceIndex, destIndex)) ); - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sourceIndex, destIndex)) + .actionGet(); Map mappings = mappingsResponse.mappings(); var destMappings = mappings.get(destIndex).sourceAsMap(); var sourceMappings = mappings.get(sourceIndex).sourceAsMap(); @@ -224,7 +225,7 @@ public void testMappingsOverridden() { ) ); - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(destIndex)).actionGet(); + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(destIndex)).actionGet(); Map mappings = mappingsResponse.mappings(); var destMappings = mappings.get(destIndex).sourceAsMap(); diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java index 46af8ab2fb4c2..71b2a2a43b343 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java @@ -213,7 +213,8 @@ public void testMappingsAddedToDestIndex() throws Exception { .actionGet() .getDestIndex(); - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sourceIndex, destIndex)) + .actionGet(); Map mappings = mappingsResponse.mappings(); var destMappings = mappings.get(destIndex).sourceAsMap(); var sourceMappings = mappings.get(sourceIndex).sourceAsMap(); @@ -319,7 +320,8 @@ public void testSettingsAndMappingsFromTemplate() throws IOException { // verify mappings from templates copied to dest index { - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sourceIndex, destIndex)) + .actionGet(); var destMappings = mappingsResponse.mappings().get(destIndex).sourceAsMap(); var sourceMappings = mappingsResponse.mappings().get(sourceIndex).sourceAsMap(); assertEquals(sourceMappings, destMappings); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java index ff6c3c8444783..7ce0fbe760644 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java @@ -255,7 +255,7 @@ public void testMultipleSimultaneousJobCreations() { // Assert that the mappings contain all the additional fields: field1, field2, field3, etc. String sharedResultsIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; - GetMappingsRequest request = new GetMappingsRequest().indices(sharedResultsIndex); + GetMappingsRequest request = new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sharedResultsIndex); GetMappingsResponse response = client().execute(GetMappingsAction.INSTANCE, request).actionGet(); Map indexMappings = response.getMappings(); assertNotNull(indexMappings); @@ -506,7 +506,7 @@ public void testGetDataCountsModelSizeAndTimingStatsWithSomeDocs() throws Except } private Map getIndexMappingProperties(String index) { - GetMappingsRequest request = new GetMappingsRequest().indices(index); + GetMappingsRequest request = new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(index); GetMappingsResponse response = client().execute(GetMappingsAction.INSTANCE, request).actionGet(); Map indexMappings = response.getMappings(); assertNotNull(indexMappings); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java index 759538b4cdc63..dea5e891bd8dd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java @@ -223,7 +223,7 @@ public void onFailure(Exception e) { ); // Get start context - getStartContext(request.getId(), task, startContextListener); + getStartContext(request.getId(), task, startContextListener, request.masterNodeTimeout()); } private void estimateMemoryUsageAndUpdateMemoryTracker(StartContext startContext, ActionListener listener) { @@ -264,7 +264,7 @@ private void estimateMemoryUsageAndUpdateMemoryTracker(StartContext startContext } - private void getStartContext(String id, Task task, ActionListener finalListener) { + private void getStartContext(String id, Task task, ActionListener finalListener, TimeValue masterTimeout) { ParentTaskAssigningClient parentTaskClient = new ParentTaskAssigningClient(client, task.getParentTaskId()); @@ -320,6 +320,7 @@ private void getStartContext(String id, Task task, ActionListener .andThen( (l, startContext) -> MappingsMerger.mergeMappings( parentTaskClient, + masterTimeout, startContext.config.getHeaders(), startContext.config.getSource(), l.map(ignored -> startContext) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java index ed12f54ab86b5..ba69c8733dd13 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java @@ -38,6 +38,7 @@ import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest; import org.elasticsearch.xpack.core.ml.dataframe.analyses.RequiredField; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.MachineLearning; import java.time.Clock; import java.util.Arrays; @@ -147,6 +148,7 @@ private static void prepareCreateIndexRequest( final Settings settings = settings(settingsResponse, destIndexAllowedSettings); MappingsMerger.mergeMappings( client, + MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, config.getHeaders(), config.getSource(), delegate.delegateFailureAndWrap( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java index 8fee0160cf92e..817020d788554 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource; @@ -34,6 +35,7 @@ private MappingsMerger() {} public static void mergeMappings( Client client, + TimeValue masterTimeout, Map headers, DataFrameAnalyticsSource source, ActionListener listener @@ -43,7 +45,7 @@ public static void mergeMappings( listener::onFailure ); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest(masterTimeout); getMappingsRequest.indices(source.getIndex()); ClientHelper.executeWithHeadersAsync(headers, ML_ORIGIN, client, GetMappingsAction.INSTANCE, getMappingsRequest, mappingsListener); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java index 51b3e0b55d75b..23b12a23e7495 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java @@ -126,6 +126,7 @@ import org.elasticsearch.xpack.core.ml.stats.StatsAccumulator; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.security.support.Exceptions; +import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.categorization.GrokPatternCreator; import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery; import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams; @@ -392,7 +393,10 @@ private void getLatestIndexMappingsAndAddTerms(String indexName, Collection startPersistentTask(job, listener, persistentTasksService), e -> { if (e instanceof ResourceAlreadyExistsException) { logger.debug("Rolled index already exists for rollup job [" + job.getConfig().getId() + "], updating metadata."); - updateMapping(job, listener, persistentTasksService, client, logger); + updateMapping(job, listener, persistentTasksService, client, logger, request.masterNodeTimeout()); } else { String msg = "Could not create index for rollup job [" + job.getConfig().getId() + "]"; logger.error(msg); @@ -249,7 +249,8 @@ static void updateMapping( ActionListener listener, PersistentTasksService persistentTasksService, Client client, - Logger logger + Logger logger, + TimeValue masterTimeout ) { final String indexName = job.getConfig().getRollupIndex(); @@ -304,7 +305,7 @@ static void updateMapping( ); }; - GetMappingsRequest request = new GetMappingsRequest(); + GetMappingsRequest request = new GetMappingsRequest(masterTimeout); client.execute(GetMappingsAction.INSTANCE, request, ActionListener.wrap(getMappingResponseHandler, e -> { String msg = "Could not update mappings for rollup job [" + job.getConfig().getId() + "]"; logger.error(msg); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java index ee8b4c79d1893..fed2439e513c8 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java @@ -166,7 +166,14 @@ public void testGetMappingFails() { return null; }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); } @@ -201,7 +208,14 @@ public void testNoMetadataInMapping() { return null; }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); } @@ -238,7 +252,14 @@ public void testMetadataButNotRollup() { return null; }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); } @@ -272,7 +293,14 @@ public void testJobAlreadyInMapping() { return null; }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); } @@ -318,7 +346,14 @@ public void testAddJobToMapping() { return null; }).when(client).execute(eq(TransportPutMappingAction.TYPE), any(PutMappingRequest.class), requestCaptor2.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); verify(client).execute(eq(TransportPutMappingAction.TYPE), any(PutMappingRequest.class), any()); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java index 57d18abaf1a92..f7361480f0a51 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java @@ -287,28 +287,28 @@ public void testGetMappingsIsFiltered() { { GetMappingsResponse getMappingsResponse = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("test").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field1"); } { GetMappingsResponse getMappingsResponse = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("test").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field2"); } { GetMappingsResponse getMappingsResponse = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("test").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field1"); } { GetMappingsResponse getMappingsResponse = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("test").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field1", "field2"); } } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java index bc730b5695c19..b501ba69d673b 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java @@ -166,7 +166,7 @@ public void testGetMappings() throws Exception { GetMappingsResponse response = client().filterWithHeader( singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue("kibana_user", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("logstash-*").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "logstash-*").get(); Map mappingsMap = response.getMappings(); assertNotNull(mappingsMap); assertNotNull(mappingsMap.get(index)); diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java index ffac36846414e..ea9d76fb643fb 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java @@ -172,7 +172,8 @@ public void testExceptionMapping() throws Exception { // ensure that enabled is set to false List indexed = new ArrayList<>(); - GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(HistoryStoreField.INDEX_PREFIX + "*").get(); + GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, HistoryStoreField.INDEX_PREFIX + "*") + .get(); for (MappingMetadata mapping : mappingsResponse.getMappings().values()) { Map docMapping = mapping.getSourceAsMap(); if (abortAtInput) { diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java index c78315383df2f..34d38dfde064d 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java @@ -46,7 +46,7 @@ public void testTimeFields() throws Exception { assertWatchWithMinimumActionsCount("_id", ExecutionState.EXECUTED, 1); assertBusy(() -> { - GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings() + GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT) .setIndicesOptions(IndicesOptions.strictExpandHidden()) .get(); assertThat(mappingsResponse, notNullValue()); diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java index 1bcdd060994ce..732b0800b4316 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java @@ -120,7 +120,7 @@ public void testFailedInputResultWithDotsInFieldNameGetsStored() throws Exceptio assertBusy(() -> { assertHitCount(getWatchHistory(), 1); }); // as fields with dots are allowed in 5.0 again, the mapping must be checked in addition - GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, ".watcher-history*").get(); XContentSource source = new XContentSource( response.getMappings().values().iterator().next().source().uncompressed(), XContentType.JSON @@ -162,7 +162,7 @@ public void testPayloadInputWithDotsInFieldNameWorks() throws Exception { assertBusy(() -> { assertHitCount(getWatchHistory(), 1); }); // as fields with dots are allowed in 5.0 again, the mapping must be checked in addition - GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, ".watcher-history*").get(); XContentSource source = new XContentSource( response.getMappings().values().iterator().next().source().uncompressed(), XContentType.JSON @@ -225,7 +225,7 @@ public void testThatHistoryContainsStatus() throws Exception { assertBusy(() -> { // also ensure that the status field is disabled in the watch history - GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, ".watcher-history*").get(); XContentSource mappingSource = new XContentSource( response.getMappings().values().iterator().next().source().uncompressed(), XContentType.JSON From 987da8bd967fb5a5c806f96830485403252ee43c Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Tue, 14 Jan 2025 12:40:30 +1000 Subject: [PATCH 27/44] Remove trappy timeouts from `ExplainLifecycleRequest` (#120036) Relates #107984 --- .../existence/FrozenExistenceDeciderIT.java | 2 +- .../core/ilm/ExplainLifecycleRequest.java | 5 +-- .../ilm/ExplainLifecycleRequestTests.java | 4 +-- .../ClusterStateWaitThresholdBreachTests.java | 10 +++--- ...ataStreamAndIndexLifecycleMixingTests.java | 36 ++++++++++++------- .../xpack/ilm/DataTiersMigrationsTests.java | 12 +++---- .../xpack/ilm/ILMMultiNodeIT.java | 6 ++-- .../ilm/ILMMultiNodeWithCCRDisabledIT.java | 6 ++-- .../IndexLifecycleInitialisationTests.java | 2 +- .../action/RestExplainLifecycleAction.java | 3 +- 10 files changed, 52 insertions(+), 34 deletions(-) diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java index df1022e45c5a5..2357b80a402b3 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java @@ -123,7 +123,7 @@ public void testZeroToOne() throws Exception { assertBusy(() -> { ExplainLifecycleResponse response = client().execute( ExplainLifecycleAction.INSTANCE, - new ExplainLifecycleRequest().indices(INDEX_NAME) + new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(INDEX_NAME) ).actionGet(); IndexLifecycleExplainResponse indexResponse = response.getIndexResponses().get(INDEX_NAME); assertNotNull(indexResponse); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java index 5c607335bff39..beff1e6ab39de 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.master.info.ClusterInfoRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import java.io.IOException; import java.util.Arrays; @@ -27,8 +28,8 @@ public class ExplainLifecycleRequest extends ClusterInfoRequest onlyManaged = onlyManaged == false; default -> throw new AssertionError("Illegal randomisation branch"); } - ExplainLifecycleRequest newRequest = new ExplainLifecycleRequest(); + ExplainLifecycleRequest newRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT); newRequest.indices(indices); newRequest.indicesOptions(indicesOptions); newRequest.onlyErrors(onlyErrors); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java index f25028824b56e..5d4b837dc4d1d 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java @@ -108,7 +108,7 @@ public void testWaitInShrunkShardsAllocatedExceedsThreshold() throws Exception { String[] firstAttemptShrinkIndexName = new String[1]; assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -118,7 +118,7 @@ public void testWaitInShrunkShardsAllocatedExceedsThreshold() throws Exception { // let's check ILM for the managed index is waiting in the `shrunk-shards-allocated` step assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -160,7 +160,7 @@ public void onFailure(Exception e) { String[] secondCycleShrinkIndexName = new String[1]; assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -180,7 +180,9 @@ public void onFailure(Exception e) { // situation and allow for shrink to complete by reducing the number of shards for the shrunk index to 0 setReplicaCount(0, secondCycleShrinkIndexName[0]); assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(secondCycleShrinkIndexName[0]); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + secondCycleShrinkIndexName[0] + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses() .get(secondCycleShrinkIndexName[0]); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java index 21924634ff6ab..d9cc4de344a39 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java @@ -149,7 +149,10 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { List backingIndices = getBackingIndices(dataStreamName); String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(firstGenerationIndex, secondGenerationIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + firstGenerationIndex, + secondGenerationIndex + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse firstGenerationExplain = explainResponse.getIndexResponses().get(firstGenerationIndex); @@ -193,7 +196,7 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); String writeIndex = backingIndices.get(2); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, writeIndex @@ -270,7 +273,7 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, @@ -348,7 +351,10 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro List backingIndices = getBackingIndices(dataStreamName); String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(firstGenerationIndex, secondGenerationIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + firstGenerationIndex, + secondGenerationIndex + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse firstGenerationExplain = explainResponse.getIndexResponses().get(firstGenerationIndex); @@ -401,7 +407,7 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex @@ -465,7 +471,7 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, @@ -543,7 +549,10 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce List backingIndices = getBackingIndices(dataStreamName); String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(firstGenerationIndex, secondGenerationIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + firstGenerationIndex, + secondGenerationIndex + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse firstGenerationExplain = explainResponse.getIndexResponses().get(firstGenerationIndex); @@ -591,7 +600,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); String writeIndex = backingIndices.get(2); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, writeIndex @@ -663,7 +672,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, @@ -720,7 +729,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, @@ -846,7 +855,7 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw assertThat(index.isManagedByLifecycle(), is(true)); } - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(writeIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(writeIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(writeIndex); @@ -875,7 +884,10 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(thirdGenerationIndex, writeIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + thirdGenerationIndex, + writeIndex + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse thirdGenerationExplain = explainResponse.getIndexResponses().get(thirdGenerationIndex); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java index 6d409bf474cfc..d483401931830 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java @@ -116,7 +116,7 @@ public void testIndexDataTierMigration() throws Exception { assertTrue(res.isAcknowledged()); assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -127,7 +127,7 @@ public void testIndexDataTierMigration() throws Exception { logger.info("starting a warm data node"); internalCluster().startNode(warmNode(Settings.EMPTY)); assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -140,7 +140,7 @@ public void testIndexDataTierMigration() throws Exception { // wait for lifecycle to complete in the cold phase after the index has been migrated to the cold node assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -177,7 +177,7 @@ public void testUserOptsOutOfTierMigration() throws Exception { assertTrue(res.isAcknowledged()); assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -197,7 +197,7 @@ public void testUserOptsOutOfTierMigration() throws Exception { // 1. start another cold node so both the primary and replica can relocate to the cold nodes // 2. remove the tier routing setting from the index again (we're doing this below) assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -210,7 +210,7 @@ public void testUserOptsOutOfTierMigration() throws Exception { // wait for lifecycle to complete in the cold phase assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java index 2c4c1c9e20bb6..3f92abc7e3da2 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java @@ -99,8 +99,10 @@ public void testShrinkOnTiers() throws Exception { prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").get(); assertBusy(() -> { - ExplainLifecycleResponse explain = client().execute(ExplainLifecycleAction.INSTANCE, new ExplainLifecycleRequest().indices("*")) - .get(); + ExplainLifecycleResponse explain = client().execute( + ExplainLifecycleAction.INSTANCE, + new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices("*") + ).get(); logger.info("--> explain: {}", Strings.toString(explain)); String backingIndexName = DataStream.getDefaultBackingIndexName(index, 1); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java index b91a309a23ae5..c695dc8f87b68 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java @@ -99,8 +99,10 @@ public void testShrinkOnTiers() throws Exception { prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").get(); assertBusy(() -> { - ExplainLifecycleResponse explain = client().execute(ExplainLifecycleAction.INSTANCE, new ExplainLifecycleRequest().indices("*")) - .get(); + ExplainLifecycleResponse explain = client().execute( + ExplainLifecycleAction.INSTANCE, + new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices("*") + ).get(); logger.info("--> explain: {}", Strings.toString(explain)); String backingIndexName = DataStream.getDefaultBackingIndexName(index, 1); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java index 644f88dc533b9..8999adc08f97b 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java @@ -388,7 +388,7 @@ public void testExplainParseOriginationDate() throws Exception { private IndexLifecycleExplainResponse executeExplainRequestAndGetTestIndexResponse(String indexName) throws ExecutionException, InterruptedException { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); assertThat(explainResponse.getIndexResponses().size(), equalTo(1)); return explainResponse.getIndexResponses().get(indexName); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java index 195f989eab055..3950edf648e18 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java @@ -36,12 +36,11 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { String[] indexes = Strings.splitStringByCommaToArray(restRequest.param("index")); - ExplainLifecycleRequest explainLifecycleRequest = new ExplainLifecycleRequest(); + ExplainLifecycleRequest explainLifecycleRequest = new ExplainLifecycleRequest(getMasterNodeTimeout(restRequest)); explainLifecycleRequest.indices(indexes); explainLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, IndicesOptions.strictExpandOpen())); explainLifecycleRequest.onlyManaged(restRequest.paramAsBoolean("only_managed", false)); explainLifecycleRequest.onlyErrors(restRequest.paramAsBoolean("only_errors", false)); - explainLifecycleRequest.masterNodeTimeout(getMasterNodeTimeout(restRequest)); return channel -> client.execute(ExplainLifecycleAction.INSTANCE, explainLifecycleRequest, new RestToXContentListener<>(channel)); } } From 27a9c4d911c4308755e3cb8139599b145284d357 Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Tue, 14 Jan 2025 12:41:05 +1000 Subject: [PATCH 28/44] Run template simulation actions on local node (#120038) The actions `TransportSimulateTemplateAction` and `TransportSimulateIndexTemplateAction` solely need the cluster state, they can run on any node. Additionally, they need to be cancellable to avoid doing unnecessary work after a client failure or timeout. As a drive-by, this removes more usages of the trappy default master node timeout. --- docs/changelog/120038.yaml | 5 ++ .../http/RestActionCancellationIT.java | 17 ++++++ .../post/SimulateIndexTemplateRequest.java | 33 ++++++------ .../post/SimulateIndexTemplateResponse.java | 30 +++-------- .../template/post/SimulateTemplateAction.java | 53 +++++++------------ .../TransportSimulateIndexTemplateAction.java | 35 +++++++----- .../post/TransportSimulateTemplateAction.java | 35 +++++++----- .../RestSimulateIndexTemplateAction.java | 9 ++-- .../indices/RestSimulateTemplateAction.java | 14 +++-- .../SimulateIndexTemplateRequestTests.java | 34 ++---------- .../post/SimulateTemplateRequestTests.java | 38 ++----------- 11 files changed, 132 insertions(+), 171 deletions(-) create mode 100644 docs/changelog/120038.yaml diff --git a/docs/changelog/120038.yaml b/docs/changelog/120038.yaml new file mode 100644 index 0000000000000..fe3a2ccccc095 --- /dev/null +++ b/docs/changelog/120038.yaml @@ -0,0 +1,5 @@ +pr: 120038 +summary: Run template simulation actions on local node +area: Ingest Node +type: enhancement +issues: [] diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java index ce514c5f1b1e7..0af12a41fa979 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java @@ -18,6 +18,8 @@ import org.elasticsearch.action.admin.indices.template.get.GetComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesAction; +import org.elasticsearch.action.admin.indices.template.post.SimulateIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.post.SimulateTemplateAction; import org.elasticsearch.action.support.CancellableActionTestPlugin; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.RefCountingListener; @@ -81,6 +83,21 @@ public void testGetComposableTemplateCancellation() { runRestActionCancellationTest(new Request(HttpGet.METHOD_NAME, "/_index_template"), GetComposableIndexTemplateAction.NAME); } + public void testSimulateTemplateCancellation() { + runRestActionCancellationTest( + new Request(HttpPost.METHOD_NAME, "/_index_template/_simulate/random_index_template"), + SimulateTemplateAction.NAME + ); + } + + public void testSimulateIndexTemplateCancellation() { + createIndex("test"); + runRestActionCancellationTest( + new Request(HttpPost.METHOD_NAME, "/_index_template/_simulate_index/test"), + SimulateIndexTemplateAction.NAME + ); + } + private void runRestActionCancellationTest(Request request, String actionName) { final var node = usually() ? internalCluster().getRandomNodeName() : internalCluster().startCoordinatingOnlyNode(Settings.EMPTY); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java index ce29d65ececf9..003be58d19554 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java @@ -12,16 +12,20 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.local.LocalClusterStateRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.Map; import java.util.Objects; -public class SimulateIndexTemplateRequest extends MasterNodeReadRequest { +public class SimulateIndexTemplateRequest extends LocalClusterStateRequest { private String indexName; @@ -30,14 +34,18 @@ public class SimulateIndexTemplateRequest extends MasterNodeReadRequest headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + public String getIndexName() { return indexName; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index a521dac60e96a..1a04b6e4d7633 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -12,13 +12,11 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; -import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.ResettableValue; import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -67,27 +65,11 @@ public RolloverConfiguration getRolloverConfiguration() { return rolloverConfiguration; } - public SimulateIndexTemplateResponse(StreamInput in) throws IOException { - super(in); - resolvedTemplate = in.readOptionalWriteable(Template::new); - if (in.readBoolean()) { - int overlappingTemplatesCount = in.readInt(); - overlappingTemplates = Maps.newMapWithExpectedSize(overlappingTemplatesCount); - for (int i = 0; i < overlappingTemplatesCount; i++) { - String templateName = in.readString(); - overlappingTemplates.put(templateName, in.readStringCollectionAsList()); - } - } else { - this.overlappingTemplates = null; - } - rolloverConfiguration = in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) - ? in.readOptionalWriteable(RolloverConfiguration::new) - : null; - if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { - in.readOptionalWriteable(DataStreamGlobalRetention::read); - } - } - + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(resolvedTemplate); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java index 75cc72416a854..15015b910767e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java @@ -14,12 +14,16 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.local.LocalClusterStateRequest; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.Map; import java.util.Objects; /** @@ -35,7 +39,7 @@ private SimulateTemplateAction() { super(NAME); } - public static class Request extends MasterNodeReadRequest { + public static class Request extends LocalClusterStateRequest { @Nullable private String templateName; @@ -44,26 +48,15 @@ public static class Request extends MasterNodeReadRequest { private TransportPutComposableIndexTemplateAction.Request indexTemplateRequest; private boolean includeDefaults = false; - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - } - - public Request(String templateName) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - if (templateName == null) { - throw new IllegalArgumentException("template name cannot be null"); - } + public Request(TimeValue masterTimeout, String templateName) { + super(masterTimeout); this.templateName = templateName; } - public Request(TransportPutComposableIndexTemplateAction.Request indexTemplateRequest) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - if (indexTemplateRequest == null) { - throw new IllegalArgumentException("index template body must be present"); - } - this.indexTemplateRequest = indexTemplateRequest; - } - + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until + * we no longer need to support calling this action remotely. + */ public Request(StreamInput in) throws IOException { super(in); templateName = in.readOptionalString(); @@ -73,16 +66,6 @@ public Request(StreamInput in) throws IOException { } } - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeOptionalString(templateName); - out.writeOptionalWriteable(indexTemplateRequest); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { - out.writeBoolean(includeDefaults); - } - } - @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; @@ -98,6 +81,11 @@ public ActionRequestValidationException validate() { return validationException; } + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + @Nullable public String getTemplateName() { return templateName; @@ -112,11 +100,6 @@ public TransportPutComposableIndexTemplateAction.Request getIndexTemplateRequest return indexTemplateRequest; } - public Request templateName(String templateName) { - this.templateName = templateName; - return this; - } - public Request indexTemplateRequest(TransportPutComposableIndexTemplateAction.Request indexTemplateRequest) { this.indexTemplateRequest = indexTemplateRequest; return this; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index d3d557b598b3a..74936128caa25 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -11,7 +11,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.action.support.ChannelActionListener; +import org.elasticsearch.action.support.local.TransportLocalClusterStateAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -20,7 +21,6 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; @@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettingProviders; @@ -42,7 +43,6 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -65,7 +65,7 @@ import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveLifecycle; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveSettings; -public class TransportSimulateIndexTemplateAction extends TransportMasterNodeReadAction< +public class TransportSimulateIndexTemplateAction extends TransportLocalClusterStateAction< SimulateIndexTemplateRequest, SimulateIndexTemplateResponse> { @@ -77,14 +77,18 @@ public class TransportSimulateIndexTemplateAction extends TransportMasterNodeRea private final ClusterSettings clusterSettings; private final boolean isDslOnlyMode; + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) + @SuppressWarnings("this-escape") @Inject public TransportSimulateIndexTemplateAction( TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, MetadataIndexTemplateService indexTemplateService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, NamedXContentRegistry xContentRegistry, IndicesService indicesService, SystemIndices systemIndices, @@ -92,13 +96,9 @@ public TransportSimulateIndexTemplateAction( ) { super( SimulateIndexTemplateAction.NAME, - transportService, - clusterService, - threadPool, actionFilters, - SimulateIndexTemplateRequest::new, - indexNameExpressionResolver, - SimulateIndexTemplateResponse::new, + transportService.getTaskManager(), + clusterService, EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; @@ -108,10 +108,19 @@ public TransportSimulateIndexTemplateAction( this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders(); this.clusterSettings = clusterService.getClusterSettings(); this.isDslOnlyMode = isDataStreamsLifecycleOnlyMode(clusterService.getSettings()); + + transportService.registerRequestHandler( + actionName, + executor, + false, + true, + SimulateIndexTemplateRequest::new, + (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel)) + ); } @Override - protected void masterOperation( + protected void localClusterStateOperation( Task task, SimulateIndexTemplateRequest request, ClusterState state, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index 30bbad0b57df0..692f027b23f9e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -11,26 +11,26 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.action.support.ChannelActionListener; +import org.elasticsearch.action.support.local.TransportLocalClusterStateAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -48,7 +48,7 @@ * Handles simulating an index template either by name (looking it up in the * cluster state), or by a provided template configuration */ -public class TransportSimulateTemplateAction extends TransportMasterNodeReadAction< +public class TransportSimulateTemplateAction extends TransportLocalClusterStateAction< SimulateTemplateAction.Request, SimulateIndexTemplateResponse> { @@ -60,14 +60,18 @@ public class TransportSimulateTemplateAction extends TransportMasterNodeReadActi private final ClusterSettings clusterSettings; private final boolean isDslOnlyMode; + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) + @SuppressWarnings("this-escape") @Inject public TransportSimulateTemplateAction( TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, MetadataIndexTemplateService indexTemplateService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, NamedXContentRegistry xContentRegistry, IndicesService indicesService, SystemIndices systemIndices, @@ -75,13 +79,9 @@ public TransportSimulateTemplateAction( ) { super( SimulateTemplateAction.NAME, - transportService, - clusterService, - threadPool, actionFilters, - SimulateTemplateAction.Request::new, - indexNameExpressionResolver, - SimulateIndexTemplateResponse::new, + transportService.getTaskManager(), + clusterService, EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; @@ -91,10 +91,19 @@ public TransportSimulateTemplateAction( this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders(); this.clusterSettings = clusterService.getClusterSettings(); this.isDslOnlyMode = isDataStreamsLifecycleOnlyMode(clusterService.getSettings()); + + transportService.registerRequestHandler( + actionName, + executor, + false, + true, + SimulateTemplateAction.Request::new, + (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel)) + ); } @Override - protected void masterOperation( + protected void localClusterStateOperation( Task task, SimulateTemplateAction.Request request, ClusterState state, diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java index 4cc010c15ffb9..1689d234387cc 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -41,8 +42,10 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - SimulateIndexTemplateRequest simulateIndexTemplateRequest = new SimulateIndexTemplateRequest(request.param("name")); - simulateIndexTemplateRequest.masterNodeTimeout(getMasterNodeTimeout(request)); + SimulateIndexTemplateRequest simulateIndexTemplateRequest = new SimulateIndexTemplateRequest( + getMasterNodeTimeout(request), + request.param("name") + ); simulateIndexTemplateRequest.includeDefaults(request.paramAsBoolean("include_defaults", false)); if (request.hasContent()) { TransportPutComposableIndexTemplateAction.Request indexTemplateRequest = new TransportPutComposableIndexTemplateAction.Request( @@ -57,7 +60,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC simulateIndexTemplateRequest.indexTemplateRequest(indexTemplateRequest); } - return channel -> client.execute( + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( SimulateIndexTemplateAction.INSTANCE, simulateIndexTemplateRequest, new RestToXContentListener<>(channel) diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java index cc2e3136a4163..47b68e24a9ed0 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -39,8 +40,10 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request(); - simulateRequest.templateName(request.param("name")); + SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request( + getMasterNodeTimeout(request), + request.param("name") + ); simulateRequest.includeDefaults(request.paramAsBoolean("include_defaults", false)); if (request.hasContent()) { TransportPutComposableIndexTemplateAction.Request indexTemplateRequest = new TransportPutComposableIndexTemplateAction.Request( @@ -54,8 +57,11 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli simulateRequest.indexTemplateRequest(indexTemplateRequest); } - simulateRequest.masterNodeTimeout(getMasterNodeTimeout(request)); - return channel -> client.execute(SimulateTemplateAction.INSTANCE, simulateRequest, new RestToXContentListener<>(channel)); + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( + SimulateTemplateAction.INSTANCE, + simulateRequest, + new RestToXContentListener<>(channel) + ); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java index 02a4c134ccd15..1d861b340543d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java @@ -12,45 +12,21 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplateTests; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; import java.util.List; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -public class SimulateIndexTemplateRequestTests extends AbstractWireSerializingTestCase { - - @Override - protected Writeable.Reader instanceReader() { - return SimulateIndexTemplateRequest::new; - } - - @Override - protected SimulateIndexTemplateRequest createTestInstance() { - SimulateIndexTemplateRequest req = new SimulateIndexTemplateRequest(randomAlphaOfLength(10)); - TransportPutComposableIndexTemplateAction.Request newTemplateRequest = new TransportPutComposableIndexTemplateAction.Request( - randomAlphaOfLength(4) - ); - newTemplateRequest.indexTemplate(ComposableIndexTemplateTests.randomInstance()); - req.indexTemplateRequest(newTemplateRequest); - req.includeDefaults(randomBoolean()); - return req; - } - - @Override - protected SimulateIndexTemplateRequest mutateInstance(SimulateIndexTemplateRequest instance) { - return randomValueOtherThan(instance, this::createTestInstance); - } +public class SimulateIndexTemplateRequestTests extends ESTestCase { public void testIndexNameCannotBeNullOrEmpty() { - expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest((String) null)); - expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest("")); + expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest(TEST_REQUEST_TIMEOUT, null)); + expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest(TEST_REQUEST_TIMEOUT, "")); } public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { @@ -60,7 +36,7 @@ public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); - SimulateIndexTemplateRequest simulateRequest = new SimulateIndexTemplateRequest("testing"); + SimulateIndexTemplateRequest simulateRequest = new SimulateIndexTemplateRequest(TEST_REQUEST_TIMEOUT, "testing"); simulateRequest.indexTemplateRequest(request); ActionRequestValidationException validationException = simulateRequest.validate(); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java index 14ebf260d3bf9..6163566a5e593 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java @@ -12,49 +12,17 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplateTests; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; import java.util.List; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -public class SimulateTemplateRequestTests extends AbstractWireSerializingTestCase { - - @Override - protected Writeable.Reader instanceReader() { - return SimulateTemplateAction.Request::new; - } - - @Override - protected SimulateTemplateAction.Request createTestInstance() { - SimulateTemplateAction.Request req = new SimulateTemplateAction.Request(randomAlphaOfLength(10)); - TransportPutComposableIndexTemplateAction.Request newTemplateRequest = new TransportPutComposableIndexTemplateAction.Request( - randomAlphaOfLength(4) - ); - newTemplateRequest.indexTemplate(ComposableIndexTemplateTests.randomInstance()); - req.indexTemplateRequest(newTemplateRequest); - req.includeDefaults(randomBoolean()); - return req; - } - - @Override - protected SimulateTemplateAction.Request mutateInstance(SimulateTemplateAction.Request instance) { - return randomValueOtherThan(instance, this::createTestInstance); - } - - public void testIndexNameCannotBeNullOrEmpty() { - expectThrows(IllegalArgumentException.class, () -> new SimulateTemplateAction.Request((String) null)); - expectThrows( - IllegalArgumentException.class, - () -> new SimulateTemplateAction.Request((TransportPutComposableIndexTemplateAction.Request) null) - ); - } +public class SimulateTemplateRequestTests extends ESTestCase { public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); @@ -63,7 +31,7 @@ public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); - SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request("testing"); + SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request(TEST_REQUEST_TIMEOUT, "testing"); simulateRequest.indexTemplateRequest(request); ActionRequestValidationException validationException = simulateRequest.validate(); From 4ccd377d27f34f8e9b1c0618e10083af0460b5d0 Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Tue, 14 Jan 2025 13:45:58 +1000 Subject: [PATCH 29/44] Run `TransportClusterGetSettingsAction` on local node (#119831) This action solely needs the cluster state, it can run on any node. Additionally, it needs to be cancellable to avoid doing unnecessary work after a client failure or timeout. The `?local` parameter becomes a no-op and is marked as deprecated. --- docs/changelog/119831.yaml | 5 ++ .../http/RestActionCancellationIT.java | 5 ++ .../api/cluster.get_settings.json | 2 +- .../settings/ClusterGetSettingsAction.java | 36 ++++++++------ .../TransportClusterGetSettingsAction.java | 41 ++++++++++------ .../cluster/RestClusterGetSettingsAction.java | 12 ++--- .../ClusterGetSettingsSerializationTests.java | 49 ------------------- .../settings/ClusterGetSettingsTests.java | 10 ++-- 8 files changed, 66 insertions(+), 94 deletions(-) create mode 100644 docs/changelog/119831.yaml delete mode 100644 server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java diff --git a/docs/changelog/119831.yaml b/docs/changelog/119831.yaml new file mode 100644 index 0000000000000..61c09d7d54de0 --- /dev/null +++ b/docs/changelog/119831.yaml @@ -0,0 +1,5 @@ +pr: 119831 +summary: Run `TransportClusterGetSettingsAction` on local node +area: Infra/Settings +type: enhancement +issues: [] diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java index 0af12a41fa979..c48ae9ba1843b 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java @@ -12,6 +12,7 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction; +import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; @@ -98,6 +99,10 @@ public void testSimulateIndexTemplateCancellation() { ); } + public void testClusterGetSettingsCancellation() { + runRestActionCancellationTest(new Request(HttpGet.METHOD_NAME, "/_cluster/settings"), ClusterGetSettingsAction.NAME); + } + private void runRestActionCancellationTest(Request request, String actionName) { final var node = usually() ? internalCluster().getRandomNodeName() : internalCluster().startCoordinatingOnlyNode(Settings.EMPTY); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json index 5862804257c67..5004ab8de697d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json @@ -26,7 +26,7 @@ }, "master_timeout":{ "type":"time", - "description":"Explicit operation timeout for connection to master node" + "description":"Timeout for waiting for new cluster state in case it is blocked" }, "timeout":{ "type":"time", diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java index 7e3c38c735091..ca02d19749ae7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java @@ -13,13 +13,18 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.local.LocalClusterStateRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.UpdateForV10; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.Map; import java.util.Objects; public class ClusterGetSettingsAction extends ActionType { @@ -34,25 +39,29 @@ public ClusterGetSettingsAction() { /** * Request to retrieve the cluster settings */ - public static class Request extends MasterNodeReadRequest { + public static class Request extends LocalClusterStateRequest { public Request(TimeValue masterNodeTimeout) { super(masterNodeTimeout); } + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) public Request(StreamInput in) throws IOException { super(in); assert in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0); } @Override - public void writeTo(StreamOutput out) throws IOException { - assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0); - super.writeTo(out); + public ActionRequestValidationException validate() { + return null; } @Override - public ActionRequestValidationException validate() { - return null; + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); } } @@ -79,20 +88,17 @@ public int hashCode() { return Objects.hash(persistentSettings, transientSettings, settings); } - public Response(StreamInput in) throws IOException { - super(in); - assert in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0); - persistentSettings = Settings.readSettingsFromStream(in); - transientSettings = Settings.readSettingsFromStream(in); - settings = Settings.readSettingsFromStream(in); - } - public Response(Settings persistentSettings, Settings transientSettings, Settings settings) { this.persistentSettings = Objects.requireNonNullElse(persistentSettings, Settings.EMPTY); this.transientSettings = Objects.requireNonNullElse(transientSettings, Settings.EMPTY); this.settings = Objects.requireNonNullElse(settings, Settings.EMPTY); } + /** + * NB prior to 9.0 get-component was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) @Override public void writeTo(StreamOutput out) throws IOException { assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java index dce6a38001392..71b976e012aad 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java @@ -11,57 +11,66 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.action.support.ChannelActionListener; +import org.elasticsearch.action.support.local.TransportLocalClusterStateAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -public class TransportClusterGetSettingsAction extends TransportMasterNodeReadAction< +public class TransportClusterGetSettingsAction extends TransportLocalClusterStateAction< ClusterGetSettingsAction.Request, ClusterGetSettingsAction.Response> { private final SettingsFilter settingsFilter; + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) + @SuppressWarnings("this-escape") @Inject public TransportClusterGetSettingsAction( TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, SettingsFilter settingsFilter, - ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + ActionFilters actionFilters ) { super( ClusterGetSettingsAction.NAME, - false, - transportService, - clusterService, - threadPool, actionFilters, - ClusterGetSettingsAction.Request::new, - indexNameExpressionResolver, - ClusterGetSettingsAction.Response::new, + transportService.getTaskManager(), + clusterService, EsExecutors.DIRECT_EXECUTOR_SERVICE ); - this.settingsFilter = settingsFilter; + + transportService.registerRequestHandler( + actionName, + executor, + false, + true, + ClusterGetSettingsAction.Request::new, + (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel)) + ); } @Override - protected void masterOperation( + protected void localClusterStateOperation( Task task, ClusterGetSettingsAction.Request request, ClusterState state, ActionListener listener ) throws Exception { + ((CancellableTask) task).ensureNotCancelled(); Metadata metadata = state.metadata(); listener.onResponse( new ClusterGetSettingsAction.Response( diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java index ca9e4abcaeec7..477cc1acb8319 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java @@ -11,15 +11,16 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsAction; import org.elasticsearch.action.admin.cluster.settings.RestClusterGetSettingsResponse; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -52,19 +53,14 @@ public String getName() { return "cluster_get_settings_action"; } - private static void setUpRequestParams(MasterNodeReadRequest clusterRequest, RestRequest request) { - clusterRequest.local(request.paramAsBoolean("local", clusterRequest.local())); - } - @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { final boolean renderDefaults = request.paramAsBoolean("include_defaults", false); ClusterGetSettingsAction.Request clusterSettingsRequest = new ClusterGetSettingsAction.Request(getMasterNodeTimeout(request)); + RestUtils.consumeDeprecatedLocalParameter(request); - setUpRequestParams(clusterSettingsRequest, request); - - return channel -> client.execute( + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( ClusterGetSettingsAction.INSTANCE, clusterSettingsRequest, new RestToXContentListener(channel).map( diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java deleted file mode 100644 index 5954de0586985..0000000000000 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.action.admin.cluster.settings; - -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.AbstractWireSerializingTestCase; - -public class ClusterGetSettingsSerializationTests extends AbstractWireSerializingTestCase { - @Override - protected Writeable.Reader instanceReader() { - return ClusterGetSettingsAction.Response::new; - } - - @Override - protected ClusterGetSettingsAction.Response createTestInstance() { - final Settings persistentSettings = Settings.builder() - .put("persistent.foo.filtered", "bar") - .put("persistent.foo.non_filtered", "baz") - .build(); - - final Settings transientSettings = Settings.builder() - .put("transient.foo.filtered", "bar") - .put("transient.foo.non_filtered", "baz") - .build(); - - final Settings allSettings = Settings.builder().put(persistentSettings).put(transientSettings).build(); - - return new ClusterGetSettingsAction.Response(persistentSettings, transientSettings, allSettings); - } - - @Override - protected ClusterGetSettingsAction.Response mutateInstance(ClusterGetSettingsAction.Response instance) { - final Settings otherSettings = Settings.builder().put("random.setting", randomAlphaOfLength(randomIntBetween(1, 12))).build(); - return switch (between(0, 2)) { - case 0 -> new ClusterGetSettingsAction.Response(otherSettings, instance.transientSettings(), instance.settings()); - case 1 -> new ClusterGetSettingsAction.Response(instance.persistentSettings(), otherSettings, instance.settings()); - case 2 -> new ClusterGetSettingsAction.Response(instance.persistentSettings(), instance.transientSettings(), otherSettings); - default -> throw new IllegalStateException("Unexpected switch value"); - }; - } -} diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java index 47f3f71bcc11b..58341b7d7fb9a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java @@ -12,17 +12,18 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.util.List; +import java.util.Map; import static org.mockito.Mockito.mock; @@ -54,10 +55,8 @@ public void testTransportFilters() throws Exception { TransportClusterGetSettingsAction action = new TransportClusterGetSettingsAction( transportService, mock(ClusterService.class), - threadPool, filter, - mock(ActionFilters.class), - mock(IndexNameExpressionResolver.class) + mock(ActionFilters.class) ); final Settings persistentSettings = Settings.builder() @@ -74,7 +73,8 @@ public void testTransportFilters() throws Exception { final ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metadata(metadata).build(); final PlainActionFuture future = new PlainActionFuture<>(); - action.masterOperation(null, null, clusterState, future); + final var task = new CancellableTask(1, "test", ClusterGetSettingsAction.NAME, "", null, Map.of()); + action.localClusterStateOperation(task, null, clusterState, future); assertTrue(future.isDone()); final ClusterGetSettingsAction.Response response = future.get(); From 476797e523ff7c226c7cb850066684b1ec4a6e25 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:29:16 +1100 Subject: [PATCH 30/44] Mute org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped #118406 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 2ede31c946207..9c640909cee6a 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -265,6 +265,9 @@ tests: - class: org.elasticsearch.entitlement.qa.EntitlementsDeniedIT method: testCheckThrows {pathPrefix=denied_nonmodular actionName=sslSessionImpl_getSessionContext} issue: https://github.com/elastic/elasticsearch/issues/120054 +- class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT + method: testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped + issue: https://github.com/elastic/elasticsearch/issues/118406 # Examples: # From b1f4fa7225c40192f2a8e4a538686d60e3002d64 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:58:05 +1100 Subject: [PATCH 31/44] Mute org.elasticsearch.xpack.ml.integration.DatafeedJobsIT org.elasticsearch.xpack.ml.integration.DatafeedJobsIT #120088 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 9c640909cee6a..64087660a9b64 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -268,6 +268,8 @@ tests: - class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT method: testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped issue: https://github.com/elastic/elasticsearch/issues/118406 +- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT + issue: https://github.com/elastic/elasticsearch/issues/120088 # Examples: # From 4b52cf4f2f51c33369a4aefb58dd8a1cce841e77 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 14 Jan 2025 08:24:58 +0100 Subject: [PATCH 32/44] Fix potential file leak in ES816BinaryQuantizedVectorsWriter (#120014) We are creating tmp files that might not get closed if an exception happens just after it. This commit makes sure all errors are handle properly and files are getting closed and deleted. --- docs/changelog/120014.yaml | 6 ++ muted-tests.yml | 3 - .../ES816BinaryQuantizedVectorsWriter.java | 81 +++++++++++-------- 3 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 docs/changelog/120014.yaml diff --git a/docs/changelog/120014.yaml b/docs/changelog/120014.yaml new file mode 100644 index 0000000000000..bef1f3ba49939 --- /dev/null +++ b/docs/changelog/120014.yaml @@ -0,0 +1,6 @@ +pr: 120014 +summary: Fix potential file leak in ES816BinaryQuantizedVectorsWriter +area: Search +type: bug +issues: + - 119981 diff --git a/muted-tests.yml b/muted-tests.yml index 64087660a9b64..3e69b7be0b2c3 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -242,9 +242,6 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 8.18.0]} issue: https://github.com/elastic/elasticsearch/issues/119980 -- class: org.elasticsearch.index.codec.vectors.es816.ES816HnswBinaryQuantizedVectorsFormatTests - method: testRandomExceptions - issue: https://github.com/elastic/elasticsearch/issues/119981 - class: org.elasticsearch.multi_cluster.MultiClusterYamlTestSuiteIT issue: https://github.com/elastic/elasticsearch/issues/119983 - class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java index 4d97235c5fae5..61bd5323b5b43 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java @@ -437,32 +437,35 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( float[] centroid, float cDotC ) throws IOException { - long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); - final IndexOutput tempQuantizedVectorData = segmentWriteState.directory.createTempOutput( - binarizedVectorData.getName(), - "temp", - segmentWriteState.context - ); - final IndexOutput tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput( - binarizedVectorData.getName(), - "score_temp", - segmentWriteState.context - ); - IndexInput binarizedDataInput = null; - IndexInput binarizedScoreDataInput = null; - boolean success = false; - int descritizedDimension = BQVectorUtils.discretize(fieldInfo.getVectorDimension(), 64); - BinaryQuantizer quantizer = new BinaryQuantizer( + final long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); + final int descritizedDimension = BQVectorUtils.discretize(fieldInfo.getVectorDimension(), 64); + final BinaryQuantizer quantizer = new BinaryQuantizer( fieldInfo.getVectorDimension(), descritizedDimension, fieldInfo.getVectorSimilarityFunction() ); + + IndexOutput tempQuantizedVectorData = null; + IndexOutput tempScoreQuantizedVectorData = null; + final DocsWithFieldSet docsWithField; + boolean success = false; + try { + tempQuantizedVectorData = segmentWriteState.directory.createTempOutput( + binarizedVectorData.getName(), + "temp", + segmentWriteState.context + ); + tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput( + binarizedVectorData.getName(), + "score_temp", + segmentWriteState.context + ); FloatVectorValues floatVectorValues = KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); if (fieldInfo.getVectorSimilarityFunction() == COSINE) { floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues); } - DocsWithFieldSet docsWithField = writeBinarizedVectorAndQueryData( + docsWithField = writeBinarizedVectorAndQueryData( tempQuantizedVectorData, tempScoreQuantizedVectorData, floatVectorValues, @@ -470,13 +473,30 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( quantizer ); CodecUtil.writeFooter(tempQuantizedVectorData); - IOUtils.close(tempQuantizedVectorData); + CodecUtil.writeFooter(tempScoreQuantizedVectorData); + success = true; + } finally { + if (success) { + IOUtils.close(tempQuantizedVectorData, tempScoreQuantizedVectorData); + } else { + IOUtils.closeWhileHandlingException(tempQuantizedVectorData, tempScoreQuantizedVectorData); + if (tempQuantizedVectorData != null) { + IOUtils.deleteFilesIgnoringExceptions(segmentWriteState.directory, tempQuantizedVectorData.getName()); + } + if (tempScoreQuantizedVectorData != null) { + IOUtils.deleteFilesIgnoringExceptions(segmentWriteState.directory, tempScoreQuantizedVectorData.getName()); + } + } + } + + IndexInput binarizedDataInput = null; + IndexInput binarizedScoreDataInput = null; + success = false; + try { binarizedDataInput = segmentWriteState.directory.openInput(tempQuantizedVectorData.getName(), segmentWriteState.context); binarizedVectorData.copyBytes(binarizedDataInput, binarizedDataInput.length() - CodecUtil.footerLength()); - long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset; + final long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset; CodecUtil.retrieveChecksum(binarizedDataInput); - CodecUtil.writeFooter(tempScoreQuantizedVectorData); - IOUtils.close(tempScoreQuantizedVectorData); binarizedScoreDataInput = segmentWriteState.directory.openInput( tempScoreQuantizedVectorData.getName(), segmentWriteState.context @@ -490,10 +510,9 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( cDotC, docsWithField ); - success = true; final IndexInput finalBinarizedDataInput = binarizedDataInput; final IndexInput finalBinarizedScoreDataInput = binarizedScoreDataInput; - OffHeapBinarizedVectorValues vectorValues = new OffHeapBinarizedVectorValues.DenseOffHeapVectorValues( + final OffHeapBinarizedVectorValues vectorValues = new OffHeapBinarizedVectorValues.DenseOffHeapVectorValues( fieldInfo.getVectorDimension(), docsWithField.cardinality(), centroid, @@ -503,7 +522,7 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( vectorsScorer, finalBinarizedDataInput ); - RandomVectorScorerSupplier scorerSupplier = vectorsScorer.getRandomVectorScorerSupplier( + final RandomVectorScorerSupplier scorerSupplier = vectorsScorer.getRandomVectorScorerSupplier( fieldInfo.getVectorSimilarityFunction(), new OffHeapBinarizedQueryVectorValues( finalBinarizedScoreDataInput, @@ -513,22 +532,20 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( ), vectorValues ); + final String tempQuantizedVectorDataName = tempQuantizedVectorData.getName(); + final String tempScoreQuantizedVectorDataName = tempScoreQuantizedVectorData.getName(); + success = true; return new BinarizedCloseableRandomVectorScorerSupplier(scorerSupplier, vectorValues, () -> { IOUtils.close(finalBinarizedDataInput, finalBinarizedScoreDataInput); IOUtils.deleteFilesIgnoringExceptions( segmentWriteState.directory, - tempQuantizedVectorData.getName(), - tempScoreQuantizedVectorData.getName() + tempQuantizedVectorDataName, + tempScoreQuantizedVectorDataName ); }); } finally { if (success == false) { - IOUtils.closeWhileHandlingException( - tempQuantizedVectorData, - tempScoreQuantizedVectorData, - binarizedDataInput, - binarizedScoreDataInput - ); + IOUtils.closeWhileHandlingException(binarizedDataInput, binarizedScoreDataInput); IOUtils.deleteFilesIgnoringExceptions( segmentWriteState.directory, tempQuantizedVectorData.getName(), From 5123b948a9c320ee806b2f7016e394bbbfb1a518 Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Tue, 14 Jan 2025 08:41:11 +0100 Subject: [PATCH 33/44] Remove deprecated tracing.apm.* settings for v9 (#119926) --- .../server/cli/APMJvmOptions.java | 50 +----- .../server/cli/APMJvmOptionsTests.java | 143 ++++++------------ docs/changelog/119926.yaml | 11 ++ .../org/elasticsearch/telemetry/apm/APM.java | 9 +- .../apm/internal/APMAgentSettings.java | 96 ++---------- .../apm/internal/APMAgentSettingsTests.java | 116 +------------- 6 files changed, 81 insertions(+), 344 deletions(-) create mode 100644 docs/changelog/119926.yaml diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java index c3b9768946767..1e57d9fab7cfd 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java @@ -187,20 +187,12 @@ static String agentCommandLineOption(Path agentJar, Path tmpPropertiesFile) { static void extractSecureSettings(SecureSettings secrets, Map propertiesMap) { final Set settingNames = secrets.getSettingNames(); for (String key : List.of("api_key", "secret_token")) { - for (String prefix : List.of("telemetry.", "tracing.apm.")) { - if (settingNames.contains(prefix + key)) { - if (propertiesMap.containsKey(key)) { - throw new IllegalStateException( - Strings.format("Duplicate telemetry setting: [telemetry.%s] and [tracing.apm.%s]", key, key) - ); - } - - try (SecureString token = secrets.getString(prefix + key)) { - propertiesMap.put(key, token.toString()); - } + String prefix = "telemetry."; + if (settingNames.contains(prefix + key)) { + try (SecureString token = secrets.getString(prefix + key)) { + propertiesMap.put(key, token.toString()); } } - } } @@ -227,44 +219,12 @@ private static Map extractDynamicSettings(Map pr static Map extractApmSettings(Settings settings) throws UserException { final Map propertiesMap = new HashMap<>(); - // tracing.apm.agent. is deprecated by telemetry.agent. final String telemetryAgentPrefix = "telemetry.agent."; - final String deprecatedTelemetryAgentPrefix = "tracing.apm.agent."; final Settings telemetryAgentSettings = settings.getByPrefix(telemetryAgentPrefix); telemetryAgentSettings.keySet().forEach(key -> propertiesMap.put(key, String.valueOf(telemetryAgentSettings.get(key)))); - final Settings apmAgentSettings = settings.getByPrefix(deprecatedTelemetryAgentPrefix); - for (String key : apmAgentSettings.keySet()) { - if (propertiesMap.containsKey(key)) { - throw new IllegalStateException( - Strings.format( - "Duplicate telemetry setting: [%s%s] and [%s%s]", - telemetryAgentPrefix, - key, - deprecatedTelemetryAgentPrefix, - key - ) - ); - } - propertiesMap.put(key, String.valueOf(apmAgentSettings.get(key))); - } - StringJoiner globalLabels = extractGlobalLabels(telemetryAgentPrefix, propertiesMap, settings); - if (globalLabels.length() == 0) { - globalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings); - } else { - StringJoiner tracingGlobalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings); - if (tracingGlobalLabels.length() != 0) { - throw new IllegalArgumentException( - "Cannot have global labels with tracing.agent prefix [" - + globalLabels - + "] and telemetry.apm.agent prefix [" - + tracingGlobalLabels - + "]" - ); - } - } if (globalLabels.length() > 0) { propertiesMap.put("global_labels", globalLabels.toString()); } @@ -274,7 +234,7 @@ static Map extractApmSettings(Settings settings) throws UserExce if (propertiesMap.containsKey(key)) { throw new UserException( ExitCodes.CONFIG, - "Do not set a value for [tracing.apm.agent." + key + "], as this is configured automatically by Elasticsearch" + "Do not set a value for [telemetry.agent." + key + "], as this is configured automatically by Elasticsearch" ); } } diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java index a7ba8eb11fbcc..0e067afc1aa73 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java @@ -25,18 +25,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -82,109 +79,63 @@ public void testFileDeleteWorks() throws IOException { } public void testExtractSecureSettings() { - MockSecureSettings duplicateSecureSettings = new MockSecureSettings(); + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("telemetry.secret_token", "token"); + secureSettings.setString("telemetry.api_key", "key"); - for (String prefix : List.of("telemetry.", "tracing.apm.")) { - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(prefix + "secret_token", "token"); - secureSettings.setString(prefix + "api_key", "key"); - - duplicateSecureSettings.setString(prefix + "api_key", "secret"); - - Map propertiesMap = new HashMap<>(); - APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap); - - assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key"))); - } - - Exception exception = expectThrows( - IllegalStateException.class, - () -> APMJvmOptions.extractSecureSettings(duplicateSecureSettings, new HashMap<>()) - ); - assertThat(exception.getMessage(), containsString("Duplicate telemetry setting")); - assertThat(exception.getMessage(), containsString("telemetry.api_key")); - assertThat(exception.getMessage(), containsString("tracing.apm.api_key")); + Map propertiesMap = new HashMap<>(); + APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap); + assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key"))); } public void testExtractSettings() throws UserException { - Function buildSettings = (prefix) -> Settings.builder() - .put(prefix + "server_url", "https://myurl:443") - .put(prefix + "service_node_name", "instance-0000000001"); - - for (String prefix : List.of("tracing.apm.agent.", "telemetry.agent.")) { - var name = "APM Tracing"; - var deploy = "123"; - var org = "456"; - var extracted = APMJvmOptions.extractApmSettings( - buildSettings.apply(prefix) - .put(prefix + "global_labels.deployment_name", name) - .put(prefix + "global_labels.deployment_id", deploy) - .put(prefix + "global_labels.organization_id", org) - .build() - ); - - assertThat( - extracted, - allOf( - hasEntry("server_url", "https://myurl:443"), - hasEntry("service_node_name", "instance-0000000001"), - hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one - not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys - ) - ); - - List labels = Arrays.stream(extracted.get("global_labels").split(",")).toList(); - assertThat(labels, hasSize(3)); - assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy)); - - // test replacing with underscores and skipping empty - name = "APM=Tracing"; - deploy = ""; - org = ",456"; - extracted = APMJvmOptions.extractApmSettings( - buildSettings.apply(prefix) - .put(prefix + "global_labels.deployment_name", name) - .put(prefix + "global_labels.deployment_id", deploy) - .put(prefix + "global_labels.organization_id", org) - .build() - ); - labels = Arrays.stream(extracted.get("global_labels").split(",")).toList(); - assertThat(labels, hasSize(2)); - assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456")); - } - - IllegalStateException err = expectThrows( - IllegalStateException.class, - () -> APMJvmOptions.extractApmSettings( - Settings.builder() - .put("tracing.apm.agent.server_url", "https://myurl:443") - .put("telemetry.agent.server_url", "https://myurl-2:443") - .build() - ) - ); - assertThat(err.getMessage(), is("Duplicate telemetry setting: [telemetry.agent.server_url] and [tracing.apm.agent.server_url]")); - } - - public void testNoMixedLabels() { - String telemetryAgent = "telemetry.agent."; - String tracingAgent = "tracing.apm.agent."; - Settings settings = Settings.builder() - .put("tracing.apm.enabled", true) - .put(telemetryAgent + "server_url", "https://myurl:443") - .put(telemetryAgent + "service_node_name", "instance-0000000001") - .put(tracingAgent + "global_labels.deployment_id", "123") - .put(telemetryAgent + "global_labels.organization_id", "456") + Settings defaults = Settings.builder() + .put("telemetry.agent.server_url", "https://myurl:443") + .put("telemetry.agent.service_node_name", "instance-0000000001") .build(); - IllegalArgumentException err = assertThrows(IllegalArgumentException.class, () -> APMJvmOptions.extractApmSettings(settings)); + var name = "APM Tracing"; + var deploy = "123"; + var org = "456"; + var extracted = APMJvmOptions.extractApmSettings( + Settings.builder() + .put(defaults) + .put("telemetry.agent.global_labels.deployment_name", name) + .put("telemetry.agent.global_labels.deployment_id", deploy) + .put("telemetry.agent.global_labels.organization_id", org) + .build() + ); + assertThat( - err.getMessage(), - is( - "Cannot have global labels with tracing.agent prefix [organization_id=456] and" - + " telemetry.apm.agent prefix [deployment_id=123]" + extracted, + allOf( + hasEntry("server_url", "https://myurl:443"), + hasEntry("service_node_name", "instance-0000000001"), + hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one + not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys ) ); + + List labels = Arrays.stream(extracted.get("global_labels").split(",")).toList(); + assertThat(labels, hasSize(3)); + assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy)); + + // test replacing with underscores and skipping empty + name = "APM=Tracing"; + deploy = ""; + org = ",456"; + extracted = APMJvmOptions.extractApmSettings( + Settings.builder() + .put(defaults) + .put("telemetry.agent.global_labels.deployment_name", name) + .put("telemetry.agent.global_labels.deployment_id", deploy) + .put("telemetry.agent.global_labels.organization_id", org) + .build() + ); + labels = Arrays.stream(extracted.get("global_labels").split(",")).toList(); + assertThat(labels, hasSize(2)); + assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456")); } private Path makeFakeAgentJar() throws IOException { diff --git a/docs/changelog/119926.yaml b/docs/changelog/119926.yaml new file mode 100644 index 0000000000000..3afafd5b2117f --- /dev/null +++ b/docs/changelog/119926.yaml @@ -0,0 +1,11 @@ +pr: 119926 +summary: "Deprecated tracing.apm.* settings got removed." +area: Infra/Metrics +type: breaking +issues: [] +breaking: + title: "Deprecated tracing.apm.* settings got removed." + area: Cluster and node setting + details: Deprecated `tracing.apm.*` settings got removed, use respective `telemetry.*` / `telemetry.tracing.*` settings instead. + impact: 9.x nodes will refuse to start if any such setting (including secret settings) is still present. + notable: false diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java index 339a4ec24ca13..43447cfa21a62 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java @@ -92,14 +92,7 @@ public List> getSettings() { APMAgentSettings.TELEMETRY_TRACING_ENABLED_SETTING, APMAgentSettings.TELEMETRY_TRACING_NAMES_INCLUDE_SETTING, APMAgentSettings.TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING, - APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES, - // The settings below are deprecated and are currently kept as fallback. - APMAgentSettings.TRACING_APM_SECRET_TOKEN_SETTING, - APMAgentSettings.TRACING_APM_API_KEY_SETTING, - APMAgentSettings.TRACING_APM_ENABLED_SETTING, - APMAgentSettings.TRACING_APM_NAMES_INCLUDE_SETTING, - APMAgentSettings.TRACING_APM_NAMES_EXCLUDE_SETTING, - APMAgentSettings.TRACING_APM_SANITIZE_FIELD_NAMES + APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES ); } } diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java index f66683a787bc0..8647761e2defe 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java @@ -25,9 +25,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.function.Function; -import static org.elasticsearch.common.settings.Setting.Property.Deprecated; import static org.elasticsearch.common.settings.Setting.Property.NodeScope; import static org.elasticsearch.common.settings.Setting.Property.OperatorDynamic; @@ -101,9 +99,6 @@ public void setAgentSetting(String key, String value) { private static final String TELEMETRY_SETTING_PREFIX = "telemetry."; - // The old legacy prefix - private static final String LEGACY_TRACING_APM_SETTING_PREFIX = "tracing.apm."; - /** * Allow-list of APM agent config keys users are permitted to configure. * @see APM Java Agent Configuration @@ -248,56 +243,24 @@ private static Setting concreteAgentSetting(String namespace, String qua public static final Setting.AffixSetting APM_AGENT_SETTINGS = Setting.prefixKeySetting( TELEMETRY_SETTING_PREFIX + "agent.", - LEGACY_TRACING_APM_SETTING_PREFIX + "agent.", - (namespace, qualifiedKey) -> qualifiedKey.startsWith(LEGACY_TRACING_APM_SETTING_PREFIX) - ? concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic, Deprecated) - : concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic) + null, // no fallback + (namespace, qualifiedKey) -> concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic) ); - /** - * @deprecated in favor of TELEMETRY_TRACING_NAMES_INCLUDE_SETTING. - */ - @Deprecated - public static final Setting> TRACING_APM_NAMES_INCLUDE_SETTING = Setting.stringListSetting( - LEGACY_TRACING_APM_SETTING_PREFIX + "names.include", - OperatorDynamic, - NodeScope, - Deprecated - ); - - public static final Setting> TELEMETRY_TRACING_NAMES_INCLUDE_SETTING = Setting.listSetting( + public static final Setting> TELEMETRY_TRACING_NAMES_INCLUDE_SETTING = Setting.stringListSetting( TELEMETRY_SETTING_PREFIX + "tracing.names.include", - TRACING_APM_NAMES_INCLUDE_SETTING, - Function.identity(), OperatorDynamic, NodeScope ); - /** - * @deprecated in favor of TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING. - */ - @Deprecated - public static final Setting> TRACING_APM_NAMES_EXCLUDE_SETTING = Setting.stringListSetting( - LEGACY_TRACING_APM_SETTING_PREFIX + "names.exclude", - OperatorDynamic, - NodeScope, - Deprecated - ); - - public static final Setting> TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING = Setting.listSetting( + public static final Setting> TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING = Setting.stringListSetting( TELEMETRY_SETTING_PREFIX + "tracing.names.exclude", - TRACING_APM_NAMES_EXCLUDE_SETTING, - Function.identity(), OperatorDynamic, NodeScope ); - /** - * @deprecated in favor of TELEMETRY_TRACING_SANITIZE_FIELD_NAMES. - */ - @Deprecated - public static final Setting> TRACING_APM_SANITIZE_FIELD_NAMES = Setting.stringListSetting( - LEGACY_TRACING_APM_SETTING_PREFIX + "sanitize_field_names", + public static final Setting> TELEMETRY_TRACING_SANITIZE_FIELD_NAMES = Setting.stringListSetting( + TELEMETRY_SETTING_PREFIX + "tracing.sanitize_field_names", List.of( "password", "passwd", @@ -313,33 +276,12 @@ private static Setting concreteAgentSetting(String namespace, String qua "set-cookie" ), OperatorDynamic, - NodeScope, - Deprecated - ); - - public static final Setting> TELEMETRY_TRACING_SANITIZE_FIELD_NAMES = Setting.listSetting( - TELEMETRY_SETTING_PREFIX + "tracing.sanitize_field_names", - TRACING_APM_SANITIZE_FIELD_NAMES, - Function.identity(), - OperatorDynamic, NodeScope ); - /** - * @deprecated in favor of TELEMETRY_TRACING_ENABLED_SETTING. - */ - @Deprecated - public static final Setting TRACING_APM_ENABLED_SETTING = Setting.boolSetting( - LEGACY_TRACING_APM_SETTING_PREFIX + "enabled", - false, - OperatorDynamic, - NodeScope, - Deprecated - ); - public static final Setting TELEMETRY_TRACING_ENABLED_SETTING = Setting.boolSetting( TELEMETRY_SETTING_PREFIX + "tracing.enabled", - TRACING_APM_ENABLED_SETTING, + false, OperatorDynamic, NodeScope ); @@ -351,33 +293,13 @@ private static Setting concreteAgentSetting(String namespace, String qua NodeScope ); - /** - * @deprecated in favor of TELEMETRY_SECRET_TOKEN_SETTING. - */ - @Deprecated - public static final Setting TRACING_APM_SECRET_TOKEN_SETTING = SecureSetting.secureString( - LEGACY_TRACING_APM_SETTING_PREFIX + "secret_token", - null, - Deprecated - ); - public static final Setting TELEMETRY_SECRET_TOKEN_SETTING = SecureSetting.secureString( TELEMETRY_SETTING_PREFIX + "secret_token", - TRACING_APM_SECRET_TOKEN_SETTING - ); - - /** - * @deprecated in favor of TELEMETRY_API_KEY_SETTING. - */ - @Deprecated - public static final Setting TRACING_APM_API_KEY_SETTING = SecureSetting.secureString( - LEGACY_TRACING_APM_SETTING_PREFIX + "api_key", - null, - Deprecated + null ); public static final Setting TELEMETRY_API_KEY_SETTING = SecureSetting.secureString( TELEMETRY_SETTING_PREFIX + "api_key", - TRACING_APM_API_KEY_SETTING + null ); } diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java index a60048c82a3c9..5516672420924 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java @@ -11,8 +11,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.MockSecureSettings; -import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.mockito.Mockito; @@ -21,21 +19,13 @@ import java.util.Set; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.APM_AGENT_SETTINGS; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_API_KEY_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_METRICS_ENABLED_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_SECRET_TOKEN_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_ENABLED_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_NAMES_INCLUDE_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_API_KEY_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_ENABLED_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_NAMES_EXCLUDE_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_NAMES_INCLUDE_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_SANITIZE_FIELD_NAMES; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_SECRET_TOKEN_SETTING; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasItem; import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.mock; @@ -70,14 +60,6 @@ public void testEnableTracing() { } } - public void testEnableTracingUsingLegacySetting() { - Settings settings = Settings.builder().put(TRACING_APM_ENABLED_SETTING.getKey(), true).build(); - apmAgentSettings.initAgentSystemProperties(settings); - - verify(apmAgentSettings).setAgentSetting("recording", "true"); - assertWarnings("[tracing.apm.enabled] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - public void testEnableMetrics() { for (boolean tracingEnabled : List.of(true, false)) { clearInvocations(apmAgentSettings, apmTelemetryProvider.getMeterService()); @@ -121,14 +103,6 @@ public void testDisableTracing() { } } - public void testDisableTracingUsingLegacySetting() { - Settings settings = Settings.builder().put(TRACING_APM_ENABLED_SETTING.getKey(), false).build(); - apmAgentSettings.initAgentSystemProperties(settings); - - verify(apmAgentSettings).setAgentSetting("recording", "false"); - assertWarnings("[tracing.apm.enabled] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - public void testDisableMetrics() { for (boolean tracingEnabled : List.of(true, false)) { clearInvocations(apmAgentSettings, apmTelemetryProvider.getMeterService()); @@ -181,70 +155,18 @@ public void testSetAgentSettings() { verify(apmAgentSettings).setAgentSetting("span_compression_enabled", "true"); } - public void testSetAgentsSettingsWithLegacyPrefix() { - Settings settings = Settings.builder() - .put(TELEMETRY_TRACING_ENABLED_SETTING.getKey(), true) - .put("tracing.apm.agent.span_compression_enabled", "true") - .build(); - apmAgentSettings.initAgentSystemProperties(settings); - - verify(apmAgentSettings).setAgentSetting("recording", "true"); - verify(apmAgentSettings).setAgentSetting("span_compression_enabled", "true"); - assertWarnings( - "[tracing.apm.agent.span_compression_enabled] setting was deprecated in Elasticsearch and will be removed in a future release." - ); - } - /** * Check that invalid or forbidden APM agent settings are rejected. */ public void testRejectForbiddenOrUnknownAgentSettings() { - List prefixes = List.of(APM_AGENT_SETTINGS.getKey(), "tracing.apm.agent."); - for (String prefix : prefixes) { - Settings settings = Settings.builder().put(prefix + "unknown", "true").build(); - Exception exception = expectThrows(IllegalArgumentException.class, () -> APM_AGENT_SETTINGS.getAsMap(settings)); - assertThat(exception.getMessage(), containsString("[" + prefix + "unknown]")); - } - // though, accept / ignore nested global_labels - for (String prefix : prefixes) { - Settings settings = Settings.builder().put(prefix + "global_labels.abc", "123").build(); - APMAgentSettings.APM_AGENT_SETTINGS.getAsMap(settings); - - if (prefix.startsWith("tracing.apm.agent.")) { - assertWarnings( - "[tracing.apm.agent.global_labels.abc] setting was deprecated in Elasticsearch and will be removed in a future release." - ); - } - } - } - - public void testTelemetryTracingNamesIncludeFallback() { - Settings settings = Settings.builder().put(TRACING_APM_NAMES_INCLUDE_SETTING.getKey(), "abc,xyz").build(); - - List included = TELEMETRY_TRACING_NAMES_INCLUDE_SETTING.get(settings); - - assertThat(included, containsInAnyOrder("abc", "xyz")); - assertWarnings("[tracing.apm.names.include] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - - public void testTelemetryTracingNamesExcludeFallback() { - Settings settings = Settings.builder().put(TRACING_APM_NAMES_EXCLUDE_SETTING.getKey(), "abc,xyz").build(); - - List included = TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING.get(settings); - - assertThat(included, containsInAnyOrder("abc", "xyz")); - assertWarnings("[tracing.apm.names.exclude] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - - public void testTelemetryTracingSanitizeFieldNamesFallback() { - Settings settings = Settings.builder().put(TRACING_APM_SANITIZE_FIELD_NAMES.getKey(), "abc,xyz").build(); - - List included = TELEMETRY_TRACING_SANITIZE_FIELD_NAMES.get(settings); + String prefix = APM_AGENT_SETTINGS.getKey(); + Settings settings = Settings.builder().put(prefix + "unknown", "true").build(); + Exception exception = expectThrows(IllegalArgumentException.class, () -> APM_AGENT_SETTINGS.getAsMap(settings)); + assertThat(exception.getMessage(), containsString("[" + prefix + "unknown]")); - assertThat(included, containsInAnyOrder("abc", "xyz")); - assertWarnings( - "[tracing.apm.sanitize_field_names] setting was deprecated in Elasticsearch and will be removed in a future release." - ); + // though, accept / ignore nested global_labels + var map = APMAgentSettings.APM_AGENT_SETTINGS.getAsMap(Settings.builder().put(prefix + "global_labels.abc", "123").build()); + assertThat(map, hasEntry("global_labels.abc", "123")); } public void testTelemetryTracingSanitizeFieldNamesFallbackDefault() { @@ -252,28 +174,6 @@ public void testTelemetryTracingSanitizeFieldNamesFallbackDefault() { assertThat(included, hasItem("password")); // and more defaults } - public void testTelemetrySecretTokenFallback() { - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(TRACING_APM_SECRET_TOKEN_SETTING.getKey(), "verysecret"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - - try (SecureString secureString = TELEMETRY_SECRET_TOKEN_SETTING.get(settings)) { - assertEquals("verysecret", secureString.toString()); - } - assertWarnings("[tracing.apm.secret_token] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - - public void testTelemetryApiKeyFallback() { - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(TRACING_APM_API_KEY_SETTING.getKey(), "abc"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - - try (SecureString secureString = TELEMETRY_API_KEY_SETTING.get(settings)) { - assertEquals("abc", secureString.toString()); - } - assertWarnings("[tracing.apm.api_key] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - /** * Check that invalid or forbidden APM agent settings are rejected if their last part resembles an allowed setting. */ From 1be9253779797157c397e8a1931d06fe613ca35b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Tue, 14 Jan 2025 09:08:15 +0100 Subject: [PATCH 34/44] [Entitlements] Network entitlement classes + Datagram socket check functions (#119735) --- .../bridge/EntitlementChecker.java | 30 ++++ .../qa/common/DummyImplementations.java | 78 ++++++++- .../common/RestEntitlementsCheckAction.java | 160 ++++++++++-------- .../plugin-metadata/entitlement-policy.yaml | 5 + .../plugin-metadata/entitlement-policy.yaml | 5 + .../api/ElasticsearchEntitlementChecker.java | 71 ++++++++ .../runtime/policy/NetworkEntitlement.java | 107 ++++++++++++ .../runtime/policy/PolicyManager.java | 34 +++- .../runtime/policy/PolicyParser.java | 3 +- .../policy/NetworkEntitlementTests.java | 49 ++++++ .../runtime/policy/PolicyParserTests.java | 16 ++ 11 files changed, 485 insertions(+), 73 deletions(-) create mode 100644 libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java create mode 100644 libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java index 3a359eb921fc8..69fc57973f68a 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -13,10 +13,16 @@ import java.io.PrintStream; import java.io.PrintWriter; import java.net.ContentHandlerFactory; +import java.net.DatagramPacket; +import java.net.DatagramSocket; import java.net.DatagramSocketImplFactory; import java.net.FileNameMap; +import java.net.InetAddress; +import java.net.MulticastSocket; +import java.net.NetworkInterface; import java.net.ProxySelector; import java.net.ResponseCache; +import java.net.SocketAddress; import java.net.SocketImplFactory; import java.net.URL; import java.net.URLStreamHandler; @@ -189,4 +195,28 @@ public interface EntitlementChecker { // The only implementation of SSLSession#getSessionContext(); unfortunately it's an interface, so we need to check the implementation void check$sun_security_ssl_SSLSessionImpl$getSessionContext(Class callerClass, SSLSession sslSession); + + void check$java_net_DatagramSocket$bind(Class callerClass, DatagramSocket that, SocketAddress addr); + + void check$java_net_DatagramSocket$connect(Class callerClass, DatagramSocket that, InetAddress addr); + + void check$java_net_DatagramSocket$connect(Class callerClass, DatagramSocket that, SocketAddress addr); + + void check$java_net_DatagramSocket$send(Class callerClass, DatagramSocket that, DatagramPacket p); + + void check$java_net_DatagramSocket$receive(Class callerClass, DatagramSocket that, DatagramPacket p); + + void check$java_net_DatagramSocket$joinGroup(Class callerClass, DatagramSocket that, SocketAddress addr, NetworkInterface ni); + + void check$java_net_DatagramSocket$leaveGroup(Class callerClass, DatagramSocket that, SocketAddress addr, NetworkInterface ni); + + void check$java_net_MulticastSocket$joinGroup(Class callerClass, MulticastSocket that, InetAddress addr); + + void check$java_net_MulticastSocket$joinGroup(Class callerClass, MulticastSocket that, SocketAddress addr, NetworkInterface ni); + + void check$java_net_MulticastSocket$leaveGroup(Class callerClass, MulticastSocket that, InetAddress addr); + + void check$java_net_MulticastSocket$leaveGroup(Class callerClass, MulticastSocket that, SocketAddress addr, NetworkInterface ni); + + void check$java_net_MulticastSocket$send(Class callerClass, MulticastSocket that, DatagramPacket p, byte ttl); } diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java index 6dbb684c71514..fae873123528d 100644 --- a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java +++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java @@ -9,8 +9,15 @@ package org.elasticsearch.entitlement.qa.common; +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.DatagramSocketImpl; import java.net.InetAddress; +import java.net.NetworkInterface; import java.net.Socket; +import java.net.SocketAddress; +import java.net.SocketException; import java.security.cert.Certificate; import java.text.BreakIterator; import java.text.Collator; @@ -327,8 +334,77 @@ public Socket createSocket(Socket s, String host, int port, boolean autoClose) { } } + static class DummyDatagramSocket extends DatagramSocket { + DummyDatagramSocket() throws SocketException { + super(new DatagramSocketImpl() { + @Override + protected void create() throws SocketException {} + + @Override + protected void bind(int lport, InetAddress laddr) throws SocketException {} + + @Override + protected void send(DatagramPacket p) throws IOException {} + + @Override + protected int peek(InetAddress i) throws IOException { + return 0; + } + + @Override + protected int peekData(DatagramPacket p) throws IOException { + return 0; + } + + @Override + protected void receive(DatagramPacket p) throws IOException {} + + @Override + protected void setTTL(byte ttl) throws IOException {} + + @Override + protected byte getTTL() throws IOException { + return 0; + } + + @Override + protected void setTimeToLive(int ttl) throws IOException {} + + @Override + protected int getTimeToLive() throws IOException { + return 0; + } + + @Override + protected void join(InetAddress inetaddr) throws IOException {} + + @Override + protected void leave(InetAddress inetaddr) throws IOException {} + + @Override + protected void joinGroup(SocketAddress mcastaddr, NetworkInterface netIf) throws IOException {} + + @Override + protected void leaveGroup(SocketAddress mcastaddr, NetworkInterface netIf) throws IOException {} + + @Override + protected void close() {} + + @Override + public void setOption(int optID, Object value) throws SocketException {} + + @Override + public Object getOption(int optID) throws SocketException { + return null; + } + + @Override + protected void connect(InetAddress address, int port) throws SocketException {} + }); + } + } + private static RuntimeException unexpected() { return new IllegalStateException("This method isn't supposed to be called"); } - } diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java index 1dd8daf556226..3a5480f468528 100644 --- a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java +++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyBreakIteratorProvider; import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyCalendarDataProvider; @@ -32,14 +33,18 @@ import org.elasticsearch.rest.RestStatus; import java.io.IOException; -import java.io.UncheckedIOException; +import java.net.DatagramPacket; import java.net.DatagramSocket; import java.net.HttpURLConnection; +import java.net.InetAddress; +import java.net.InetSocketAddress; import java.net.MalformedURLException; +import java.net.NetworkInterface; import java.net.ProxySelector; import java.net.ResponseCache; import java.net.ServerSocket; import java.net.Socket; +import java.net.SocketException; import java.net.URL; import java.net.URLClassLoader; import java.net.URLConnection; @@ -71,20 +76,20 @@ public class RestEntitlementsCheckAction extends BaseRestHandler { public static final Thread NO_OP_SHUTDOWN_HOOK = new Thread(() -> {}, "Shutdown hook for testing"); private final String prefix; - record CheckAction(Runnable action, boolean isAlwaysDeniedToPlugins) { + record CheckAction(CheckedRunnable action, boolean isAlwaysDeniedToPlugins) { /** * These cannot be granted to plugins, so our test plugins cannot test the "allowed" case. * Used both for always-denied entitlements as well as those granted only to the server itself. */ - static CheckAction deniedToPlugins(Runnable action) { + static CheckAction deniedToPlugins(CheckedRunnable action) { return new CheckAction(action, true); } - static CheckAction forPlugins(Runnable action) { + static CheckAction forPlugins(CheckedRunnable action) { return new CheckAction(action, false); } - static CheckAction alwaysDenied(Runnable action) { + static CheckAction alwaysDenied(CheckedRunnable action) { return new CheckAction(action, true); } } @@ -142,7 +147,13 @@ static CheckAction alwaysDenied(Runnable action) { entry("createURLStreamHandlerProvider", alwaysDenied(RestEntitlementsCheckAction::createURLStreamHandlerProvider)), entry("createURLWithURLStreamHandler", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler)), entry("createURLWithURLStreamHandler2", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler2)), - entry("sslSessionImpl_getSessionContext", alwaysDenied(RestEntitlementsCheckAction::sslSessionImplGetSessionContext)) + entry("sslSessionImpl_getSessionContext", alwaysDenied(RestEntitlementsCheckAction::sslSessionImplGetSessionContext)), + entry("datagram_socket_bind", forPlugins(RestEntitlementsCheckAction::bindDatagramSocket)), + entry("datagram_socket_connect", forPlugins(RestEntitlementsCheckAction::connectDatagramSocket)), + entry("datagram_socket_send", forPlugins(RestEntitlementsCheckAction::sendDatagramSocket)), + entry("datagram_socket_receive", forPlugins(RestEntitlementsCheckAction::receiveDatagramSocket)), + entry("datagram_socket_join_group", forPlugins(RestEntitlementsCheckAction::joinGroupDatagramSocket)), + entry("datagram_socket_leave_group", forPlugins(RestEntitlementsCheckAction::leaveGroupDatagramSocket)) ); private static void createURLStreamHandlerProvider() { @@ -154,43 +165,33 @@ public URLStreamHandler createURLStreamHandler(String protocol) { }; } - private static void sslSessionImplGetSessionContext() { + private static void sslSessionImplGetSessionContext() throws IOException { SSLSocketFactory factory = HttpsURLConnection.getDefaultSSLSocketFactory(); try (SSLSocket socket = (SSLSocket) factory.createSocket()) { SSLSession session = socket.getSession(); session.getSessionContext(); - } catch (IOException e) { - throw new RuntimeException(e); } } @SuppressWarnings("deprecation") - private static void createURLWithURLStreamHandler() { - try { - var x = new URL("http", "host", 1234, "file", new URLStreamHandler() { - @Override - protected URLConnection openConnection(URL u) { - return null; - } - }); - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } + private static void createURLWithURLStreamHandler() throws MalformedURLException { + var x = new URL("http", "host", 1234, "file", new URLStreamHandler() { + @Override + protected URLConnection openConnection(URL u) { + return null; + } + }); } @SuppressWarnings("deprecation") - private static void createURLWithURLStreamHandler2() { - try { - var x = new URL(null, "spec", new URLStreamHandler() { - @Override - protected URLConnection openConnection(URL u) { - return null; - } - }); - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } + private static void createURLWithURLStreamHandler2() throws MalformedURLException { + var x = new URL(null, "spec", new URLStreamHandler() { + @Override + protected URLConnection openConnection(URL u) { + return null; + } + }); } private static void createInetAddressResolverProvider() { @@ -215,12 +216,8 @@ private static void setDefaultProxySelector() { ProxySelector.setDefault(null); } - private static void setDefaultSSLContext() { - try { - SSLContext.setDefault(SSLContext.getDefault()); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } + private static void setDefaultSSLContext() throws NoSuchAlgorithmException { + SSLContext.setDefault(SSLContext.getDefault()); } private static void setDefaultHostnameVerifier() { @@ -246,28 +243,18 @@ private static void systemExit() { System.exit(123); } - private static void createClassLoader() { + private static void createClassLoader() throws IOException { try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) { logger.info("Created URLClassLoader [{}]", classLoader.getName()); - } catch (IOException e) { - throw new UncheckedIOException(e); } } - private static void processBuilder_start() { - try { - new ProcessBuilder("").start(); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void processBuilder_start() throws IOException { + new ProcessBuilder("").start(); } - private static void processBuilder_startPipeline() { - try { - ProcessBuilder.startPipeline(List.of()); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void processBuilder_startPipeline() throws IOException { + ProcessBuilder.startPipeline(List.of()); } private static void setHttpsConnectionProperties() { @@ -355,12 +342,8 @@ private static void setHttpsConnectionProperties() { @SuppressWarnings("deprecation") @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API") - private static void datagramSocket$$setDatagramSocketImplFactory() { - try { - DatagramSocket.setDatagramSocketImplFactory(() -> { throw new IllegalStateException(); }); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void datagramSocket$$setDatagramSocketImplFactory() throws IOException { + DatagramSocket.setDatagramSocketImplFactory(() -> { throw new IllegalStateException(); }); } private static void httpURLConnection$$setFollowRedirects() { @@ -369,22 +352,14 @@ private static void setHttpsConnectionProperties() { @SuppressWarnings("deprecation") @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API") - private static void serverSocket$$setSocketFactory() { - try { - ServerSocket.setSocketFactory(() -> { throw new IllegalStateException(); }); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void serverSocket$$setSocketFactory() throws IOException { + ServerSocket.setSocketFactory(() -> { throw new IllegalStateException(); }); } @SuppressWarnings("deprecation") @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API") - private static void socket$$setSocketImplFactory() { - try { - Socket.setSocketImplFactory(() -> { throw new IllegalStateException(); }); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void socket$$setSocketImplFactory() throws IOException { + Socket.setSocketImplFactory(() -> { throw new IllegalStateException(); }); } private static void url$$setURLStreamHandlerFactory() { @@ -399,6 +374,51 @@ private static void setHttpsConnectionProperties() { URLConnection.setContentHandlerFactory(__ -> { throw new IllegalStateException(); }); } + private static void bindDatagramSocket() throws SocketException { + try (var socket = new DatagramSocket(null)) { + socket.bind(null); + } + } + + @SuppressForbidden(reason = "testing entitlements") + private static void connectDatagramSocket() throws SocketException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.connect(new InetSocketAddress(1234)); + } + } + + private static void joinGroupDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.joinGroup( + new InetSocketAddress(InetAddress.getByAddress(new byte[] { (byte) 230, 0, 0, 1 }), 1234), + NetworkInterface.getByIndex(0) + ); + } + } + + private static void leaveGroupDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.leaveGroup( + new InetSocketAddress(InetAddress.getByAddress(new byte[] { (byte) 230, 0, 0, 1 }), 1234), + NetworkInterface.getByIndex(0) + ); + } + } + + @SuppressForbidden(reason = "testing entitlements") + private static void sendDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.send(new DatagramPacket(new byte[] { 0 }, 1, InetAddress.getLocalHost(), 1234)); + } + } + + @SuppressForbidden(reason = "testing entitlements") + private static void receiveDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.receive(new DatagramPacket(new byte[1], 1, InetAddress.getLocalHost(), 1234)); + } + } + public RestEntitlementsCheckAction(String prefix) { this.prefix = prefix; } diff --git a/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml index 30fc9f0abeec0..05a94f09264a8 100644 --- a/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml +++ b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,3 +1,8 @@ ALL-UNNAMED: - create_class_loader - set_https_connection_properties + - network: + actions: + - listen + - accept + - connect diff --git a/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml b/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml index 0a25570a9f624..0d2c66c2daa2c 100644 --- a/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml +++ b/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,3 +1,8 @@ org.elasticsearch.entitlement.qa.common: - create_class_loader - set_https_connection_properties + - network: + actions: + - listen + - accept + - connect diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index ca4aaceabcebf..dd39ec3c5fe43 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -10,16 +10,23 @@ package org.elasticsearch.entitlement.runtime.api; import org.elasticsearch.entitlement.bridge.EntitlementChecker; +import org.elasticsearch.entitlement.runtime.policy.NetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.PolicyManager; import java.io.InputStream; import java.io.PrintStream; import java.io.PrintWriter; import java.net.ContentHandlerFactory; +import java.net.DatagramPacket; +import java.net.DatagramSocket; import java.net.DatagramSocketImplFactory; import java.net.FileNameMap; +import java.net.InetAddress; +import java.net.MulticastSocket; +import java.net.NetworkInterface; import java.net.ProxySelector; import java.net.ResponseCache; +import java.net.SocketAddress; import java.net.SocketImplFactory; import java.net.URL; import java.net.URLStreamHandler; @@ -349,4 +356,68 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { public void check$sun_security_ssl_SSLSessionImpl$getSessionContext(Class callerClass, SSLSession sslSession) { policyManager.checkReadSensitiveNetworkInformation(callerClass); } + + @Override + public void check$java_net_DatagramSocket$bind(Class callerClass, DatagramSocket that, SocketAddress addr) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.LISTEN_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$connect(Class callerClass, DatagramSocket that, InetAddress addr) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$connect(Class callerClass, DatagramSocket that, SocketAddress addr) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$send(Class callerClass, DatagramSocket that, DatagramPacket p) { + var actions = NetworkEntitlement.CONNECT_ACTION; + if (p.getAddress().isMulticastAddress()) { + actions |= NetworkEntitlement.ACCEPT_ACTION; + } + policyManager.checkNetworkAccess(callerClass, actions); + } + + @Override + public void check$java_net_DatagramSocket$receive(Class callerClass, DatagramSocket that, DatagramPacket p) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$joinGroup(Class caller, DatagramSocket that, SocketAddress addr, NetworkInterface ni) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$leaveGroup(Class caller, DatagramSocket that, SocketAddress addr, NetworkInterface ni) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$joinGroup(Class callerClass, MulticastSocket that, InetAddress addr) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$joinGroup(Class caller, MulticastSocket that, SocketAddress addr, NetworkInterface ni) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$leaveGroup(Class caller, MulticastSocket that, InetAddress addr) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$leaveGroup(Class caller, MulticastSocket that, SocketAddress addr, NetworkInterface ni) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$send(Class callerClass, MulticastSocket that, DatagramPacket p, byte ttl) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java new file mode 100644 index 0000000000000..b6c6a41d5be7f --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.core.Strings; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.StringJoiner; + +import static java.util.Map.entry; + +/** + * Describes a network entitlement (sockets) with actions. + */ +public class NetworkEntitlement implements Entitlement { + + public static final int LISTEN_ACTION = 0x1; + public static final int CONNECT_ACTION = 0x2; + public static final int ACCEPT_ACTION = 0x4; + + static final String LISTEN = "listen"; + static final String CONNECT = "connect"; + static final String ACCEPT = "accept"; + + private static final Map ACTION_MAP = Map.ofEntries( + entry(LISTEN, LISTEN_ACTION), + entry(CONNECT, CONNECT_ACTION), + entry(ACCEPT, ACCEPT_ACTION) + ); + + private final int actions; + + @ExternalEntitlement(parameterNames = { "actions" }, esModulesOnly = false) + public NetworkEntitlement(List actionsList) { + + int actionsInt = 0; + + for (String actionString : actionsList) { + var action = ACTION_MAP.get(actionString); + if (action == null) { + throw new IllegalArgumentException("unknown network action [" + actionString + "]"); + } + if ((actionsInt & action) == action) { + throw new IllegalArgumentException(Strings.format("network action [%s] specified multiple times", actionString)); + } + actionsInt |= action; + } + + this.actions = actionsInt; + } + + public static Object printActions(int actions) { + var joiner = new StringJoiner(","); + for (var entry : ACTION_MAP.entrySet()) { + var action = entry.getValue(); + if ((actions & action) == action) { + joiner.add(entry.getKey()); + } + } + return joiner.toString(); + } + + /** + * For the actions to match, the actions present in this entitlement must be a superset + * of the actions required by a check. + * There is only one "negative" case (action required by the check but not present in the entitlement), + * and it can be expressed efficiently via this truth table: + * this.actions | requiredActions | + * 0 | 0 | 0 + * 0 | 1 | 1 --> NOT this.action AND requiredActions + * 1 | 0 | 0 + * 1 | 1 | 0 + * + * @param requiredActions the actions required to be present for a check to pass + * @return true if requiredActions are present, false otherwise + */ + public boolean matchActions(int requiredActions) { + return (~this.actions & requiredActions) == 0; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NetworkEntitlement that = (NetworkEntitlement) o; + return actions == that.actions; + } + + @Override + public int hashCode() { + return Objects.hash(actions); + } + + @Override + public String toString() { + return "NetworkEntitlement{actions=" + actions + '}'; + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index 57449a23a8215..f039fbda3dfbd 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -52,7 +52,11 @@ public boolean hasEntitlement(Class entitlementClass) { } public Stream getEntitlements(Class entitlementClass) { - return entitlementsByType.get(entitlementClass).stream().map(entitlementClass::cast); + var entitlements = entitlementsByType.get(entitlementClass); + if (entitlements == null) { + return Stream.empty(); + } + return entitlements.stream().map(entitlementClass::cast); } } @@ -190,6 +194,34 @@ private String operationDescription(String methodName) { return methodName.substring(methodName.indexOf('$')); } + public void checkNetworkAccess(Class callerClass, int actions) { + var requestingClass = requestingClass(callerClass); + if (isTriviallyAllowed(requestingClass)) { + return; + } + + ModuleEntitlements entitlements = getEntitlements(requestingClass); + if (entitlements.getEntitlements(NetworkEntitlement.class).anyMatch(n -> n.matchActions(actions))) { + logger.debug( + () -> Strings.format( + "Entitled: class [%s], module [%s], entitlement [Network], actions [Ox%X]", + requestingClass, + requestingClass.getModule().getName(), + actions + ) + ); + return; + } + throw new NotEntitledException( + Strings.format( + "Missing entitlement: class [%s], module [%s], entitlement [Network], actions [%s]", + requestingClass, + requestingClass.getModule().getName(), + NetworkEntitlement.printActions(actions) + ) + ); + } + private void checkEntitlementPresent(Class callerClass, Class entitlementClass) { var requestingClass = requestingClass(callerClass); if (isTriviallyAllowed(requestingClass)) { diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java index 013acf8f22fae..ac4d4afdd97f8 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java @@ -37,7 +37,8 @@ public class PolicyParser { private static final Map> EXTERNAL_ENTITLEMENTS = Stream.of( FileEntitlement.class, CreateClassLoaderEntitlement.class, - SetHttpsConnectionPropertiesEntitlement.class + SetHttpsConnectionPropertiesEntitlement.class, + NetworkEntitlement.class ).collect(Collectors.toUnmodifiableMap(PolicyParser::getEntitlementTypeName, Function.identity())); protected final XContentParser policyParser; diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java new file mode 100644 index 0000000000000..91051d48c365f --- /dev/null +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.test.ESTestCase; + +import java.util.List; + +import static org.hamcrest.Matchers.is; + +public class NetworkEntitlementTests extends ESTestCase { + + public void testMatchesActions() { + var listenEntitlement = new NetworkEntitlement(List.of(NetworkEntitlement.LISTEN)); + var emptyEntitlement = new NetworkEntitlement(List.of()); + var connectAcceptEntitlement = new NetworkEntitlement(List.of(NetworkEntitlement.CONNECT, NetworkEntitlement.ACCEPT)); + + assertThat(listenEntitlement.matchActions(0), is(true)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION), is(true)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + + assertThat(connectAcceptEntitlement.matchActions(0), is(true)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION), is(false)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.ACCEPT_ACTION), is(true)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION), is(true)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(true)); + + assertThat(emptyEntitlement.matchActions(0), is(true)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + } +} diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java index 4d17fc92e1578..1e0c31d2280b8 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java @@ -52,6 +52,22 @@ public void testPolicyBuilderOnExternalPlugin() throws IOException { assertEquals(expected, parsedPolicy); } + public void testParseNetwork() throws IOException { + Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - network: + actions: + - listen + - accept + - connect + """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy(); + Policy expected = new Policy( + "test-policy.yaml", + List.of(new Scope("entitlement-module-name", List.of(new NetworkEntitlement(List.of("listen", "accept", "connect"))))) + ); + assertEquals(expected, parsedPolicy); + } + public void testParseCreateClassloader() throws IOException { Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: From d953079380243f518ca3847fdcb4e1816a4c8163 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 14 Jan 2025 10:28:59 +0100 Subject: [PATCH 35/44] Cleanup swapSourceProvider(...) workaround (#118480) This reverts the workaround that was introduced in #117792 to avoid EOF error when an es|ql query uses multiple runtime fields that fallback to source when source mode is synthetic. This is now covered by the ReinitializingSourceProvider workaround that covers that and the concurrency problem. With this change, the main code for the required workarounds are now in isolated in ReinitializingSourceProvider. Additional another in `ReinitializingSourceProvider` was fixed, the issue was the lastSeenDoc field was reused overwritten by different threads, the latest commit moves the lastSeenDoc field to PerThreadSourceProvider so that each thread gets its own place to store the last seen docid. --- .../search/lookup/SearchLookup.java | 11 ---------- .../planner/EsPhysicalOperationProviders.java | 12 +---------- .../plugin/ReinitializingSourceProvider.java | 20 ++++++++++++------- 3 files changed, 14 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index 9eb0170af5efb..d899a390d8be7 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -102,14 +102,6 @@ private SearchLookup(SearchLookup searchLookup, Set fieldChain) { this.fieldLookupProvider = searchLookup.fieldLookupProvider; } - private SearchLookup(SearchLookup searchLookup, SourceProvider sourceProvider, Set fieldChain) { - this.fieldChain = Collections.unmodifiableSet(fieldChain); - this.sourceProvider = sourceProvider; - this.fieldTypeLookup = searchLookup.fieldTypeLookup; - this.fieldDataLookup = searchLookup.fieldDataLookup; - this.fieldLookupProvider = searchLookup.fieldLookupProvider; - } - /** * Creates a copy of the current {@link SearchLookup} that looks fields up in the same way, but also tracks field references * in order to detect cycles and prevent resolving fields that depend on more than {@link #MAX_FIELD_CHAIN_DEPTH} other fields. @@ -153,7 +145,4 @@ public Source getSource(LeafReaderContext ctx, int doc) throws IOException { return sourceProvider.getSource(ctx, doc); } - public SearchLookup swapSourceProvider(SourceProvider sourceProvider) { - return new SearchLookup(this, sourceProvider, fieldChain); - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 8b63a146f2e5d..112f101ad842b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -33,7 +33,6 @@ import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedLookup; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -339,16 +338,7 @@ public MappedFieldType.FieldExtractPreference fieldExtractPreference() { @Override public SearchLookup lookup() { - boolean syntheticSource = SourceFieldMapper.isSynthetic(indexSettings()); - var searchLookup = ctx.lookup(); - if (syntheticSource) { - // in the context of scripts and when synthetic source is used the search lookup can't always be reused between - // users of SearchLookup. This is only an issue when scripts fallback to _source, but since we can't always - // accurately determine whether a script uses _source, we should do this for all script usages. - // This lookup() method is only invoked for scripts / runtime fields, so it is ok to do here. - searchLookup = searchLookup.swapSourceProvider(ctx.createSourceProvider()); - } - return searchLookup; + return ctx.lookup(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java index 8dee3478b3b64..61ac67674b252 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java @@ -28,10 +28,6 @@ final class ReinitializingSourceProvider implements SourceProvider { private PerThreadSourceProvider perThreadProvider; private final Supplier sourceProviderFactory; - // Keeping track of last seen doc and if current doc is before last seen doc then source provider is initialized: - // (when source mode is synthetic then _source is read from doc values and doc values don't support going backwards) - private int lastSeenDocId; - ReinitializingSourceProvider(Supplier sourceProviderFactory) { this.sourceProviderFactory = sourceProviderFactory; } @@ -40,15 +36,25 @@ final class ReinitializingSourceProvider implements SourceProvider { public Source getSource(LeafReaderContext ctx, int doc) throws IOException { var currentThread = Thread.currentThread(); PerThreadSourceProvider provider = perThreadProvider; - if (provider == null || provider.creatingThread != currentThread || doc < lastSeenDocId) { + if (provider == null || provider.creatingThread != currentThread || doc < provider.lastSeenDocId) { provider = new PerThreadSourceProvider(sourceProviderFactory.get(), currentThread); this.perThreadProvider = provider; } - lastSeenDocId = doc; + provider.lastSeenDocId = doc; return provider.source.getSource(ctx, doc); } - private record PerThreadSourceProvider(SourceProvider source, Thread creatingThread) { + private static final class PerThreadSourceProvider { + final SourceProvider source; + final Thread creatingThread; + // Keeping track of last seen doc and if current doc is before last seen doc then source provider is initialized: + // (when source mode is synthetic then _source is read from doc values and doc values don't support going backwards) + int lastSeenDocId; + + private PerThreadSourceProvider(SourceProvider source, Thread creatingThread) { + this.source = source; + this.creatingThread = creatingThread; + } } } From d85b90ad8c76d4b6b06af9c26c0a0c99aa7ab463 Mon Sep 17 00:00:00 2001 From: Pete Gillin Date: Tue, 14 Jan 2025 10:34:46 +0000 Subject: [PATCH 36/44] Remove unfreeze REST endpoint (#119227) This adds a sentence to `redirects.asciidoc` explaining what frozen indices were - otherwise, everything will point to the message about the unfreeze API having gone away, which is not very helpful. Some cross-references are updated to point to this rather than to the notice about the removal of the unfreeze API. ES-9736 #comment Removed `_unfreeze` REST endpoint in https://github.com/elastic/elasticsearch/pull/119227 --- docs/changelog/119227.yaml | 13 ++++ docs/reference/indices.asciidoc | 2 - docs/reference/indices/apis/unfreeze.asciidoc | 61 ----------------- docs/reference/indices/index-mgmt.asciidoc | 2 +- docs/reference/redirects.asciidoc | 16 +++-- docs/reference/sql/language/indices.asciidoc | 2 +- .../rest-api-spec/api/indices.unfreeze.json | 67 ------------------- .../xpack/frozen/FrozenIndices.java | 29 -------- .../rest/action/RestFreezeIndexAction.java | 67 ------------------- 9 files changed, 27 insertions(+), 232 deletions(-) create mode 100644 docs/changelog/119227.yaml delete mode 100644 docs/reference/indices/apis/unfreeze.asciidoc delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json delete mode 100644 x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java diff --git a/docs/changelog/119227.yaml b/docs/changelog/119227.yaml new file mode 100644 index 0000000000000..1e3d4f97a3d27 --- /dev/null +++ b/docs/changelog/119227.yaml @@ -0,0 +1,13 @@ +pr: 119227 +summary: Remove unfreeze REST endpoint +area: Indices APIs +type: breaking +issues: [] +breaking: + title: Remove unfreeze REST endpoint + area: REST API + details: >- + The `/{index}/_unfreeze` REST endpoint is no longer supported. This API was deprecated, and the corresponding + `/{index}/_freeze` endpoint was removed in 8.0. + impact: None, since it is not possible to have a frozen index in a version which is readable by Elasticsearch 9.0 + notable: false diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc index ca7de396147a8..b6b82422cbb4a 100644 --- a/docs/reference/indices.asciidoc +++ b/docs/reference/indices.asciidoc @@ -24,7 +24,6 @@ index settings, aliases, mappings, and index templates. * <> * <> * <> -* <> * <> * <> * <> @@ -143,6 +142,5 @@ include::indices/shrink-index.asciidoc[] include::indices/simulate-index.asciidoc[] include::indices/simulate-template.asciidoc[] include::indices/split-index.asciidoc[] -include::indices/apis/unfreeze.asciidoc[] include::indices/update-settings.asciidoc[] include::indices/put-mapping.asciidoc[] diff --git a/docs/reference/indices/apis/unfreeze.asciidoc b/docs/reference/indices/apis/unfreeze.asciidoc deleted file mode 100644 index 5d04d44db7443..0000000000000 --- a/docs/reference/indices/apis/unfreeze.asciidoc +++ /dev/null @@ -1,61 +0,0 @@ -[role="xpack"] -[[unfreeze-index-api]] -=== Unfreeze index API -++++ -Unfreeze index -++++ - -[WARNING] -.Deprecated in 7.14 -==== -In 8.0, we removed the ability to freeze an index. In previous versions, -freezing an index reduced its memory overhead. However, frozen indices are no -longer useful due to -https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[recent -improvements in heap memory usage]. -You can use this API to unfreeze indices that were frozen in 7.x. Frozen indices -are not related to the frozen data tier. -==== - -.New API reference -[sidebar] --- -For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. --- - -Unfreezes an index. - -[[unfreeze-index-api-request]] -==== {api-request-title} - -`POST //_unfreeze` - -[[unfreeze-index-api-prereqs]] -==== {api-prereq-title} - -* If the {es} {security-features} are enabled, you must have the `manage` -<> for the target index or index alias. - -[[unfreeze-index-api-desc]] -==== {api-description-title} - -When a frozen index is unfrozen, the index goes through the normal recovery -process and becomes writeable again. - -[[unfreeze-index-api-path-parms]] -==== {api-path-parms-title} - -``:: - (Required, string) Identifier for the index. - -[[unfreeze-index-api-examples]] -==== {api-examples-title} - -The following example unfreezes an index: - -[source,console] --------------------------------------------------- -POST /my-index-000001/_unfreeze --------------------------------------------------- -// TEST[s/^/PUT my-index-000001\n/] -// TEST[skip:unable to ignore deprecation warning] diff --git a/docs/reference/indices/index-mgmt.asciidoc b/docs/reference/indices/index-mgmt.asciidoc index 73643dbfd4b3b..131bc79faa40c 100644 --- a/docs/reference/indices/index-mgmt.asciidoc +++ b/docs/reference/indices/index-mgmt.asciidoc @@ -43,7 +43,7 @@ For more information on managing indices, refer to <>. * To filter the list of indices, use the search bar or click a badge. Badges indicate if an index is a <>, a -<>, or <>. +<>, or <>. * To drill down into the index <>, <>, and statistics, diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index c3bf84fa600d2..9c0f0092214ed 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -156,10 +156,16 @@ See <>. The freeze index API was removed in 8.0. // tag::frozen-removal-explanation[] Frozen indices are no longer useful due to -https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[recent -improvements in heap memory usage]. +https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[improvements +in heap memory usage]. // end::frozen-removal-explanation[] +[role="exclude",id="unfreeze-index-api"] +=== Unfreeze index API + +The unfreeze index API was removed in 9.0. +include::redirects.asciidoc[tag=frozen-removal-explanation] + [role="exclude",id="ilm-freeze"] === Freeze {ilm-init} action @@ -1749,8 +1755,10 @@ See <>. === Frozen indices // tag::frozen-index-redirect[] - -For API documentation, see <>. +Older versions of {es} provided the option to reduce the amount of data kept in memory for an index, at the expense of +increasing search latency. This was known as 'freezing' the index. +include::redirects.asciidoc[tag=frozen-removal-explanation] +The freeze index API was removed in 8.0, and the unfreeze index API was removed in 9.0. // end::frozen-index-redirect[] [role="exclude",id="best_practices"] diff --git a/docs/reference/sql/language/indices.asciidoc b/docs/reference/sql/language/indices.asciidoc index 1dee7f0840ade..1912a020ab0be 100644 --- a/docs/reference/sql/language/indices.asciidoc +++ b/docs/reference/sql/language/indices.asciidoc @@ -100,7 +100,7 @@ requires the keyword `LIKE` for SQL `LIKE` pattern. [[sql-index-frozen]] === Frozen Indices -By default, {es-sql} doesn't search <>. To +By default, {es-sql} doesn't search <>. To search frozen indices, use one of the following features: dedicated configuration parameter:: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json deleted file mode 100644 index 2327519ff2816..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "indices.unfreeze":{ - "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/unfreeze-index-api.html", - "description":"Unfreezes an index. When a frozen index is unfrozen, the index goes through the normal recovery process and becomes writeable again." - }, - "stability":"stable", - "visibility":"public", - "headers":{ - "accept": [ "application/json"] - }, - "url":{ - "paths":[ - { - "path":"/{index}/_unfreeze", - "methods":[ - "POST" - ], - "parts":{ - "index":{ - "type":"string", - "description":"The name of the index to unfreeze" - } - }, - "deprecated":{ - "version":"7.14.0", - "description":"Frozen indices are deprecated because they provide no benefit given improvements in heap memory utilization. They will be removed in a future release." - } - } - ] - }, - "params":{ - "timeout":{ - "type":"time", - "description":"Explicit operation timeout" - }, - "master_timeout":{ - "type":"time", - "description":"Specify timeout for connection to master" - }, - "ignore_unavailable":{ - "type":"boolean", - "description":"Whether specified concrete indices should be ignored when unavailable (missing or closed)" - }, - "allow_no_indices":{ - "type":"boolean", - "description":"Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)" - }, - "expand_wildcards":{ - "type":"enum", - "options":[ - "open", - "closed", - "hidden", - "none", - "all" - ], - "default":"closed", - "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both." - }, - "wait_for_active_shards":{ - "type":"string", - "description":"Sets the number of active shards to wait for before the operation returns." - } - } - } -} diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java index 05b75fe6b01ca..0b106f6e304da 100644 --- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java +++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java @@ -8,36 +8,22 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.engine.frozen.FrozenEngine; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.rest.RestController; -import org.elasticsearch.rest.RestHandler; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; import org.elasticsearch.xpack.frozen.action.TransportFreezeIndexAction; -import org.elasticsearch.xpack.frozen.rest.action.RestFreezeIndexAction; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.function.Predicate; -import java.util.function.Supplier; public class FrozenIndices extends Plugin implements ActionPlugin, EnginePlugin { @@ -63,19 +49,4 @@ public List> getSettings() { actions.add(new ActionHandler<>(FreezeIndexAction.INSTANCE, TransportFreezeIndexAction.class)); return actions; } - - @Override - public List getRestHandlers( - Settings settings, - NamedWriteableRegistry namedWriteableRegistry, - RestController restController, - ClusterSettings clusterSettings, - IndexScopedSettings indexScopedSettings, - SettingsFilter settingsFilter, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster, - Predicate clusterSupportsFeature - ) { - return Collections.singletonList(new RestFreezeIndexAction()); - } } diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java deleted file mode 100644 index f07d17fee87f5..0000000000000 --- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.frozen.rest.action; - -import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.core.UpdateForV9; -import org.elasticsearch.protocol.xpack.frozen.FreezeRequest; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.RestToXContentListener; -import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; - -import java.util.List; - -import static org.elasticsearch.rest.RestRequest.Method.POST; -import static org.elasticsearch.rest.RestUtils.getAckTimeout; -import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; - -public final class RestFreezeIndexAction extends BaseRestHandler { - - private static final String FREEZE_REMOVED = "It is no longer possible to freeze indices, but existing frozen indices can still be " - + "unfrozen"; - - private static final String UNFREEZE_DEPRECATED = "Frozen indices are deprecated because they provide no benefit given improvements " - + "in heap memory utilization. They will be removed in a future release."; - - @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_INDEXING) - // these routes were ".deprecated" in RestApiVersion.V_8 which will require use of REST API compatibility headers to access - // this API in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will - // continue to emit deprecations warnings but will not require any special headers to access the API in v9. - // Please review and update the code and tests as needed. The original code remains commented out below for reference. - @Override - public List routes() { - return List.of( - // Route.builder(POST, "/{index}/_unfreeze").deprecated(UNFREEZE_DEPRECATED, RestApiVersion.V_8).build() - Route.builder(POST, "/{index}/_unfreeze").deprecateAndKeep(UNFREEZE_DEPRECATED).build() - ); - } - - @Override - protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { - final var freezeRequest = new FreezeRequest( - getMasterNodeTimeout(request), - getAckTimeout(request), - Strings.splitStringByCommaToArray(request.param("index")) - ); - freezeRequest.indicesOptions(IndicesOptions.fromRequest(request, freezeRequest.indicesOptions())); - String waitForActiveShards = request.param("wait_for_active_shards"); - if (waitForActiveShards != null) { - freezeRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); - } - freezeRequest.setFreeze(false); - return channel -> client.execute(FreezeIndexAction.INSTANCE, freezeRequest, new RestToXContentListener<>(channel)); - } - - @Override - public String getName() { - return "freeze_index"; - } -} From 60a54e6ab702ef2d6cf10ac0c0d5d4a4c1b528d0 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Jan 2025 21:38:09 +1100 Subject: [PATCH 37/44] [Test] Upgrade minio docker image used for tests (#120086) This PR upgrades the minio docker image from RELEASE.2021-03-01T04-20-55Z which is 3+ years old to the latest RELEASE.2024-12-18T13-15-44Z. Relates: #118548 --- .../s3/RepositoryS3MinioBasicCredentialsRestIT.java | 6 ++++-- .../test/fixtures/minio/MinioTestContainer.java | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java index 93915e8491d5b..3d7c8dd150610 100644 --- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java @@ -19,13 +19,15 @@ import org.junit.rules.RuleChain; import org.junit.rules.TestRule; +import java.util.Locale; + @ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 public class RepositoryS3MinioBasicCredentialsRestIT extends AbstractRepositoryS3RestTestCase { - private static final String PREFIX = getIdentifierPrefix("RepositoryS3MinioBasicCredentialsRestIT"); + private static final String PREFIX = getIdentifierPrefix("RepositoryS3MinioBasicCredentialsRestIT").toLowerCase(Locale.ROOT); private static final String BUCKET = PREFIX + "bucket"; - private static final String BASE_PATH = PREFIX + "base_path"; + private static final String BASE_PATH = PREFIX + "base-path"; private static final String ACCESS_KEY = PREFIX + "access-key"; private static final String SECRET_KEY = PREFIX + "secret-key"; private static final String CLIENT = "minio_client"; diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java index 3ee18d71a5a79..56e702d2f76a0 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java +++ b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java @@ -15,7 +15,7 @@ public final class MinioTestContainer extends DockerEnvironmentAwareTestContainer { private static final int servicePort = 9000; - public static final String DOCKER_BASE_IMAGE = "minio/minio:RELEASE.2021-03-01T04-20-55Z"; + public static final String DOCKER_BASE_IMAGE = "minio/minio:RELEASE.2024-12-18T13-15-44Z"; private final boolean enabled; public MinioTestContainer(boolean enabled, String accessKey, String secretKey, String bucketName) { From 212e16bc3a5f6bef412f2ba158d5ace7f62ea300 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Tue, 14 Jan 2025 12:06:35 +0100 Subject: [PATCH 38/44] ESQL: Drop support for brackets from METADATA syntax (#119846) This drops the deprecated support for having the METADATA clause provided within square brakets. Fixes #115401 --- docs/changelog/119846.yaml | 12 + .../esql/src/main/antlr/EsqlBaseParser.g4 | 9 - .../xpack/esql/parser/EsqlBaseParser.interp | 4 +- .../xpack/esql/parser/EsqlBaseParser.java | 1799 ++++++++--------- .../parser/EsqlBaseParserBaseListener.java | 24 - .../parser/EsqlBaseParserBaseVisitor.java | 14 - .../esql/parser/EsqlBaseParserListener.java | 20 - .../esql/parser/EsqlBaseParserVisitor.java | 12 - .../xpack/esql/parser/LogicalPlanBuilder.java | 18 +- .../esql/parser/StatementParserTests.java | 19 +- .../test/querying_cluster/80_esql.yml | 6 +- 11 files changed, 863 insertions(+), 1074 deletions(-) create mode 100644 docs/changelog/119846.yaml diff --git a/docs/changelog/119846.yaml b/docs/changelog/119846.yaml new file mode 100644 index 0000000000000..9e7d99fe1be13 --- /dev/null +++ b/docs/changelog/119846.yaml @@ -0,0 +1,12 @@ +pr: 119846 +summary: Drop support for brackets from METADATA syntax +area: ES|QL +type: deprecation +issues: + - 115401 +deprecation: + title: Drop support for brackets from METADATA syntax + area: ES|QL + details: Please describe the details of this change for the release notes. You can + use asciidoc. + impact: Please describe the impact of this change to users diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index efc2e36609902..4d4cb7d2caac4 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -143,18 +143,9 @@ indexString ; metadata - : metadataOption - | deprecated_metadata - ; - -metadataOption : METADATA UNQUOTED_SOURCE (COMMA UNQUOTED_SOURCE)* ; -deprecated_metadata - : OPENING_BRACKET metadataOption CLOSING_BRACKET - ; - metricsCommand : DEV_METRICS indexPattern (COMMA indexPattern)* aggregates=aggFields? (BY grouping=fields)? ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index c5b37fa411f65..492df7fbc1608 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -283,8 +283,6 @@ indexPattern clusterString indexString metadata -metadataOption -deprecated_metadata metricsCommand evalCommand statsCommand @@ -330,4 +328,4 @@ joinPredicate atn: -[4, 1, 128, 639, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 142, 8, 1, 10, 1, 12, 1, 145, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 153, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 173, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 185, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 192, 8, 5, 10, 5, 12, 5, 195, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 202, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 207, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 215, 8, 5, 10, 5, 12, 5, 218, 9, 5, 1, 6, 1, 6, 3, 6, 222, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 229, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 234, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 239, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 249, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 255, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 263, 8, 9, 10, 9, 12, 9, 266, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 276, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 281, 8, 10, 10, 10, 12, 10, 284, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 292, 8, 11, 10, 11, 12, 11, 295, 9, 11, 3, 11, 297, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 311, 8, 15, 10, 15, 12, 15, 314, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 319, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 327, 8, 17, 10, 17, 12, 17, 330, 9, 17, 1, 17, 3, 17, 333, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 338, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 3, 21, 348, 8, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 354, 8, 22, 10, 22, 12, 22, 357, 9, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 367, 8, 24, 10, 24, 12, 24, 370, 9, 24, 1, 24, 3, 24, 373, 8, 24, 1, 24, 1, 24, 3, 24, 377, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 384, 8, 26, 1, 26, 1, 26, 3, 26, 388, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 393, 8, 27, 10, 27, 12, 27, 396, 9, 27, 1, 28, 1, 28, 1, 28, 3, 28, 401, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 406, 8, 29, 10, 29, 12, 29, 409, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 414, 8, 30, 10, 30, 12, 30, 417, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 422, 8, 31, 10, 31, 12, 31, 425, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 432, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 447, 8, 34, 10, 34, 12, 34, 450, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 458, 8, 34, 10, 34, 12, 34, 461, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 469, 8, 34, 10, 34, 12, 34, 472, 9, 34, 1, 34, 1, 34, 3, 34, 476, 8, 34, 1, 35, 1, 35, 3, 35, 480, 8, 35, 1, 36, 1, 36, 1, 36, 3, 36, 485, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 494, 8, 38, 10, 38, 12, 38, 497, 9, 38, 1, 39, 1, 39, 3, 39, 501, 8, 39, 1, 39, 1, 39, 3, 39, 505, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 517, 8, 42, 10, 42, 12, 42, 520, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 530, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 542, 8, 47, 10, 47, 12, 47, 545, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 555, 8, 50, 1, 51, 3, 51, 558, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 563, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 585, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 591, 8, 58, 10, 58, 12, 58, 594, 9, 58, 3, 58, 596, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 601, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 614, 8, 61, 1, 62, 3, 62, 617, 8, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 3, 63, 626, 8, 63, 1, 64, 1, 64, 1, 64, 1, 64, 5, 64, 632, 8, 64, 10, 64, 12, 64, 635, 9, 64, 1, 65, 1, 65, 1, 65, 0, 4, 2, 10, 18, 20, 66, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 0, 9, 1, 0, 64, 65, 1, 0, 66, 68, 2, 0, 30, 30, 81, 81, 1, 0, 72, 73, 2, 0, 35, 35, 40, 40, 2, 0, 43, 43, 46, 46, 2, 0, 42, 42, 56, 56, 2, 0, 57, 57, 59, 63, 1, 0, 22, 24, 665, 0, 132, 1, 0, 0, 0, 2, 135, 1, 0, 0, 0, 4, 152, 1, 0, 0, 0, 6, 172, 1, 0, 0, 0, 8, 174, 1, 0, 0, 0, 10, 206, 1, 0, 0, 0, 12, 233, 1, 0, 0, 0, 14, 235, 1, 0, 0, 0, 16, 248, 1, 0, 0, 0, 18, 254, 1, 0, 0, 0, 20, 275, 1, 0, 0, 0, 22, 285, 1, 0, 0, 0, 24, 300, 1, 0, 0, 0, 26, 302, 1, 0, 0, 0, 28, 304, 1, 0, 0, 0, 30, 307, 1, 0, 0, 0, 32, 318, 1, 0, 0, 0, 34, 322, 1, 0, 0, 0, 36, 337, 1, 0, 0, 0, 38, 341, 1, 0, 0, 0, 40, 343, 1, 0, 0, 0, 42, 347, 1, 0, 0, 0, 44, 349, 1, 0, 0, 0, 46, 358, 1, 0, 0, 0, 48, 362, 1, 0, 0, 0, 50, 378, 1, 0, 0, 0, 52, 381, 1, 0, 0, 0, 54, 389, 1, 0, 0, 0, 56, 397, 1, 0, 0, 0, 58, 402, 1, 0, 0, 0, 60, 410, 1, 0, 0, 0, 62, 418, 1, 0, 0, 0, 64, 426, 1, 0, 0, 0, 66, 431, 1, 0, 0, 0, 68, 475, 1, 0, 0, 0, 70, 479, 1, 0, 0, 0, 72, 484, 1, 0, 0, 0, 74, 486, 1, 0, 0, 0, 76, 489, 1, 0, 0, 0, 78, 498, 1, 0, 0, 0, 80, 506, 1, 0, 0, 0, 82, 509, 1, 0, 0, 0, 84, 512, 1, 0, 0, 0, 86, 521, 1, 0, 0, 0, 88, 525, 1, 0, 0, 0, 90, 531, 1, 0, 0, 0, 92, 535, 1, 0, 0, 0, 94, 538, 1, 0, 0, 0, 96, 546, 1, 0, 0, 0, 98, 550, 1, 0, 0, 0, 100, 554, 1, 0, 0, 0, 102, 557, 1, 0, 0, 0, 104, 562, 1, 0, 0, 0, 106, 566, 1, 0, 0, 0, 108, 568, 1, 0, 0, 0, 110, 570, 1, 0, 0, 0, 112, 573, 1, 0, 0, 0, 114, 577, 1, 0, 0, 0, 116, 580, 1, 0, 0, 0, 118, 600, 1, 0, 0, 0, 120, 604, 1, 0, 0, 0, 122, 609, 1, 0, 0, 0, 124, 616, 1, 0, 0, 0, 126, 622, 1, 0, 0, 0, 128, 627, 1, 0, 0, 0, 130, 636, 1, 0, 0, 0, 132, 133, 3, 2, 1, 0, 133, 134, 5, 0, 0, 1, 134, 1, 1, 0, 0, 0, 135, 136, 6, 1, -1, 0, 136, 137, 3, 4, 2, 0, 137, 143, 1, 0, 0, 0, 138, 139, 10, 1, 0, 0, 139, 140, 5, 29, 0, 0, 140, 142, 3, 6, 3, 0, 141, 138, 1, 0, 0, 0, 142, 145, 1, 0, 0, 0, 143, 141, 1, 0, 0, 0, 143, 144, 1, 0, 0, 0, 144, 3, 1, 0, 0, 0, 145, 143, 1, 0, 0, 0, 146, 153, 3, 110, 55, 0, 147, 153, 3, 34, 17, 0, 148, 153, 3, 28, 14, 0, 149, 153, 3, 114, 57, 0, 150, 151, 4, 2, 1, 0, 151, 153, 3, 48, 24, 0, 152, 146, 1, 0, 0, 0, 152, 147, 1, 0, 0, 0, 152, 148, 1, 0, 0, 0, 152, 149, 1, 0, 0, 0, 152, 150, 1, 0, 0, 0, 153, 5, 1, 0, 0, 0, 154, 173, 3, 50, 25, 0, 155, 173, 3, 8, 4, 0, 156, 173, 3, 80, 40, 0, 157, 173, 3, 74, 37, 0, 158, 173, 3, 52, 26, 0, 159, 173, 3, 76, 38, 0, 160, 173, 3, 82, 41, 0, 161, 173, 3, 84, 42, 0, 162, 173, 3, 88, 44, 0, 163, 173, 3, 90, 45, 0, 164, 173, 3, 116, 58, 0, 165, 173, 3, 92, 46, 0, 166, 167, 4, 3, 2, 0, 167, 173, 3, 122, 61, 0, 168, 169, 4, 3, 3, 0, 169, 173, 3, 120, 60, 0, 170, 171, 4, 3, 4, 0, 171, 173, 3, 124, 62, 0, 172, 154, 1, 0, 0, 0, 172, 155, 1, 0, 0, 0, 172, 156, 1, 0, 0, 0, 172, 157, 1, 0, 0, 0, 172, 158, 1, 0, 0, 0, 172, 159, 1, 0, 0, 0, 172, 160, 1, 0, 0, 0, 172, 161, 1, 0, 0, 0, 172, 162, 1, 0, 0, 0, 172, 163, 1, 0, 0, 0, 172, 164, 1, 0, 0, 0, 172, 165, 1, 0, 0, 0, 172, 166, 1, 0, 0, 0, 172, 168, 1, 0, 0, 0, 172, 170, 1, 0, 0, 0, 173, 7, 1, 0, 0, 0, 174, 175, 5, 16, 0, 0, 175, 176, 3, 10, 5, 0, 176, 9, 1, 0, 0, 0, 177, 178, 6, 5, -1, 0, 178, 179, 5, 49, 0, 0, 179, 207, 3, 10, 5, 8, 180, 207, 3, 16, 8, 0, 181, 207, 3, 12, 6, 0, 182, 184, 3, 16, 8, 0, 183, 185, 5, 49, 0, 0, 184, 183, 1, 0, 0, 0, 184, 185, 1, 0, 0, 0, 185, 186, 1, 0, 0, 0, 186, 187, 5, 44, 0, 0, 187, 188, 5, 48, 0, 0, 188, 193, 3, 16, 8, 0, 189, 190, 5, 39, 0, 0, 190, 192, 3, 16, 8, 0, 191, 189, 1, 0, 0, 0, 192, 195, 1, 0, 0, 0, 193, 191, 1, 0, 0, 0, 193, 194, 1, 0, 0, 0, 194, 196, 1, 0, 0, 0, 195, 193, 1, 0, 0, 0, 196, 197, 5, 55, 0, 0, 197, 207, 1, 0, 0, 0, 198, 199, 3, 16, 8, 0, 199, 201, 5, 45, 0, 0, 200, 202, 5, 49, 0, 0, 201, 200, 1, 0, 0, 0, 201, 202, 1, 0, 0, 0, 202, 203, 1, 0, 0, 0, 203, 204, 5, 50, 0, 0, 204, 207, 1, 0, 0, 0, 205, 207, 3, 14, 7, 0, 206, 177, 1, 0, 0, 0, 206, 180, 1, 0, 0, 0, 206, 181, 1, 0, 0, 0, 206, 182, 1, 0, 0, 0, 206, 198, 1, 0, 0, 0, 206, 205, 1, 0, 0, 0, 207, 216, 1, 0, 0, 0, 208, 209, 10, 5, 0, 0, 209, 210, 5, 34, 0, 0, 210, 215, 3, 10, 5, 6, 211, 212, 10, 4, 0, 0, 212, 213, 5, 52, 0, 0, 213, 215, 3, 10, 5, 5, 214, 208, 1, 0, 0, 0, 214, 211, 1, 0, 0, 0, 215, 218, 1, 0, 0, 0, 216, 214, 1, 0, 0, 0, 216, 217, 1, 0, 0, 0, 217, 11, 1, 0, 0, 0, 218, 216, 1, 0, 0, 0, 219, 221, 3, 16, 8, 0, 220, 222, 5, 49, 0, 0, 221, 220, 1, 0, 0, 0, 221, 222, 1, 0, 0, 0, 222, 223, 1, 0, 0, 0, 223, 224, 5, 47, 0, 0, 224, 225, 3, 106, 53, 0, 225, 234, 1, 0, 0, 0, 226, 228, 3, 16, 8, 0, 227, 229, 5, 49, 0, 0, 228, 227, 1, 0, 0, 0, 228, 229, 1, 0, 0, 0, 229, 230, 1, 0, 0, 0, 230, 231, 5, 54, 0, 0, 231, 232, 3, 106, 53, 0, 232, 234, 1, 0, 0, 0, 233, 219, 1, 0, 0, 0, 233, 226, 1, 0, 0, 0, 234, 13, 1, 0, 0, 0, 235, 238, 3, 58, 29, 0, 236, 237, 5, 37, 0, 0, 237, 239, 3, 26, 13, 0, 238, 236, 1, 0, 0, 0, 238, 239, 1, 0, 0, 0, 239, 240, 1, 0, 0, 0, 240, 241, 5, 38, 0, 0, 241, 242, 3, 68, 34, 0, 242, 15, 1, 0, 0, 0, 243, 249, 3, 18, 9, 0, 244, 245, 3, 18, 9, 0, 245, 246, 3, 108, 54, 0, 246, 247, 3, 18, 9, 0, 247, 249, 1, 0, 0, 0, 248, 243, 1, 0, 0, 0, 248, 244, 1, 0, 0, 0, 249, 17, 1, 0, 0, 0, 250, 251, 6, 9, -1, 0, 251, 255, 3, 20, 10, 0, 252, 253, 7, 0, 0, 0, 253, 255, 3, 18, 9, 3, 254, 250, 1, 0, 0, 0, 254, 252, 1, 0, 0, 0, 255, 264, 1, 0, 0, 0, 256, 257, 10, 2, 0, 0, 257, 258, 7, 1, 0, 0, 258, 263, 3, 18, 9, 3, 259, 260, 10, 1, 0, 0, 260, 261, 7, 0, 0, 0, 261, 263, 3, 18, 9, 2, 262, 256, 1, 0, 0, 0, 262, 259, 1, 0, 0, 0, 263, 266, 1, 0, 0, 0, 264, 262, 1, 0, 0, 0, 264, 265, 1, 0, 0, 0, 265, 19, 1, 0, 0, 0, 266, 264, 1, 0, 0, 0, 267, 268, 6, 10, -1, 0, 268, 276, 3, 68, 34, 0, 269, 276, 3, 58, 29, 0, 270, 276, 3, 22, 11, 0, 271, 272, 5, 48, 0, 0, 272, 273, 3, 10, 5, 0, 273, 274, 5, 55, 0, 0, 274, 276, 1, 0, 0, 0, 275, 267, 1, 0, 0, 0, 275, 269, 1, 0, 0, 0, 275, 270, 1, 0, 0, 0, 275, 271, 1, 0, 0, 0, 276, 282, 1, 0, 0, 0, 277, 278, 10, 1, 0, 0, 278, 279, 5, 37, 0, 0, 279, 281, 3, 26, 13, 0, 280, 277, 1, 0, 0, 0, 281, 284, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 282, 283, 1, 0, 0, 0, 283, 21, 1, 0, 0, 0, 284, 282, 1, 0, 0, 0, 285, 286, 3, 24, 12, 0, 286, 296, 5, 48, 0, 0, 287, 297, 5, 66, 0, 0, 288, 293, 3, 10, 5, 0, 289, 290, 5, 39, 0, 0, 290, 292, 3, 10, 5, 0, 291, 289, 1, 0, 0, 0, 292, 295, 1, 0, 0, 0, 293, 291, 1, 0, 0, 0, 293, 294, 1, 0, 0, 0, 294, 297, 1, 0, 0, 0, 295, 293, 1, 0, 0, 0, 296, 287, 1, 0, 0, 0, 296, 288, 1, 0, 0, 0, 296, 297, 1, 0, 0, 0, 297, 298, 1, 0, 0, 0, 298, 299, 5, 55, 0, 0, 299, 23, 1, 0, 0, 0, 300, 301, 3, 72, 36, 0, 301, 25, 1, 0, 0, 0, 302, 303, 3, 64, 32, 0, 303, 27, 1, 0, 0, 0, 304, 305, 5, 12, 0, 0, 305, 306, 3, 30, 15, 0, 306, 29, 1, 0, 0, 0, 307, 312, 3, 32, 16, 0, 308, 309, 5, 39, 0, 0, 309, 311, 3, 32, 16, 0, 310, 308, 1, 0, 0, 0, 311, 314, 1, 0, 0, 0, 312, 310, 1, 0, 0, 0, 312, 313, 1, 0, 0, 0, 313, 31, 1, 0, 0, 0, 314, 312, 1, 0, 0, 0, 315, 316, 3, 58, 29, 0, 316, 317, 5, 36, 0, 0, 317, 319, 1, 0, 0, 0, 318, 315, 1, 0, 0, 0, 318, 319, 1, 0, 0, 0, 319, 320, 1, 0, 0, 0, 320, 321, 3, 10, 5, 0, 321, 33, 1, 0, 0, 0, 322, 323, 5, 6, 0, 0, 323, 328, 3, 36, 18, 0, 324, 325, 5, 39, 0, 0, 325, 327, 3, 36, 18, 0, 326, 324, 1, 0, 0, 0, 327, 330, 1, 0, 0, 0, 328, 326, 1, 0, 0, 0, 328, 329, 1, 0, 0, 0, 329, 332, 1, 0, 0, 0, 330, 328, 1, 0, 0, 0, 331, 333, 3, 42, 21, 0, 332, 331, 1, 0, 0, 0, 332, 333, 1, 0, 0, 0, 333, 35, 1, 0, 0, 0, 334, 335, 3, 38, 19, 0, 335, 336, 5, 38, 0, 0, 336, 338, 1, 0, 0, 0, 337, 334, 1, 0, 0, 0, 337, 338, 1, 0, 0, 0, 338, 339, 1, 0, 0, 0, 339, 340, 3, 40, 20, 0, 340, 37, 1, 0, 0, 0, 341, 342, 5, 81, 0, 0, 342, 39, 1, 0, 0, 0, 343, 344, 7, 2, 0, 0, 344, 41, 1, 0, 0, 0, 345, 348, 3, 44, 22, 0, 346, 348, 3, 46, 23, 0, 347, 345, 1, 0, 0, 0, 347, 346, 1, 0, 0, 0, 348, 43, 1, 0, 0, 0, 349, 350, 5, 80, 0, 0, 350, 355, 5, 81, 0, 0, 351, 352, 5, 39, 0, 0, 352, 354, 5, 81, 0, 0, 353, 351, 1, 0, 0, 0, 354, 357, 1, 0, 0, 0, 355, 353, 1, 0, 0, 0, 355, 356, 1, 0, 0, 0, 356, 45, 1, 0, 0, 0, 357, 355, 1, 0, 0, 0, 358, 359, 5, 70, 0, 0, 359, 360, 3, 44, 22, 0, 360, 361, 5, 71, 0, 0, 361, 47, 1, 0, 0, 0, 362, 363, 5, 19, 0, 0, 363, 368, 3, 36, 18, 0, 364, 365, 5, 39, 0, 0, 365, 367, 3, 36, 18, 0, 366, 364, 1, 0, 0, 0, 367, 370, 1, 0, 0, 0, 368, 366, 1, 0, 0, 0, 368, 369, 1, 0, 0, 0, 369, 372, 1, 0, 0, 0, 370, 368, 1, 0, 0, 0, 371, 373, 3, 54, 27, 0, 372, 371, 1, 0, 0, 0, 372, 373, 1, 0, 0, 0, 373, 376, 1, 0, 0, 0, 374, 375, 5, 33, 0, 0, 375, 377, 3, 30, 15, 0, 376, 374, 1, 0, 0, 0, 376, 377, 1, 0, 0, 0, 377, 49, 1, 0, 0, 0, 378, 379, 5, 4, 0, 0, 379, 380, 3, 30, 15, 0, 380, 51, 1, 0, 0, 0, 381, 383, 5, 15, 0, 0, 382, 384, 3, 54, 27, 0, 383, 382, 1, 0, 0, 0, 383, 384, 1, 0, 0, 0, 384, 387, 1, 0, 0, 0, 385, 386, 5, 33, 0, 0, 386, 388, 3, 30, 15, 0, 387, 385, 1, 0, 0, 0, 387, 388, 1, 0, 0, 0, 388, 53, 1, 0, 0, 0, 389, 394, 3, 56, 28, 0, 390, 391, 5, 39, 0, 0, 391, 393, 3, 56, 28, 0, 392, 390, 1, 0, 0, 0, 393, 396, 1, 0, 0, 0, 394, 392, 1, 0, 0, 0, 394, 395, 1, 0, 0, 0, 395, 55, 1, 0, 0, 0, 396, 394, 1, 0, 0, 0, 397, 400, 3, 32, 16, 0, 398, 399, 5, 16, 0, 0, 399, 401, 3, 10, 5, 0, 400, 398, 1, 0, 0, 0, 400, 401, 1, 0, 0, 0, 401, 57, 1, 0, 0, 0, 402, 407, 3, 72, 36, 0, 403, 404, 5, 41, 0, 0, 404, 406, 3, 72, 36, 0, 405, 403, 1, 0, 0, 0, 406, 409, 1, 0, 0, 0, 407, 405, 1, 0, 0, 0, 407, 408, 1, 0, 0, 0, 408, 59, 1, 0, 0, 0, 409, 407, 1, 0, 0, 0, 410, 415, 3, 66, 33, 0, 411, 412, 5, 41, 0, 0, 412, 414, 3, 66, 33, 0, 413, 411, 1, 0, 0, 0, 414, 417, 1, 0, 0, 0, 415, 413, 1, 0, 0, 0, 415, 416, 1, 0, 0, 0, 416, 61, 1, 0, 0, 0, 417, 415, 1, 0, 0, 0, 418, 423, 3, 60, 30, 0, 419, 420, 5, 39, 0, 0, 420, 422, 3, 60, 30, 0, 421, 419, 1, 0, 0, 0, 422, 425, 1, 0, 0, 0, 423, 421, 1, 0, 0, 0, 423, 424, 1, 0, 0, 0, 424, 63, 1, 0, 0, 0, 425, 423, 1, 0, 0, 0, 426, 427, 7, 3, 0, 0, 427, 65, 1, 0, 0, 0, 428, 432, 5, 85, 0, 0, 429, 430, 4, 33, 10, 0, 430, 432, 3, 70, 35, 0, 431, 428, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 432, 67, 1, 0, 0, 0, 433, 476, 5, 50, 0, 0, 434, 435, 3, 104, 52, 0, 435, 436, 5, 72, 0, 0, 436, 476, 1, 0, 0, 0, 437, 476, 3, 102, 51, 0, 438, 476, 3, 104, 52, 0, 439, 476, 3, 98, 49, 0, 440, 476, 3, 70, 35, 0, 441, 476, 3, 106, 53, 0, 442, 443, 5, 70, 0, 0, 443, 448, 3, 100, 50, 0, 444, 445, 5, 39, 0, 0, 445, 447, 3, 100, 50, 0, 446, 444, 1, 0, 0, 0, 447, 450, 1, 0, 0, 0, 448, 446, 1, 0, 0, 0, 448, 449, 1, 0, 0, 0, 449, 451, 1, 0, 0, 0, 450, 448, 1, 0, 0, 0, 451, 452, 5, 71, 0, 0, 452, 476, 1, 0, 0, 0, 453, 454, 5, 70, 0, 0, 454, 459, 3, 98, 49, 0, 455, 456, 5, 39, 0, 0, 456, 458, 3, 98, 49, 0, 457, 455, 1, 0, 0, 0, 458, 461, 1, 0, 0, 0, 459, 457, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 462, 1, 0, 0, 0, 461, 459, 1, 0, 0, 0, 462, 463, 5, 71, 0, 0, 463, 476, 1, 0, 0, 0, 464, 465, 5, 70, 0, 0, 465, 470, 3, 106, 53, 0, 466, 467, 5, 39, 0, 0, 467, 469, 3, 106, 53, 0, 468, 466, 1, 0, 0, 0, 469, 472, 1, 0, 0, 0, 470, 468, 1, 0, 0, 0, 470, 471, 1, 0, 0, 0, 471, 473, 1, 0, 0, 0, 472, 470, 1, 0, 0, 0, 473, 474, 5, 71, 0, 0, 474, 476, 1, 0, 0, 0, 475, 433, 1, 0, 0, 0, 475, 434, 1, 0, 0, 0, 475, 437, 1, 0, 0, 0, 475, 438, 1, 0, 0, 0, 475, 439, 1, 0, 0, 0, 475, 440, 1, 0, 0, 0, 475, 441, 1, 0, 0, 0, 475, 442, 1, 0, 0, 0, 475, 453, 1, 0, 0, 0, 475, 464, 1, 0, 0, 0, 476, 69, 1, 0, 0, 0, 477, 480, 5, 53, 0, 0, 478, 480, 5, 69, 0, 0, 479, 477, 1, 0, 0, 0, 479, 478, 1, 0, 0, 0, 480, 71, 1, 0, 0, 0, 481, 485, 3, 64, 32, 0, 482, 483, 4, 36, 11, 0, 483, 485, 3, 70, 35, 0, 484, 481, 1, 0, 0, 0, 484, 482, 1, 0, 0, 0, 485, 73, 1, 0, 0, 0, 486, 487, 5, 9, 0, 0, 487, 488, 5, 31, 0, 0, 488, 75, 1, 0, 0, 0, 489, 490, 5, 14, 0, 0, 490, 495, 3, 78, 39, 0, 491, 492, 5, 39, 0, 0, 492, 494, 3, 78, 39, 0, 493, 491, 1, 0, 0, 0, 494, 497, 1, 0, 0, 0, 495, 493, 1, 0, 0, 0, 495, 496, 1, 0, 0, 0, 496, 77, 1, 0, 0, 0, 497, 495, 1, 0, 0, 0, 498, 500, 3, 10, 5, 0, 499, 501, 7, 4, 0, 0, 500, 499, 1, 0, 0, 0, 500, 501, 1, 0, 0, 0, 501, 504, 1, 0, 0, 0, 502, 503, 5, 51, 0, 0, 503, 505, 7, 5, 0, 0, 504, 502, 1, 0, 0, 0, 504, 505, 1, 0, 0, 0, 505, 79, 1, 0, 0, 0, 506, 507, 5, 8, 0, 0, 507, 508, 3, 62, 31, 0, 508, 81, 1, 0, 0, 0, 509, 510, 5, 2, 0, 0, 510, 511, 3, 62, 31, 0, 511, 83, 1, 0, 0, 0, 512, 513, 5, 11, 0, 0, 513, 518, 3, 86, 43, 0, 514, 515, 5, 39, 0, 0, 515, 517, 3, 86, 43, 0, 516, 514, 1, 0, 0, 0, 517, 520, 1, 0, 0, 0, 518, 516, 1, 0, 0, 0, 518, 519, 1, 0, 0, 0, 519, 85, 1, 0, 0, 0, 520, 518, 1, 0, 0, 0, 521, 522, 3, 60, 30, 0, 522, 523, 5, 89, 0, 0, 523, 524, 3, 60, 30, 0, 524, 87, 1, 0, 0, 0, 525, 526, 5, 1, 0, 0, 526, 527, 3, 20, 10, 0, 527, 529, 3, 106, 53, 0, 528, 530, 3, 94, 47, 0, 529, 528, 1, 0, 0, 0, 529, 530, 1, 0, 0, 0, 530, 89, 1, 0, 0, 0, 531, 532, 5, 7, 0, 0, 532, 533, 3, 20, 10, 0, 533, 534, 3, 106, 53, 0, 534, 91, 1, 0, 0, 0, 535, 536, 5, 10, 0, 0, 536, 537, 3, 58, 29, 0, 537, 93, 1, 0, 0, 0, 538, 543, 3, 96, 48, 0, 539, 540, 5, 39, 0, 0, 540, 542, 3, 96, 48, 0, 541, 539, 1, 0, 0, 0, 542, 545, 1, 0, 0, 0, 543, 541, 1, 0, 0, 0, 543, 544, 1, 0, 0, 0, 544, 95, 1, 0, 0, 0, 545, 543, 1, 0, 0, 0, 546, 547, 3, 64, 32, 0, 547, 548, 5, 36, 0, 0, 548, 549, 3, 68, 34, 0, 549, 97, 1, 0, 0, 0, 550, 551, 7, 6, 0, 0, 551, 99, 1, 0, 0, 0, 552, 555, 3, 102, 51, 0, 553, 555, 3, 104, 52, 0, 554, 552, 1, 0, 0, 0, 554, 553, 1, 0, 0, 0, 555, 101, 1, 0, 0, 0, 556, 558, 7, 0, 0, 0, 557, 556, 1, 0, 0, 0, 557, 558, 1, 0, 0, 0, 558, 559, 1, 0, 0, 0, 559, 560, 5, 32, 0, 0, 560, 103, 1, 0, 0, 0, 561, 563, 7, 0, 0, 0, 562, 561, 1, 0, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 1, 0, 0, 0, 564, 565, 5, 31, 0, 0, 565, 105, 1, 0, 0, 0, 566, 567, 5, 30, 0, 0, 567, 107, 1, 0, 0, 0, 568, 569, 7, 7, 0, 0, 569, 109, 1, 0, 0, 0, 570, 571, 5, 5, 0, 0, 571, 572, 3, 112, 56, 0, 572, 111, 1, 0, 0, 0, 573, 574, 5, 70, 0, 0, 574, 575, 3, 2, 1, 0, 575, 576, 5, 71, 0, 0, 576, 113, 1, 0, 0, 0, 577, 578, 5, 13, 0, 0, 578, 579, 5, 105, 0, 0, 579, 115, 1, 0, 0, 0, 580, 581, 5, 3, 0, 0, 581, 584, 5, 95, 0, 0, 582, 583, 5, 93, 0, 0, 583, 585, 3, 60, 30, 0, 584, 582, 1, 0, 0, 0, 584, 585, 1, 0, 0, 0, 585, 595, 1, 0, 0, 0, 586, 587, 5, 94, 0, 0, 587, 592, 3, 118, 59, 0, 588, 589, 5, 39, 0, 0, 589, 591, 3, 118, 59, 0, 590, 588, 1, 0, 0, 0, 591, 594, 1, 0, 0, 0, 592, 590, 1, 0, 0, 0, 592, 593, 1, 0, 0, 0, 593, 596, 1, 0, 0, 0, 594, 592, 1, 0, 0, 0, 595, 586, 1, 0, 0, 0, 595, 596, 1, 0, 0, 0, 596, 117, 1, 0, 0, 0, 597, 598, 3, 60, 30, 0, 598, 599, 5, 36, 0, 0, 599, 601, 1, 0, 0, 0, 600, 597, 1, 0, 0, 0, 600, 601, 1, 0, 0, 0, 601, 602, 1, 0, 0, 0, 602, 603, 3, 60, 30, 0, 603, 119, 1, 0, 0, 0, 604, 605, 5, 18, 0, 0, 605, 606, 3, 36, 18, 0, 606, 607, 5, 93, 0, 0, 607, 608, 3, 62, 31, 0, 608, 121, 1, 0, 0, 0, 609, 610, 5, 17, 0, 0, 610, 613, 3, 54, 27, 0, 611, 612, 5, 33, 0, 0, 612, 614, 3, 30, 15, 0, 613, 611, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 123, 1, 0, 0, 0, 615, 617, 7, 8, 0, 0, 616, 615, 1, 0, 0, 0, 616, 617, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 619, 5, 20, 0, 0, 619, 620, 3, 126, 63, 0, 620, 621, 3, 128, 64, 0, 621, 125, 1, 0, 0, 0, 622, 625, 3, 64, 32, 0, 623, 624, 5, 89, 0, 0, 624, 626, 3, 64, 32, 0, 625, 623, 1, 0, 0, 0, 625, 626, 1, 0, 0, 0, 626, 127, 1, 0, 0, 0, 627, 628, 5, 93, 0, 0, 628, 633, 3, 130, 65, 0, 629, 630, 5, 39, 0, 0, 630, 632, 3, 130, 65, 0, 631, 629, 1, 0, 0, 0, 632, 635, 1, 0, 0, 0, 633, 631, 1, 0, 0, 0, 633, 634, 1, 0, 0, 0, 634, 129, 1, 0, 0, 0, 635, 633, 1, 0, 0, 0, 636, 637, 3, 16, 8, 0, 637, 131, 1, 0, 0, 0, 62, 143, 152, 172, 184, 193, 201, 206, 214, 216, 221, 228, 233, 238, 248, 254, 262, 264, 275, 282, 293, 296, 312, 318, 328, 332, 337, 347, 355, 368, 372, 376, 383, 387, 394, 400, 407, 415, 423, 431, 448, 459, 470, 475, 479, 484, 495, 500, 504, 518, 529, 543, 554, 557, 562, 584, 592, 595, 600, 613, 616, 625, 633] \ No newline at end of file +[4, 1, 128, 627, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 138, 8, 1, 10, 1, 12, 1, 141, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 149, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 169, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 181, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 188, 8, 5, 10, 5, 12, 5, 191, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 198, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 203, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 211, 8, 5, 10, 5, 12, 5, 214, 9, 5, 1, 6, 1, 6, 3, 6, 218, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 225, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 230, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 235, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 245, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 251, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 259, 8, 9, 10, 9, 12, 9, 262, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 272, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 277, 8, 10, 10, 10, 12, 10, 280, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 288, 8, 11, 10, 11, 12, 11, 291, 9, 11, 3, 11, 293, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 307, 8, 15, 10, 15, 12, 15, 310, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 315, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 323, 8, 17, 10, 17, 12, 17, 326, 9, 17, 1, 17, 3, 17, 329, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 334, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 346, 8, 21, 10, 21, 12, 21, 349, 9, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 355, 8, 22, 10, 22, 12, 22, 358, 9, 22, 1, 22, 3, 22, 361, 8, 22, 1, 22, 1, 22, 3, 22, 365, 8, 22, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 3, 24, 372, 8, 24, 1, 24, 1, 24, 3, 24, 376, 8, 24, 1, 25, 1, 25, 1, 25, 5, 25, 381, 8, 25, 10, 25, 12, 25, 384, 9, 25, 1, 26, 1, 26, 1, 26, 3, 26, 389, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 394, 8, 27, 10, 27, 12, 27, 397, 9, 27, 1, 28, 1, 28, 1, 28, 5, 28, 402, 8, 28, 10, 28, 12, 28, 405, 9, 28, 1, 29, 1, 29, 1, 29, 5, 29, 410, 8, 29, 10, 29, 12, 29, 413, 9, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 3, 31, 420, 8, 31, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 435, 8, 32, 10, 32, 12, 32, 438, 9, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 446, 8, 32, 10, 32, 12, 32, 449, 9, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 457, 8, 32, 10, 32, 12, 32, 460, 9, 32, 1, 32, 1, 32, 3, 32, 464, 8, 32, 1, 33, 1, 33, 3, 33, 468, 8, 33, 1, 34, 1, 34, 1, 34, 3, 34, 473, 8, 34, 1, 35, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 482, 8, 36, 10, 36, 12, 36, 485, 9, 36, 1, 37, 1, 37, 3, 37, 489, 8, 37, 1, 37, 1, 37, 3, 37, 493, 8, 37, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 5, 40, 505, 8, 40, 10, 40, 12, 40, 508, 9, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 3, 42, 518, 8, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 5, 45, 530, 8, 45, 10, 45, 12, 45, 533, 9, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 3, 48, 543, 8, 48, 1, 49, 3, 49, 546, 8, 49, 1, 49, 1, 49, 1, 50, 3, 50, 551, 8, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 3, 56, 573, 8, 56, 1, 56, 1, 56, 1, 56, 1, 56, 5, 56, 579, 8, 56, 10, 56, 12, 56, 582, 9, 56, 3, 56, 584, 8, 56, 1, 57, 1, 57, 1, 57, 3, 57, 589, 8, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 3, 59, 602, 8, 59, 1, 60, 3, 60, 605, 8, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 3, 61, 614, 8, 61, 1, 62, 1, 62, 1, 62, 1, 62, 5, 62, 620, 8, 62, 10, 62, 12, 62, 623, 9, 62, 1, 63, 1, 63, 1, 63, 0, 4, 2, 10, 18, 20, 64, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 0, 9, 1, 0, 64, 65, 1, 0, 66, 68, 2, 0, 30, 30, 81, 81, 1, 0, 72, 73, 2, 0, 35, 35, 40, 40, 2, 0, 43, 43, 46, 46, 2, 0, 42, 42, 56, 56, 2, 0, 57, 57, 59, 63, 1, 0, 22, 24, 654, 0, 128, 1, 0, 0, 0, 2, 131, 1, 0, 0, 0, 4, 148, 1, 0, 0, 0, 6, 168, 1, 0, 0, 0, 8, 170, 1, 0, 0, 0, 10, 202, 1, 0, 0, 0, 12, 229, 1, 0, 0, 0, 14, 231, 1, 0, 0, 0, 16, 244, 1, 0, 0, 0, 18, 250, 1, 0, 0, 0, 20, 271, 1, 0, 0, 0, 22, 281, 1, 0, 0, 0, 24, 296, 1, 0, 0, 0, 26, 298, 1, 0, 0, 0, 28, 300, 1, 0, 0, 0, 30, 303, 1, 0, 0, 0, 32, 314, 1, 0, 0, 0, 34, 318, 1, 0, 0, 0, 36, 333, 1, 0, 0, 0, 38, 337, 1, 0, 0, 0, 40, 339, 1, 0, 0, 0, 42, 341, 1, 0, 0, 0, 44, 350, 1, 0, 0, 0, 46, 366, 1, 0, 0, 0, 48, 369, 1, 0, 0, 0, 50, 377, 1, 0, 0, 0, 52, 385, 1, 0, 0, 0, 54, 390, 1, 0, 0, 0, 56, 398, 1, 0, 0, 0, 58, 406, 1, 0, 0, 0, 60, 414, 1, 0, 0, 0, 62, 419, 1, 0, 0, 0, 64, 463, 1, 0, 0, 0, 66, 467, 1, 0, 0, 0, 68, 472, 1, 0, 0, 0, 70, 474, 1, 0, 0, 0, 72, 477, 1, 0, 0, 0, 74, 486, 1, 0, 0, 0, 76, 494, 1, 0, 0, 0, 78, 497, 1, 0, 0, 0, 80, 500, 1, 0, 0, 0, 82, 509, 1, 0, 0, 0, 84, 513, 1, 0, 0, 0, 86, 519, 1, 0, 0, 0, 88, 523, 1, 0, 0, 0, 90, 526, 1, 0, 0, 0, 92, 534, 1, 0, 0, 0, 94, 538, 1, 0, 0, 0, 96, 542, 1, 0, 0, 0, 98, 545, 1, 0, 0, 0, 100, 550, 1, 0, 0, 0, 102, 554, 1, 0, 0, 0, 104, 556, 1, 0, 0, 0, 106, 558, 1, 0, 0, 0, 108, 561, 1, 0, 0, 0, 110, 565, 1, 0, 0, 0, 112, 568, 1, 0, 0, 0, 114, 588, 1, 0, 0, 0, 116, 592, 1, 0, 0, 0, 118, 597, 1, 0, 0, 0, 120, 604, 1, 0, 0, 0, 122, 610, 1, 0, 0, 0, 124, 615, 1, 0, 0, 0, 126, 624, 1, 0, 0, 0, 128, 129, 3, 2, 1, 0, 129, 130, 5, 0, 0, 1, 130, 1, 1, 0, 0, 0, 131, 132, 6, 1, -1, 0, 132, 133, 3, 4, 2, 0, 133, 139, 1, 0, 0, 0, 134, 135, 10, 1, 0, 0, 135, 136, 5, 29, 0, 0, 136, 138, 3, 6, 3, 0, 137, 134, 1, 0, 0, 0, 138, 141, 1, 0, 0, 0, 139, 137, 1, 0, 0, 0, 139, 140, 1, 0, 0, 0, 140, 3, 1, 0, 0, 0, 141, 139, 1, 0, 0, 0, 142, 149, 3, 106, 53, 0, 143, 149, 3, 34, 17, 0, 144, 149, 3, 28, 14, 0, 145, 149, 3, 110, 55, 0, 146, 147, 4, 2, 1, 0, 147, 149, 3, 44, 22, 0, 148, 142, 1, 0, 0, 0, 148, 143, 1, 0, 0, 0, 148, 144, 1, 0, 0, 0, 148, 145, 1, 0, 0, 0, 148, 146, 1, 0, 0, 0, 149, 5, 1, 0, 0, 0, 150, 169, 3, 46, 23, 0, 151, 169, 3, 8, 4, 0, 152, 169, 3, 76, 38, 0, 153, 169, 3, 70, 35, 0, 154, 169, 3, 48, 24, 0, 155, 169, 3, 72, 36, 0, 156, 169, 3, 78, 39, 0, 157, 169, 3, 80, 40, 0, 158, 169, 3, 84, 42, 0, 159, 169, 3, 86, 43, 0, 160, 169, 3, 112, 56, 0, 161, 169, 3, 88, 44, 0, 162, 163, 4, 3, 2, 0, 163, 169, 3, 118, 59, 0, 164, 165, 4, 3, 3, 0, 165, 169, 3, 116, 58, 0, 166, 167, 4, 3, 4, 0, 167, 169, 3, 120, 60, 0, 168, 150, 1, 0, 0, 0, 168, 151, 1, 0, 0, 0, 168, 152, 1, 0, 0, 0, 168, 153, 1, 0, 0, 0, 168, 154, 1, 0, 0, 0, 168, 155, 1, 0, 0, 0, 168, 156, 1, 0, 0, 0, 168, 157, 1, 0, 0, 0, 168, 158, 1, 0, 0, 0, 168, 159, 1, 0, 0, 0, 168, 160, 1, 0, 0, 0, 168, 161, 1, 0, 0, 0, 168, 162, 1, 0, 0, 0, 168, 164, 1, 0, 0, 0, 168, 166, 1, 0, 0, 0, 169, 7, 1, 0, 0, 0, 170, 171, 5, 16, 0, 0, 171, 172, 3, 10, 5, 0, 172, 9, 1, 0, 0, 0, 173, 174, 6, 5, -1, 0, 174, 175, 5, 49, 0, 0, 175, 203, 3, 10, 5, 8, 176, 203, 3, 16, 8, 0, 177, 203, 3, 12, 6, 0, 178, 180, 3, 16, 8, 0, 179, 181, 5, 49, 0, 0, 180, 179, 1, 0, 0, 0, 180, 181, 1, 0, 0, 0, 181, 182, 1, 0, 0, 0, 182, 183, 5, 44, 0, 0, 183, 184, 5, 48, 0, 0, 184, 189, 3, 16, 8, 0, 185, 186, 5, 39, 0, 0, 186, 188, 3, 16, 8, 0, 187, 185, 1, 0, 0, 0, 188, 191, 1, 0, 0, 0, 189, 187, 1, 0, 0, 0, 189, 190, 1, 0, 0, 0, 190, 192, 1, 0, 0, 0, 191, 189, 1, 0, 0, 0, 192, 193, 5, 55, 0, 0, 193, 203, 1, 0, 0, 0, 194, 195, 3, 16, 8, 0, 195, 197, 5, 45, 0, 0, 196, 198, 5, 49, 0, 0, 197, 196, 1, 0, 0, 0, 197, 198, 1, 0, 0, 0, 198, 199, 1, 0, 0, 0, 199, 200, 5, 50, 0, 0, 200, 203, 1, 0, 0, 0, 201, 203, 3, 14, 7, 0, 202, 173, 1, 0, 0, 0, 202, 176, 1, 0, 0, 0, 202, 177, 1, 0, 0, 0, 202, 178, 1, 0, 0, 0, 202, 194, 1, 0, 0, 0, 202, 201, 1, 0, 0, 0, 203, 212, 1, 0, 0, 0, 204, 205, 10, 5, 0, 0, 205, 206, 5, 34, 0, 0, 206, 211, 3, 10, 5, 6, 207, 208, 10, 4, 0, 0, 208, 209, 5, 52, 0, 0, 209, 211, 3, 10, 5, 5, 210, 204, 1, 0, 0, 0, 210, 207, 1, 0, 0, 0, 211, 214, 1, 0, 0, 0, 212, 210, 1, 0, 0, 0, 212, 213, 1, 0, 0, 0, 213, 11, 1, 0, 0, 0, 214, 212, 1, 0, 0, 0, 215, 217, 3, 16, 8, 0, 216, 218, 5, 49, 0, 0, 217, 216, 1, 0, 0, 0, 217, 218, 1, 0, 0, 0, 218, 219, 1, 0, 0, 0, 219, 220, 5, 47, 0, 0, 220, 221, 3, 102, 51, 0, 221, 230, 1, 0, 0, 0, 222, 224, 3, 16, 8, 0, 223, 225, 5, 49, 0, 0, 224, 223, 1, 0, 0, 0, 224, 225, 1, 0, 0, 0, 225, 226, 1, 0, 0, 0, 226, 227, 5, 54, 0, 0, 227, 228, 3, 102, 51, 0, 228, 230, 1, 0, 0, 0, 229, 215, 1, 0, 0, 0, 229, 222, 1, 0, 0, 0, 230, 13, 1, 0, 0, 0, 231, 234, 3, 54, 27, 0, 232, 233, 5, 37, 0, 0, 233, 235, 3, 26, 13, 0, 234, 232, 1, 0, 0, 0, 234, 235, 1, 0, 0, 0, 235, 236, 1, 0, 0, 0, 236, 237, 5, 38, 0, 0, 237, 238, 3, 64, 32, 0, 238, 15, 1, 0, 0, 0, 239, 245, 3, 18, 9, 0, 240, 241, 3, 18, 9, 0, 241, 242, 3, 104, 52, 0, 242, 243, 3, 18, 9, 0, 243, 245, 1, 0, 0, 0, 244, 239, 1, 0, 0, 0, 244, 240, 1, 0, 0, 0, 245, 17, 1, 0, 0, 0, 246, 247, 6, 9, -1, 0, 247, 251, 3, 20, 10, 0, 248, 249, 7, 0, 0, 0, 249, 251, 3, 18, 9, 3, 250, 246, 1, 0, 0, 0, 250, 248, 1, 0, 0, 0, 251, 260, 1, 0, 0, 0, 252, 253, 10, 2, 0, 0, 253, 254, 7, 1, 0, 0, 254, 259, 3, 18, 9, 3, 255, 256, 10, 1, 0, 0, 256, 257, 7, 0, 0, 0, 257, 259, 3, 18, 9, 2, 258, 252, 1, 0, 0, 0, 258, 255, 1, 0, 0, 0, 259, 262, 1, 0, 0, 0, 260, 258, 1, 0, 0, 0, 260, 261, 1, 0, 0, 0, 261, 19, 1, 0, 0, 0, 262, 260, 1, 0, 0, 0, 263, 264, 6, 10, -1, 0, 264, 272, 3, 64, 32, 0, 265, 272, 3, 54, 27, 0, 266, 272, 3, 22, 11, 0, 267, 268, 5, 48, 0, 0, 268, 269, 3, 10, 5, 0, 269, 270, 5, 55, 0, 0, 270, 272, 1, 0, 0, 0, 271, 263, 1, 0, 0, 0, 271, 265, 1, 0, 0, 0, 271, 266, 1, 0, 0, 0, 271, 267, 1, 0, 0, 0, 272, 278, 1, 0, 0, 0, 273, 274, 10, 1, 0, 0, 274, 275, 5, 37, 0, 0, 275, 277, 3, 26, 13, 0, 276, 273, 1, 0, 0, 0, 277, 280, 1, 0, 0, 0, 278, 276, 1, 0, 0, 0, 278, 279, 1, 0, 0, 0, 279, 21, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 281, 282, 3, 24, 12, 0, 282, 292, 5, 48, 0, 0, 283, 293, 5, 66, 0, 0, 284, 289, 3, 10, 5, 0, 285, 286, 5, 39, 0, 0, 286, 288, 3, 10, 5, 0, 287, 285, 1, 0, 0, 0, 288, 291, 1, 0, 0, 0, 289, 287, 1, 0, 0, 0, 289, 290, 1, 0, 0, 0, 290, 293, 1, 0, 0, 0, 291, 289, 1, 0, 0, 0, 292, 283, 1, 0, 0, 0, 292, 284, 1, 0, 0, 0, 292, 293, 1, 0, 0, 0, 293, 294, 1, 0, 0, 0, 294, 295, 5, 55, 0, 0, 295, 23, 1, 0, 0, 0, 296, 297, 3, 68, 34, 0, 297, 25, 1, 0, 0, 0, 298, 299, 3, 60, 30, 0, 299, 27, 1, 0, 0, 0, 300, 301, 5, 12, 0, 0, 301, 302, 3, 30, 15, 0, 302, 29, 1, 0, 0, 0, 303, 308, 3, 32, 16, 0, 304, 305, 5, 39, 0, 0, 305, 307, 3, 32, 16, 0, 306, 304, 1, 0, 0, 0, 307, 310, 1, 0, 0, 0, 308, 306, 1, 0, 0, 0, 308, 309, 1, 0, 0, 0, 309, 31, 1, 0, 0, 0, 310, 308, 1, 0, 0, 0, 311, 312, 3, 54, 27, 0, 312, 313, 5, 36, 0, 0, 313, 315, 1, 0, 0, 0, 314, 311, 1, 0, 0, 0, 314, 315, 1, 0, 0, 0, 315, 316, 1, 0, 0, 0, 316, 317, 3, 10, 5, 0, 317, 33, 1, 0, 0, 0, 318, 319, 5, 6, 0, 0, 319, 324, 3, 36, 18, 0, 320, 321, 5, 39, 0, 0, 321, 323, 3, 36, 18, 0, 322, 320, 1, 0, 0, 0, 323, 326, 1, 0, 0, 0, 324, 322, 1, 0, 0, 0, 324, 325, 1, 0, 0, 0, 325, 328, 1, 0, 0, 0, 326, 324, 1, 0, 0, 0, 327, 329, 3, 42, 21, 0, 328, 327, 1, 0, 0, 0, 328, 329, 1, 0, 0, 0, 329, 35, 1, 0, 0, 0, 330, 331, 3, 38, 19, 0, 331, 332, 5, 38, 0, 0, 332, 334, 1, 0, 0, 0, 333, 330, 1, 0, 0, 0, 333, 334, 1, 0, 0, 0, 334, 335, 1, 0, 0, 0, 335, 336, 3, 40, 20, 0, 336, 37, 1, 0, 0, 0, 337, 338, 5, 81, 0, 0, 338, 39, 1, 0, 0, 0, 339, 340, 7, 2, 0, 0, 340, 41, 1, 0, 0, 0, 341, 342, 5, 80, 0, 0, 342, 347, 5, 81, 0, 0, 343, 344, 5, 39, 0, 0, 344, 346, 5, 81, 0, 0, 345, 343, 1, 0, 0, 0, 346, 349, 1, 0, 0, 0, 347, 345, 1, 0, 0, 0, 347, 348, 1, 0, 0, 0, 348, 43, 1, 0, 0, 0, 349, 347, 1, 0, 0, 0, 350, 351, 5, 19, 0, 0, 351, 356, 3, 36, 18, 0, 352, 353, 5, 39, 0, 0, 353, 355, 3, 36, 18, 0, 354, 352, 1, 0, 0, 0, 355, 358, 1, 0, 0, 0, 356, 354, 1, 0, 0, 0, 356, 357, 1, 0, 0, 0, 357, 360, 1, 0, 0, 0, 358, 356, 1, 0, 0, 0, 359, 361, 3, 50, 25, 0, 360, 359, 1, 0, 0, 0, 360, 361, 1, 0, 0, 0, 361, 364, 1, 0, 0, 0, 362, 363, 5, 33, 0, 0, 363, 365, 3, 30, 15, 0, 364, 362, 1, 0, 0, 0, 364, 365, 1, 0, 0, 0, 365, 45, 1, 0, 0, 0, 366, 367, 5, 4, 0, 0, 367, 368, 3, 30, 15, 0, 368, 47, 1, 0, 0, 0, 369, 371, 5, 15, 0, 0, 370, 372, 3, 50, 25, 0, 371, 370, 1, 0, 0, 0, 371, 372, 1, 0, 0, 0, 372, 375, 1, 0, 0, 0, 373, 374, 5, 33, 0, 0, 374, 376, 3, 30, 15, 0, 375, 373, 1, 0, 0, 0, 375, 376, 1, 0, 0, 0, 376, 49, 1, 0, 0, 0, 377, 382, 3, 52, 26, 0, 378, 379, 5, 39, 0, 0, 379, 381, 3, 52, 26, 0, 380, 378, 1, 0, 0, 0, 381, 384, 1, 0, 0, 0, 382, 380, 1, 0, 0, 0, 382, 383, 1, 0, 0, 0, 383, 51, 1, 0, 0, 0, 384, 382, 1, 0, 0, 0, 385, 388, 3, 32, 16, 0, 386, 387, 5, 16, 0, 0, 387, 389, 3, 10, 5, 0, 388, 386, 1, 0, 0, 0, 388, 389, 1, 0, 0, 0, 389, 53, 1, 0, 0, 0, 390, 395, 3, 68, 34, 0, 391, 392, 5, 41, 0, 0, 392, 394, 3, 68, 34, 0, 393, 391, 1, 0, 0, 0, 394, 397, 1, 0, 0, 0, 395, 393, 1, 0, 0, 0, 395, 396, 1, 0, 0, 0, 396, 55, 1, 0, 0, 0, 397, 395, 1, 0, 0, 0, 398, 403, 3, 62, 31, 0, 399, 400, 5, 41, 0, 0, 400, 402, 3, 62, 31, 0, 401, 399, 1, 0, 0, 0, 402, 405, 1, 0, 0, 0, 403, 401, 1, 0, 0, 0, 403, 404, 1, 0, 0, 0, 404, 57, 1, 0, 0, 0, 405, 403, 1, 0, 0, 0, 406, 411, 3, 56, 28, 0, 407, 408, 5, 39, 0, 0, 408, 410, 3, 56, 28, 0, 409, 407, 1, 0, 0, 0, 410, 413, 1, 0, 0, 0, 411, 409, 1, 0, 0, 0, 411, 412, 1, 0, 0, 0, 412, 59, 1, 0, 0, 0, 413, 411, 1, 0, 0, 0, 414, 415, 7, 3, 0, 0, 415, 61, 1, 0, 0, 0, 416, 420, 5, 85, 0, 0, 417, 418, 4, 31, 10, 0, 418, 420, 3, 66, 33, 0, 419, 416, 1, 0, 0, 0, 419, 417, 1, 0, 0, 0, 420, 63, 1, 0, 0, 0, 421, 464, 5, 50, 0, 0, 422, 423, 3, 100, 50, 0, 423, 424, 5, 72, 0, 0, 424, 464, 1, 0, 0, 0, 425, 464, 3, 98, 49, 0, 426, 464, 3, 100, 50, 0, 427, 464, 3, 94, 47, 0, 428, 464, 3, 66, 33, 0, 429, 464, 3, 102, 51, 0, 430, 431, 5, 70, 0, 0, 431, 436, 3, 96, 48, 0, 432, 433, 5, 39, 0, 0, 433, 435, 3, 96, 48, 0, 434, 432, 1, 0, 0, 0, 435, 438, 1, 0, 0, 0, 436, 434, 1, 0, 0, 0, 436, 437, 1, 0, 0, 0, 437, 439, 1, 0, 0, 0, 438, 436, 1, 0, 0, 0, 439, 440, 5, 71, 0, 0, 440, 464, 1, 0, 0, 0, 441, 442, 5, 70, 0, 0, 442, 447, 3, 94, 47, 0, 443, 444, 5, 39, 0, 0, 444, 446, 3, 94, 47, 0, 445, 443, 1, 0, 0, 0, 446, 449, 1, 0, 0, 0, 447, 445, 1, 0, 0, 0, 447, 448, 1, 0, 0, 0, 448, 450, 1, 0, 0, 0, 449, 447, 1, 0, 0, 0, 450, 451, 5, 71, 0, 0, 451, 464, 1, 0, 0, 0, 452, 453, 5, 70, 0, 0, 453, 458, 3, 102, 51, 0, 454, 455, 5, 39, 0, 0, 455, 457, 3, 102, 51, 0, 456, 454, 1, 0, 0, 0, 457, 460, 1, 0, 0, 0, 458, 456, 1, 0, 0, 0, 458, 459, 1, 0, 0, 0, 459, 461, 1, 0, 0, 0, 460, 458, 1, 0, 0, 0, 461, 462, 5, 71, 0, 0, 462, 464, 1, 0, 0, 0, 463, 421, 1, 0, 0, 0, 463, 422, 1, 0, 0, 0, 463, 425, 1, 0, 0, 0, 463, 426, 1, 0, 0, 0, 463, 427, 1, 0, 0, 0, 463, 428, 1, 0, 0, 0, 463, 429, 1, 0, 0, 0, 463, 430, 1, 0, 0, 0, 463, 441, 1, 0, 0, 0, 463, 452, 1, 0, 0, 0, 464, 65, 1, 0, 0, 0, 465, 468, 5, 53, 0, 0, 466, 468, 5, 69, 0, 0, 467, 465, 1, 0, 0, 0, 467, 466, 1, 0, 0, 0, 468, 67, 1, 0, 0, 0, 469, 473, 3, 60, 30, 0, 470, 471, 4, 34, 11, 0, 471, 473, 3, 66, 33, 0, 472, 469, 1, 0, 0, 0, 472, 470, 1, 0, 0, 0, 473, 69, 1, 0, 0, 0, 474, 475, 5, 9, 0, 0, 475, 476, 5, 31, 0, 0, 476, 71, 1, 0, 0, 0, 477, 478, 5, 14, 0, 0, 478, 483, 3, 74, 37, 0, 479, 480, 5, 39, 0, 0, 480, 482, 3, 74, 37, 0, 481, 479, 1, 0, 0, 0, 482, 485, 1, 0, 0, 0, 483, 481, 1, 0, 0, 0, 483, 484, 1, 0, 0, 0, 484, 73, 1, 0, 0, 0, 485, 483, 1, 0, 0, 0, 486, 488, 3, 10, 5, 0, 487, 489, 7, 4, 0, 0, 488, 487, 1, 0, 0, 0, 488, 489, 1, 0, 0, 0, 489, 492, 1, 0, 0, 0, 490, 491, 5, 51, 0, 0, 491, 493, 7, 5, 0, 0, 492, 490, 1, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 75, 1, 0, 0, 0, 494, 495, 5, 8, 0, 0, 495, 496, 3, 58, 29, 0, 496, 77, 1, 0, 0, 0, 497, 498, 5, 2, 0, 0, 498, 499, 3, 58, 29, 0, 499, 79, 1, 0, 0, 0, 500, 501, 5, 11, 0, 0, 501, 506, 3, 82, 41, 0, 502, 503, 5, 39, 0, 0, 503, 505, 3, 82, 41, 0, 504, 502, 1, 0, 0, 0, 505, 508, 1, 0, 0, 0, 506, 504, 1, 0, 0, 0, 506, 507, 1, 0, 0, 0, 507, 81, 1, 0, 0, 0, 508, 506, 1, 0, 0, 0, 509, 510, 3, 56, 28, 0, 510, 511, 5, 89, 0, 0, 511, 512, 3, 56, 28, 0, 512, 83, 1, 0, 0, 0, 513, 514, 5, 1, 0, 0, 514, 515, 3, 20, 10, 0, 515, 517, 3, 102, 51, 0, 516, 518, 3, 90, 45, 0, 517, 516, 1, 0, 0, 0, 517, 518, 1, 0, 0, 0, 518, 85, 1, 0, 0, 0, 519, 520, 5, 7, 0, 0, 520, 521, 3, 20, 10, 0, 521, 522, 3, 102, 51, 0, 522, 87, 1, 0, 0, 0, 523, 524, 5, 10, 0, 0, 524, 525, 3, 54, 27, 0, 525, 89, 1, 0, 0, 0, 526, 531, 3, 92, 46, 0, 527, 528, 5, 39, 0, 0, 528, 530, 3, 92, 46, 0, 529, 527, 1, 0, 0, 0, 530, 533, 1, 0, 0, 0, 531, 529, 1, 0, 0, 0, 531, 532, 1, 0, 0, 0, 532, 91, 1, 0, 0, 0, 533, 531, 1, 0, 0, 0, 534, 535, 3, 60, 30, 0, 535, 536, 5, 36, 0, 0, 536, 537, 3, 64, 32, 0, 537, 93, 1, 0, 0, 0, 538, 539, 7, 6, 0, 0, 539, 95, 1, 0, 0, 0, 540, 543, 3, 98, 49, 0, 541, 543, 3, 100, 50, 0, 542, 540, 1, 0, 0, 0, 542, 541, 1, 0, 0, 0, 543, 97, 1, 0, 0, 0, 544, 546, 7, 0, 0, 0, 545, 544, 1, 0, 0, 0, 545, 546, 1, 0, 0, 0, 546, 547, 1, 0, 0, 0, 547, 548, 5, 32, 0, 0, 548, 99, 1, 0, 0, 0, 549, 551, 7, 0, 0, 0, 550, 549, 1, 0, 0, 0, 550, 551, 1, 0, 0, 0, 551, 552, 1, 0, 0, 0, 552, 553, 5, 31, 0, 0, 553, 101, 1, 0, 0, 0, 554, 555, 5, 30, 0, 0, 555, 103, 1, 0, 0, 0, 556, 557, 7, 7, 0, 0, 557, 105, 1, 0, 0, 0, 558, 559, 5, 5, 0, 0, 559, 560, 3, 108, 54, 0, 560, 107, 1, 0, 0, 0, 561, 562, 5, 70, 0, 0, 562, 563, 3, 2, 1, 0, 563, 564, 5, 71, 0, 0, 564, 109, 1, 0, 0, 0, 565, 566, 5, 13, 0, 0, 566, 567, 5, 105, 0, 0, 567, 111, 1, 0, 0, 0, 568, 569, 5, 3, 0, 0, 569, 572, 5, 95, 0, 0, 570, 571, 5, 93, 0, 0, 571, 573, 3, 56, 28, 0, 572, 570, 1, 0, 0, 0, 572, 573, 1, 0, 0, 0, 573, 583, 1, 0, 0, 0, 574, 575, 5, 94, 0, 0, 575, 580, 3, 114, 57, 0, 576, 577, 5, 39, 0, 0, 577, 579, 3, 114, 57, 0, 578, 576, 1, 0, 0, 0, 579, 582, 1, 0, 0, 0, 580, 578, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 584, 1, 0, 0, 0, 582, 580, 1, 0, 0, 0, 583, 574, 1, 0, 0, 0, 583, 584, 1, 0, 0, 0, 584, 113, 1, 0, 0, 0, 585, 586, 3, 56, 28, 0, 586, 587, 5, 36, 0, 0, 587, 589, 1, 0, 0, 0, 588, 585, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 3, 56, 28, 0, 591, 115, 1, 0, 0, 0, 592, 593, 5, 18, 0, 0, 593, 594, 3, 36, 18, 0, 594, 595, 5, 93, 0, 0, 595, 596, 3, 58, 29, 0, 596, 117, 1, 0, 0, 0, 597, 598, 5, 17, 0, 0, 598, 601, 3, 50, 25, 0, 599, 600, 5, 33, 0, 0, 600, 602, 3, 30, 15, 0, 601, 599, 1, 0, 0, 0, 601, 602, 1, 0, 0, 0, 602, 119, 1, 0, 0, 0, 603, 605, 7, 8, 0, 0, 604, 603, 1, 0, 0, 0, 604, 605, 1, 0, 0, 0, 605, 606, 1, 0, 0, 0, 606, 607, 5, 20, 0, 0, 607, 608, 3, 122, 61, 0, 608, 609, 3, 124, 62, 0, 609, 121, 1, 0, 0, 0, 610, 613, 3, 60, 30, 0, 611, 612, 5, 89, 0, 0, 612, 614, 3, 60, 30, 0, 613, 611, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 123, 1, 0, 0, 0, 615, 616, 5, 93, 0, 0, 616, 621, 3, 126, 63, 0, 617, 618, 5, 39, 0, 0, 618, 620, 3, 126, 63, 0, 619, 617, 1, 0, 0, 0, 620, 623, 1, 0, 0, 0, 621, 619, 1, 0, 0, 0, 621, 622, 1, 0, 0, 0, 622, 125, 1, 0, 0, 0, 623, 621, 1, 0, 0, 0, 624, 625, 3, 16, 8, 0, 625, 127, 1, 0, 0, 0, 61, 139, 148, 168, 180, 189, 197, 202, 210, 212, 217, 224, 229, 234, 244, 250, 258, 260, 271, 278, 289, 292, 308, 314, 324, 328, 333, 347, 356, 360, 364, 371, 375, 382, 388, 395, 403, 411, 419, 436, 447, 458, 463, 467, 472, 483, 488, 492, 506, 517, 531, 542, 545, 550, 572, 580, 583, 588, 601, 604, 613, 621] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index a56035364641d..e272dc7f477a4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -59,35 +59,34 @@ public class EsqlBaseParser extends ParserConfig { RULE_primaryExpression = 10, RULE_functionExpression = 11, RULE_functionName = 12, RULE_dataType = 13, RULE_rowCommand = 14, RULE_fields = 15, RULE_field = 16, RULE_fromCommand = 17, RULE_indexPattern = 18, RULE_clusterString = 19, - RULE_indexString = 20, RULE_metadata = 21, RULE_metadataOption = 22, RULE_deprecated_metadata = 23, - RULE_metricsCommand = 24, RULE_evalCommand = 25, RULE_statsCommand = 26, - RULE_aggFields = 27, RULE_aggField = 28, RULE_qualifiedName = 29, RULE_qualifiedNamePattern = 30, - RULE_qualifiedNamePatterns = 31, RULE_identifier = 32, RULE_identifierPattern = 33, - RULE_constant = 34, RULE_parameter = 35, RULE_identifierOrParameter = 36, - RULE_limitCommand = 37, RULE_sortCommand = 38, RULE_orderExpression = 39, - RULE_keepCommand = 40, RULE_dropCommand = 41, RULE_renameCommand = 42, - RULE_renameClause = 43, RULE_dissectCommand = 44, RULE_grokCommand = 45, - RULE_mvExpandCommand = 46, RULE_commandOptions = 47, RULE_commandOption = 48, - RULE_booleanValue = 49, RULE_numericValue = 50, RULE_decimalValue = 51, - RULE_integerValue = 52, RULE_string = 53, RULE_comparisonOperator = 54, - RULE_explainCommand = 55, RULE_subqueryExpression = 56, RULE_showCommand = 57, - RULE_enrichCommand = 58, RULE_enrichWithClause = 59, RULE_lookupCommand = 60, - RULE_inlinestatsCommand = 61, RULE_joinCommand = 62, RULE_joinTarget = 63, - RULE_joinCondition = 64, RULE_joinPredicate = 65; + RULE_indexString = 20, RULE_metadata = 21, RULE_metricsCommand = 22, RULE_evalCommand = 23, + RULE_statsCommand = 24, RULE_aggFields = 25, RULE_aggField = 26, RULE_qualifiedName = 27, + RULE_qualifiedNamePattern = 28, RULE_qualifiedNamePatterns = 29, RULE_identifier = 30, + RULE_identifierPattern = 31, RULE_constant = 32, RULE_parameter = 33, + RULE_identifierOrParameter = 34, RULE_limitCommand = 35, RULE_sortCommand = 36, + RULE_orderExpression = 37, RULE_keepCommand = 38, RULE_dropCommand = 39, + RULE_renameCommand = 40, RULE_renameClause = 41, RULE_dissectCommand = 42, + RULE_grokCommand = 43, RULE_mvExpandCommand = 44, RULE_commandOptions = 45, + RULE_commandOption = 46, RULE_booleanValue = 47, RULE_numericValue = 48, + RULE_decimalValue = 49, RULE_integerValue = 50, RULE_string = 51, RULE_comparisonOperator = 52, + RULE_explainCommand = 53, RULE_subqueryExpression = 54, RULE_showCommand = 55, + RULE_enrichCommand = 56, RULE_enrichWithClause = 57, RULE_lookupCommand = 58, + RULE_inlinestatsCommand = 59, RULE_joinCommand = 60, RULE_joinTarget = 61, + RULE_joinCondition = 62, RULE_joinPredicate = 63; private static String[] makeRuleNames() { return new String[] { "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", "booleanExpression", "regexBooleanExpression", "matchBooleanExpression", "valueExpression", "operatorExpression", "primaryExpression", "functionExpression", "functionName", "dataType", "rowCommand", "fields", "field", "fromCommand", - "indexPattern", "clusterString", "indexString", "metadata", "metadataOption", - "deprecated_metadata", "metricsCommand", "evalCommand", "statsCommand", - "aggFields", "aggField", "qualifiedName", "qualifiedNamePattern", "qualifiedNamePatterns", - "identifier", "identifierPattern", "constant", "parameter", "identifierOrParameter", - "limitCommand", "sortCommand", "orderExpression", "keepCommand", "dropCommand", - "renameCommand", "renameClause", "dissectCommand", "grokCommand", "mvExpandCommand", - "commandOptions", "commandOption", "booleanValue", "numericValue", "decimalValue", - "integerValue", "string", "comparisonOperator", "explainCommand", "subqueryExpression", + "indexPattern", "clusterString", "indexString", "metadata", "metricsCommand", + "evalCommand", "statsCommand", "aggFields", "aggField", "qualifiedName", + "qualifiedNamePattern", "qualifiedNamePatterns", "identifier", "identifierPattern", + "constant", "parameter", "identifierOrParameter", "limitCommand", "sortCommand", + "orderExpression", "keepCommand", "dropCommand", "renameCommand", "renameClause", + "dissectCommand", "grokCommand", "mvExpandCommand", "commandOptions", + "commandOption", "booleanValue", "numericValue", "decimalValue", "integerValue", + "string", "comparisonOperator", "explainCommand", "subqueryExpression", "showCommand", "enrichCommand", "enrichWithClause", "lookupCommand", "inlinestatsCommand", "joinCommand", "joinTarget", "joinCondition", "joinPredicate" }; @@ -226,9 +225,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(132); + setState(128); query(0); - setState(133); + setState(129); match(EOF); } } @@ -324,11 +323,11 @@ private QueryContext query(int _p) throws RecognitionException { _ctx = _localctx; _prevctx = _localctx; - setState(136); + setState(132); sourceCommand(); } _ctx.stop = _input.LT(-1); - setState(143); + setState(139); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -339,16 +338,16 @@ private QueryContext query(int _p) throws RecognitionException { { _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_query); - setState(138); + setState(134); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(139); + setState(135); match(PIPE); - setState(140); + setState(136); processingCommand(); } } } - setState(145); + setState(141); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); } @@ -406,43 +405,43 @@ public final SourceCommandContext sourceCommand() throws RecognitionException { SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState()); enterRule(_localctx, 4, RULE_sourceCommand); try { - setState(152); + setState(148); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(146); + setState(142); explainCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(147); + setState(143); fromCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(148); + setState(144); rowCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(149); + setState(145); showCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(150); + setState(146); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(151); + setState(147); metricsCommand(); } break; @@ -530,117 +529,117 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState()); enterRule(_localctx, 6, RULE_processingCommand); try { - setState(172); + setState(168); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(154); + setState(150); evalCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(155); + setState(151); whereCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(156); + setState(152); keepCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(157); + setState(153); limitCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(158); + setState(154); statsCommand(); } break; case 6: enterOuterAlt(_localctx, 6); { - setState(159); + setState(155); sortCommand(); } break; case 7: enterOuterAlt(_localctx, 7); { - setState(160); + setState(156); dropCommand(); } break; case 8: enterOuterAlt(_localctx, 8); { - setState(161); + setState(157); renameCommand(); } break; case 9: enterOuterAlt(_localctx, 9); { - setState(162); + setState(158); dissectCommand(); } break; case 10: enterOuterAlt(_localctx, 10); { - setState(163); + setState(159); grokCommand(); } break; case 11: enterOuterAlt(_localctx, 11); { - setState(164); + setState(160); enrichCommand(); } break; case 12: enterOuterAlt(_localctx, 12); { - setState(165); + setState(161); mvExpandCommand(); } break; case 13: enterOuterAlt(_localctx, 13); { - setState(166); + setState(162); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(167); + setState(163); inlinestatsCommand(); } break; case 14: enterOuterAlt(_localctx, 14); { - setState(168); + setState(164); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(169); + setState(165); lookupCommand(); } break; case 15: enterOuterAlt(_localctx, 15); { - setState(170); + setState(166); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(171); + setState(167); joinCommand(); } break; @@ -689,9 +688,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(174); + setState(170); match(WHERE); - setState(175); + setState(171); booleanExpression(0); } } @@ -907,7 +906,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(206); + setState(202); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { case 1: @@ -916,9 +915,9 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(178); + setState(174); match(NOT); - setState(179); + setState(175); booleanExpression(8); } break; @@ -927,7 +926,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new BooleanDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(180); + setState(176); valueExpression(); } break; @@ -936,7 +935,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new RegexExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(181); + setState(177); regexBooleanExpression(); } break; @@ -945,41 +944,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalInContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(182); + setState(178); valueExpression(); - setState(184); + setState(180); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(183); + setState(179); match(NOT); } } - setState(186); + setState(182); match(IN); - setState(187); + setState(183); match(LP); - setState(188); + setState(184); valueExpression(); - setState(193); + setState(189); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(189); + setState(185); match(COMMA); - setState(190); + setState(186); valueExpression(); } } - setState(195); + setState(191); _errHandler.sync(this); _la = _input.LA(1); } - setState(196); + setState(192); match(RP); } break; @@ -988,21 +987,21 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new IsNullContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(198); + setState(194); valueExpression(); - setState(199); + setState(195); match(IS); - setState(201); + setState(197); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(200); + setState(196); match(NOT); } } - setState(203); + setState(199); match(NULL); } break; @@ -1011,13 +1010,13 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new MatchExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(205); + setState(201); matchBooleanExpression(); } break; } _ctx.stop = _input.LT(-1); - setState(216); + setState(212); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1025,7 +1024,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(214); + setState(210); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) { case 1: @@ -1033,11 +1032,11 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(208); + setState(204); if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)"); - setState(209); + setState(205); ((LogicalBinaryContext)_localctx).operator = match(AND); - setState(210); + setState(206); ((LogicalBinaryContext)_localctx).right = booleanExpression(6); } break; @@ -1046,18 +1045,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(211); + setState(207); if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(212); + setState(208); ((LogicalBinaryContext)_localctx).operator = match(OR); - setState(213); + setState(209); ((LogicalBinaryContext)_localctx).right = booleanExpression(5); } break; } } } - setState(218); + setState(214); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); } @@ -1112,48 +1111,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog enterRule(_localctx, 12, RULE_regexBooleanExpression); int _la; try { - setState(233); + setState(229); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(219); + setState(215); valueExpression(); - setState(221); + setState(217); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(220); + setState(216); match(NOT); } } - setState(223); + setState(219); ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE); - setState(224); + setState(220); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(226); + setState(222); valueExpression(); - setState(228); + setState(224); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(227); + setState(223); match(NOT); } } - setState(230); + setState(226); ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE); - setState(231); + setState(227); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; @@ -1213,23 +1212,23 @@ public final MatchBooleanExpressionContext matchBooleanExpression() throws Recog try { enterOuterAlt(_localctx, 1); { - setState(235); + setState(231); ((MatchBooleanExpressionContext)_localctx).fieldExp = qualifiedName(); - setState(238); + setState(234); _errHandler.sync(this); _la = _input.LA(1); if (_la==CAST_OP) { { - setState(236); + setState(232); match(CAST_OP); - setState(237); + setState(233); ((MatchBooleanExpressionContext)_localctx).fieldType = dataType(); } } - setState(240); + setState(236); match(COLON); - setState(241); + setState(237); ((MatchBooleanExpressionContext)_localctx).matchQuery = constant(); } } @@ -1313,14 +1312,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState()); enterRule(_localctx, 16, RULE_valueExpression); try { - setState(248); + setState(244); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { case 1: _localctx = new ValueExpressionDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(243); + setState(239); operatorExpression(0); } break; @@ -1328,11 +1327,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio _localctx = new ComparisonContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(244); + setState(240); ((ComparisonContext)_localctx).left = operatorExpression(0); - setState(245); + setState(241); comparisonOperator(); - setState(246); + setState(242); ((ComparisonContext)_localctx).right = operatorExpression(0); } break; @@ -1457,7 +1456,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE int _alt; enterOuterAlt(_localctx, 1); { - setState(254); + setState(250); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: @@ -1466,7 +1465,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _ctx = _localctx; _prevctx = _localctx; - setState(251); + setState(247); primaryExpression(0); } break; @@ -1475,7 +1474,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticUnaryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(252); + setState(248); ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1486,13 +1485,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(253); + setState(249); operatorExpression(3); } break; } _ctx.stop = _input.LT(-1); - setState(264); + setState(260); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,16,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1500,7 +1499,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(262); + setState(258); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) { case 1: @@ -1508,9 +1507,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(256); + setState(252); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(257); + setState(253); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(((((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & 7L) != 0)) ) { @@ -1521,7 +1520,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(258); + setState(254); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3); } break; @@ -1530,9 +1529,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(259); + setState(255); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(260); + setState(256); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1543,14 +1542,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(261); + setState(257); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2); } break; } } } - setState(266); + setState(262); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,16,_ctx); } @@ -1708,7 +1707,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(275); + setState(271); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,17,_ctx) ) { case 1: @@ -1717,7 +1716,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(268); + setState(264); constant(); } break; @@ -1726,7 +1725,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new DereferenceContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(269); + setState(265); qualifiedName(); } break; @@ -1735,7 +1734,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new FunctionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(270); + setState(266); functionExpression(); } break; @@ -1744,17 +1743,17 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new ParenthesizedExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(271); + setState(267); match(LP); - setState(272); + setState(268); booleanExpression(0); - setState(273); + setState(269); match(RP); } break; } _ctx.stop = _input.LT(-1); - setState(282); + setState(278); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,18,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1765,16 +1764,16 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc { _localctx = new InlineCastContext(new PrimaryExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_primaryExpression); - setState(277); + setState(273); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(278); + setState(274); match(CAST_OP); - setState(279); + setState(275); dataType(); } } } - setState(284); + setState(280); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,18,_ctx); } @@ -1836,37 +1835,37 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(285); + setState(281); functionName(); - setState(286); + setState(282); match(LP); - setState(296); + setState(292); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) { case 1: { - setState(287); + setState(283); match(ASTERISK); } break; case 2: { { - setState(288); + setState(284); booleanExpression(0); - setState(293); + setState(289); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(289); + setState(285); match(COMMA); - setState(290); + setState(286); booleanExpression(0); } } - setState(295); + setState(291); _errHandler.sync(this); _la = _input.LA(1); } @@ -1874,7 +1873,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx } break; } - setState(298); + setState(294); match(RP); } } @@ -1920,7 +1919,7 @@ public final FunctionNameContext functionName() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(300); + setState(296); identifierOrParameter(); } } @@ -1978,7 +1977,7 @@ public final DataTypeContext dataType() throws RecognitionException { _localctx = new ToDataTypeContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(302); + setState(298); identifier(); } } @@ -2025,9 +2024,9 @@ public final RowCommandContext rowCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(304); + setState(300); match(ROW); - setState(305); + setState(301); fields(); } } @@ -2081,23 +2080,23 @@ public final FieldsContext fields() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(307); + setState(303); field(); - setState(312); + setState(308); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,21,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(308); + setState(304); match(COMMA); - setState(309); + setState(305); field(); } } } - setState(314); + setState(310); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,21,_ctx); } @@ -2149,19 +2148,19 @@ public final FieldContext field() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(318); + setState(314); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) { case 1: { - setState(315); + setState(311); qualifiedName(); - setState(316); + setState(312); match(ASSIGN); } break; } - setState(320); + setState(316); booleanExpression(0); } } @@ -2219,34 +2218,34 @@ public final FromCommandContext fromCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(322); + setState(318); match(FROM); - setState(323); + setState(319); indexPattern(); - setState(328); + setState(324); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,23,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(324); + setState(320); match(COMMA); - setState(325); + setState(321); indexPattern(); } } } - setState(330); + setState(326); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,23,_ctx); } - setState(332); + setState(328); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { case 1: { - setState(331); + setState(327); metadata(); } break; @@ -2299,19 +2298,19 @@ public final IndexPatternContext indexPattern() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(337); + setState(333); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) { case 1: { - setState(334); + setState(330); clusterString(); - setState(335); + setState(331); match(COLON); } break; } - setState(339); + setState(335); indexString(); } } @@ -2355,7 +2354,7 @@ public final ClusterStringContext clusterString() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(341); + setState(337); match(UNQUOTED_SOURCE); } } @@ -2401,7 +2400,7 @@ public final IndexStringContext indexString() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(343); + setState(339); _la = _input.LA(1); if ( !(_la==QUOTED_STRING || _la==UNQUOTED_SOURCE) ) { _errHandler.recoverInline(this); @@ -2426,11 +2425,14 @@ public final IndexStringContext indexString() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class MetadataContext extends ParserRuleContext { - public MetadataOptionContext metadataOption() { - return getRuleContext(MetadataOptionContext.class,0); + public TerminalNode METADATA() { return getToken(EsqlBaseParser.METADATA, 0); } + public List UNQUOTED_SOURCE() { return getTokens(EsqlBaseParser.UNQUOTED_SOURCE); } + public TerminalNode UNQUOTED_SOURCE(int i) { + return getToken(EsqlBaseParser.UNQUOTED_SOURCE, i); } - public Deprecated_metadataContext deprecated_metadata() { - return getRuleContext(Deprecated_metadataContext.class,0); + public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } + public TerminalNode COMMA(int i) { + return getToken(EsqlBaseParser.COMMA, i); } @SuppressWarnings("this-escape") public MetadataContext(ParserRuleContext parent, int invokingState) { @@ -2455,98 +2457,31 @@ public T accept(ParseTreeVisitor visitor) { public final MetadataContext metadata() throws RecognitionException { MetadataContext _localctx = new MetadataContext(_ctx, getState()); enterRule(_localctx, 42, RULE_metadata); - try { - setState(347); - _errHandler.sync(this); - switch (_input.LA(1)) { - case METADATA: - enterOuterAlt(_localctx, 1); - { - setState(345); - metadataOption(); - } - break; - case OPENING_BRACKET: - enterOuterAlt(_localctx, 2); - { - setState(346); - deprecated_metadata(); - } - break; - default: - throw new NoViableAltException(this); - } - } - catch (RecognitionException re) { - _localctx.exception = re; - _errHandler.reportError(this, re); - _errHandler.recover(this, re); - } - finally { - exitRule(); - } - return _localctx; - } - - @SuppressWarnings("CheckReturnValue") - public static class MetadataOptionContext extends ParserRuleContext { - public TerminalNode METADATA() { return getToken(EsqlBaseParser.METADATA, 0); } - public List UNQUOTED_SOURCE() { return getTokens(EsqlBaseParser.UNQUOTED_SOURCE); } - public TerminalNode UNQUOTED_SOURCE(int i) { - return getToken(EsqlBaseParser.UNQUOTED_SOURCE, i); - } - public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } - public TerminalNode COMMA(int i) { - return getToken(EsqlBaseParser.COMMA, i); - } - @SuppressWarnings("this-escape") - public MetadataOptionContext(ParserRuleContext parent, int invokingState) { - super(parent, invokingState); - } - @Override public int getRuleIndex() { return RULE_metadataOption; } - @Override - public void enterRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterMetadataOption(this); - } - @Override - public void exitRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitMetadataOption(this); - } - @Override - public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitMetadataOption(this); - else return visitor.visitChildren(this); - } - } - - public final MetadataOptionContext metadataOption() throws RecognitionException { - MetadataOptionContext _localctx = new MetadataOptionContext(_ctx, getState()); - enterRule(_localctx, 44, RULE_metadataOption); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(349); + setState(341); match(METADATA); - setState(350); + setState(342); match(UNQUOTED_SOURCE); - setState(355); + setState(347); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,27,_ctx); + _alt = getInterpreter().adaptivePredict(_input,26,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(351); + setState(343); match(COMMA); - setState(352); + setState(344); match(UNQUOTED_SOURCE); } } } - setState(357); + setState(349); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,27,_ctx); + _alt = getInterpreter().adaptivePredict(_input,26,_ctx); } } } @@ -2561,57 +2496,6 @@ public final MetadataOptionContext metadataOption() throws RecognitionException return _localctx; } - @SuppressWarnings("CheckReturnValue") - public static class Deprecated_metadataContext extends ParserRuleContext { - public TerminalNode OPENING_BRACKET() { return getToken(EsqlBaseParser.OPENING_BRACKET, 0); } - public MetadataOptionContext metadataOption() { - return getRuleContext(MetadataOptionContext.class,0); - } - public TerminalNode CLOSING_BRACKET() { return getToken(EsqlBaseParser.CLOSING_BRACKET, 0); } - public Deprecated_metadataContext(ParserRuleContext parent, int invokingState) { - super(parent, invokingState); - } - @Override public int getRuleIndex() { return RULE_deprecated_metadata; } - @Override - public void enterRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterDeprecated_metadata(this); - } - @Override - public void exitRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitDeprecated_metadata(this); - } - @Override - public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitDeprecated_metadata(this); - else return visitor.visitChildren(this); - } - } - - public final Deprecated_metadataContext deprecated_metadata() throws RecognitionException { - Deprecated_metadataContext _localctx = new Deprecated_metadataContext(_ctx, getState()); - enterRule(_localctx, 46, RULE_deprecated_metadata); - try { - enterOuterAlt(_localctx, 1); - { - setState(358); - match(OPENING_BRACKET); - setState(359); - metadataOption(); - setState(360); - match(CLOSING_BRACKET); - } - } - catch (RecognitionException re) { - _localctx.exception = re; - _errHandler.reportError(this, re); - _errHandler.recover(this, re); - } - finally { - exitRule(); - } - return _localctx; - } - @SuppressWarnings("CheckReturnValue") public static class MetricsCommandContext extends ParserRuleContext { public AggFieldsContext aggregates; @@ -2656,51 +2540,51 @@ public T accept(ParseTreeVisitor visitor) { public final MetricsCommandContext metricsCommand() throws RecognitionException { MetricsCommandContext _localctx = new MetricsCommandContext(_ctx, getState()); - enterRule(_localctx, 48, RULE_metricsCommand); + enterRule(_localctx, 44, RULE_metricsCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(362); + setState(350); match(DEV_METRICS); - setState(363); + setState(351); indexPattern(); - setState(368); + setState(356); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,28,_ctx); + _alt = getInterpreter().adaptivePredict(_input,27,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(364); + setState(352); match(COMMA); - setState(365); + setState(353); indexPattern(); } } } - setState(370); + setState(358); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,28,_ctx); + _alt = getInterpreter().adaptivePredict(_input,27,_ctx); } - setState(372); + setState(360); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) { case 1: { - setState(371); + setState(359); ((MetricsCommandContext)_localctx).aggregates = aggFields(); } break; } - setState(376); + setState(364); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { case 1: { - setState(374); + setState(362); match(BY); - setState(375); + setState(363); ((MetricsCommandContext)_localctx).grouping = fields(); } break; @@ -2746,13 +2630,13 @@ public T accept(ParseTreeVisitor visitor) { public final EvalCommandContext evalCommand() throws RecognitionException { EvalCommandContext _localctx = new EvalCommandContext(_ctx, getState()); - enterRule(_localctx, 50, RULE_evalCommand); + enterRule(_localctx, 46, RULE_evalCommand); try { enterOuterAlt(_localctx, 1); { - setState(378); + setState(366); match(EVAL); - setState(379); + setState(367); fields(); } } @@ -2801,30 +2685,30 @@ public T accept(ParseTreeVisitor visitor) { public final StatsCommandContext statsCommand() throws RecognitionException { StatsCommandContext _localctx = new StatsCommandContext(_ctx, getState()); - enterRule(_localctx, 52, RULE_statsCommand); + enterRule(_localctx, 48, RULE_statsCommand); try { enterOuterAlt(_localctx, 1); { - setState(381); + setState(369); match(STATS); - setState(383); + setState(371); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { case 1: { - setState(382); + setState(370); ((StatsCommandContext)_localctx).stats = aggFields(); } break; } - setState(387); + setState(375); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { case 1: { - setState(385); + setState(373); match(BY); - setState(386); + setState(374); ((StatsCommandContext)_localctx).grouping = fields(); } break; @@ -2876,30 +2760,30 @@ public T accept(ParseTreeVisitor visitor) { public final AggFieldsContext aggFields() throws RecognitionException { AggFieldsContext _localctx = new AggFieldsContext(_ctx, getState()); - enterRule(_localctx, 54, RULE_aggFields); + enterRule(_localctx, 50, RULE_aggFields); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(389); + setState(377); aggField(); - setState(394); + setState(382); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,32,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(390); + setState(378); match(COMMA); - setState(391); + setState(379); aggField(); } } } - setState(396); + setState(384); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,32,_ctx); } } } @@ -2945,20 +2829,20 @@ public T accept(ParseTreeVisitor visitor) { public final AggFieldContext aggField() throws RecognitionException { AggFieldContext _localctx = new AggFieldContext(_ctx, getState()); - enterRule(_localctx, 56, RULE_aggField); + enterRule(_localctx, 52, RULE_aggField); try { enterOuterAlt(_localctx, 1); { - setState(397); + setState(385); field(); - setState(400); + setState(388); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) { case 1: { - setState(398); + setState(386); match(WHERE); - setState(399); + setState(387); booleanExpression(0); } break; @@ -3010,30 +2894,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNameContext qualifiedName() throws RecognitionException { QualifiedNameContext _localctx = new QualifiedNameContext(_ctx, getState()); - enterRule(_localctx, 58, RULE_qualifiedName); + enterRule(_localctx, 54, RULE_qualifiedName); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(402); + setState(390); identifierOrParameter(); - setState(407); + setState(395); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,35,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(403); + setState(391); match(DOT); - setState(404); + setState(392); identifierOrParameter(); } } } - setState(409); + setState(397); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,35,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); } } } @@ -3082,30 +2966,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNamePatternContext qualifiedNamePattern() throws RecognitionException { QualifiedNamePatternContext _localctx = new QualifiedNamePatternContext(_ctx, getState()); - enterRule(_localctx, 60, RULE_qualifiedNamePattern); + enterRule(_localctx, 56, RULE_qualifiedNamePattern); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(410); + setState(398); identifierPattern(); - setState(415); + setState(403); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,36,_ctx); + _alt = getInterpreter().adaptivePredict(_input,35,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(411); + setState(399); match(DOT); - setState(412); + setState(400); identifierPattern(); } } } - setState(417); + setState(405); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,36,_ctx); + _alt = getInterpreter().adaptivePredict(_input,35,_ctx); } } } @@ -3154,30 +3038,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNamePatternsContext qualifiedNamePatterns() throws RecognitionException { QualifiedNamePatternsContext _localctx = new QualifiedNamePatternsContext(_ctx, getState()); - enterRule(_localctx, 62, RULE_qualifiedNamePatterns); + enterRule(_localctx, 58, RULE_qualifiedNamePatterns); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(418); + setState(406); qualifiedNamePattern(); - setState(423); + setState(411); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,37,_ctx); + _alt = getInterpreter().adaptivePredict(_input,36,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(419); + setState(407); match(COMMA); - setState(420); + setState(408); qualifiedNamePattern(); } } } - setState(425); + setState(413); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,37,_ctx); + _alt = getInterpreter().adaptivePredict(_input,36,_ctx); } } } @@ -3218,12 +3102,12 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierContext identifier() throws RecognitionException { IdentifierContext _localctx = new IdentifierContext(_ctx, getState()); - enterRule(_localctx, 64, RULE_identifier); + enterRule(_localctx, 60, RULE_identifier); int _la; try { enterOuterAlt(_localctx, 1); { - setState(426); + setState(414); _la = _input.LA(1); if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); @@ -3274,24 +3158,24 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierPatternContext identifierPattern() throws RecognitionException { IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState()); - enterRule(_localctx, 66, RULE_identifierPattern); + enterRule(_localctx, 62, RULE_identifierPattern); try { - setState(431); + setState(419); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,37,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(428); + setState(416); match(ID_PATTERN); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(429); + setState(417); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(430); + setState(418); parameter(); } break; @@ -3561,17 +3445,17 @@ public T accept(ParseTreeVisitor visitor) { public final ConstantContext constant() throws RecognitionException { ConstantContext _localctx = new ConstantContext(_ctx, getState()); - enterRule(_localctx, 68, RULE_constant); + enterRule(_localctx, 64, RULE_constant); int _la; try { - setState(475); + setState(463); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) { case 1: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(433); + setState(421); match(NULL); } break; @@ -3579,9 +3463,9 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new QualifiedIntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(434); + setState(422); integerValue(); - setState(435); + setState(423); match(UNQUOTED_IDENTIFIER); } break; @@ -3589,7 +3473,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(437); + setState(425); decimalValue(); } break; @@ -3597,7 +3481,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(438); + setState(426); integerValue(); } break; @@ -3605,7 +3489,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(439); + setState(427); booleanValue(); } break; @@ -3613,7 +3497,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new InputParameterContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(440); + setState(428); parameter(); } break; @@ -3621,7 +3505,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(441); + setState(429); string(); } break; @@ -3629,27 +3513,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(442); + setState(430); match(OPENING_BRACKET); - setState(443); + setState(431); numericValue(); - setState(448); + setState(436); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(444); + setState(432); match(COMMA); - setState(445); + setState(433); numericValue(); } } - setState(450); + setState(438); _errHandler.sync(this); _la = _input.LA(1); } - setState(451); + setState(439); match(CLOSING_BRACKET); } break; @@ -3657,27 +3541,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(453); + setState(441); match(OPENING_BRACKET); - setState(454); + setState(442); booleanValue(); - setState(459); + setState(447); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(455); + setState(443); match(COMMA); - setState(456); + setState(444); booleanValue(); } } - setState(461); + setState(449); _errHandler.sync(this); _la = _input.LA(1); } - setState(462); + setState(450); match(CLOSING_BRACKET); } break; @@ -3685,27 +3569,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(464); + setState(452); match(OPENING_BRACKET); - setState(465); + setState(453); string(); - setState(470); + setState(458); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(466); + setState(454); match(COMMA); - setState(467); + setState(455); string(); } } - setState(472); + setState(460); _errHandler.sync(this); _la = _input.LA(1); } - setState(473); + setState(461); match(CLOSING_BRACKET); } break; @@ -3777,16 +3661,16 @@ public T accept(ParseTreeVisitor visitor) { public final ParameterContext parameter() throws RecognitionException { ParameterContext _localctx = new ParameterContext(_ctx, getState()); - enterRule(_localctx, 70, RULE_parameter); + enterRule(_localctx, 66, RULE_parameter); try { - setState(479); + setState(467); _errHandler.sync(this); switch (_input.LA(1)) { case PARAM: _localctx = new InputParamContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(477); + setState(465); match(PARAM); } break; @@ -3794,7 +3678,7 @@ public final ParameterContext parameter() throws RecognitionException { _localctx = new InputNamedOrPositionalParamContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(478); + setState(466); match(NAMED_OR_POSITIONAL_PARAM); } break; @@ -3843,24 +3727,24 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierOrParameterContext identifierOrParameter() throws RecognitionException { IdentifierOrParameterContext _localctx = new IdentifierOrParameterContext(_ctx, getState()); - enterRule(_localctx, 72, RULE_identifierOrParameter); + enterRule(_localctx, 68, RULE_identifierOrParameter); try { - setState(484); + setState(472); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(481); + setState(469); identifier(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(482); + setState(470); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(483); + setState(471); parameter(); } break; @@ -3903,13 +3787,13 @@ public T accept(ParseTreeVisitor visitor) { public final LimitCommandContext limitCommand() throws RecognitionException { LimitCommandContext _localctx = new LimitCommandContext(_ctx, getState()); - enterRule(_localctx, 74, RULE_limitCommand); + enterRule(_localctx, 70, RULE_limitCommand); try { enterOuterAlt(_localctx, 1); { - setState(486); + setState(474); match(LIMIT); - setState(487); + setState(475); match(INTEGER_LITERAL); } } @@ -3959,32 +3843,32 @@ public T accept(ParseTreeVisitor visitor) { public final SortCommandContext sortCommand() throws RecognitionException { SortCommandContext _localctx = new SortCommandContext(_ctx, getState()); - enterRule(_localctx, 76, RULE_sortCommand); + enterRule(_localctx, 72, RULE_sortCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(489); + setState(477); match(SORT); - setState(490); + setState(478); orderExpression(); - setState(495); + setState(483); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,45,_ctx); + _alt = getInterpreter().adaptivePredict(_input,44,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(491); + setState(479); match(COMMA); - setState(492); + setState(480); orderExpression(); } } } - setState(497); + setState(485); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,45,_ctx); + _alt = getInterpreter().adaptivePredict(_input,44,_ctx); } } } @@ -4033,19 +3917,19 @@ public T accept(ParseTreeVisitor visitor) { public final OrderExpressionContext orderExpression() throws RecognitionException { OrderExpressionContext _localctx = new OrderExpressionContext(_ctx, getState()); - enterRule(_localctx, 78, RULE_orderExpression); + enterRule(_localctx, 74, RULE_orderExpression); int _la; try { enterOuterAlt(_localctx, 1); { - setState(498); + setState(486); booleanExpression(0); - setState(500); + setState(488); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,45,_ctx) ) { case 1: { - setState(499); + setState(487); ((OrderExpressionContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -4059,14 +3943,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio } break; } - setState(504); + setState(492); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { case 1: { - setState(502); + setState(490); match(NULLS); - setState(503); + setState(491); ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1); _la = _input.LA(1); if ( !(_la==FIRST || _la==LAST) ) { @@ -4121,13 +4005,13 @@ public T accept(ParseTreeVisitor visitor) { public final KeepCommandContext keepCommand() throws RecognitionException { KeepCommandContext _localctx = new KeepCommandContext(_ctx, getState()); - enterRule(_localctx, 80, RULE_keepCommand); + enterRule(_localctx, 76, RULE_keepCommand); try { enterOuterAlt(_localctx, 1); { - setState(506); + setState(494); match(KEEP); - setState(507); + setState(495); qualifiedNamePatterns(); } } @@ -4170,13 +4054,13 @@ public T accept(ParseTreeVisitor visitor) { public final DropCommandContext dropCommand() throws RecognitionException { DropCommandContext _localctx = new DropCommandContext(_ctx, getState()); - enterRule(_localctx, 82, RULE_dropCommand); + enterRule(_localctx, 78, RULE_dropCommand); try { enterOuterAlt(_localctx, 1); { - setState(509); + setState(497); match(DROP); - setState(510); + setState(498); qualifiedNamePatterns(); } } @@ -4226,32 +4110,32 @@ public T accept(ParseTreeVisitor visitor) { public final RenameCommandContext renameCommand() throws RecognitionException { RenameCommandContext _localctx = new RenameCommandContext(_ctx, getState()); - enterRule(_localctx, 84, RULE_renameCommand); + enterRule(_localctx, 80, RULE_renameCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(512); + setState(500); match(RENAME); - setState(513); + setState(501); renameClause(); - setState(518); + setState(506); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,48,_ctx); + _alt = getInterpreter().adaptivePredict(_input,47,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(514); + setState(502); match(COMMA); - setState(515); + setState(503); renameClause(); } } } - setState(520); + setState(508); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,48,_ctx); + _alt = getInterpreter().adaptivePredict(_input,47,_ctx); } } } @@ -4299,15 +4183,15 @@ public T accept(ParseTreeVisitor visitor) { public final RenameClauseContext renameClause() throws RecognitionException { RenameClauseContext _localctx = new RenameClauseContext(_ctx, getState()); - enterRule(_localctx, 86, RULE_renameClause); + enterRule(_localctx, 82, RULE_renameClause); try { enterOuterAlt(_localctx, 1); { - setState(521); + setState(509); ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern(); - setState(522); + setState(510); match(AS); - setState(523); + setState(511); ((RenameClauseContext)_localctx).newName = qualifiedNamePattern(); } } @@ -4356,22 +4240,22 @@ public T accept(ParseTreeVisitor visitor) { public final DissectCommandContext dissectCommand() throws RecognitionException { DissectCommandContext _localctx = new DissectCommandContext(_ctx, getState()); - enterRule(_localctx, 88, RULE_dissectCommand); + enterRule(_localctx, 84, RULE_dissectCommand); try { enterOuterAlt(_localctx, 1); { - setState(525); + setState(513); match(DISSECT); - setState(526); + setState(514); primaryExpression(0); - setState(527); + setState(515); string(); - setState(529); + setState(517); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) { case 1: { - setState(528); + setState(516); commandOptions(); } break; @@ -4420,15 +4304,15 @@ public T accept(ParseTreeVisitor visitor) { public final GrokCommandContext grokCommand() throws RecognitionException { GrokCommandContext _localctx = new GrokCommandContext(_ctx, getState()); - enterRule(_localctx, 90, RULE_grokCommand); + enterRule(_localctx, 86, RULE_grokCommand); try { enterOuterAlt(_localctx, 1); { - setState(531); + setState(519); match(GROK); - setState(532); + setState(520); primaryExpression(0); - setState(533); + setState(521); string(); } } @@ -4471,13 +4355,13 @@ public T accept(ParseTreeVisitor visitor) { public final MvExpandCommandContext mvExpandCommand() throws RecognitionException { MvExpandCommandContext _localctx = new MvExpandCommandContext(_ctx, getState()); - enterRule(_localctx, 92, RULE_mvExpandCommand); + enterRule(_localctx, 88, RULE_mvExpandCommand); try { enterOuterAlt(_localctx, 1); { - setState(535); + setState(523); match(MV_EXPAND); - setState(536); + setState(524); qualifiedName(); } } @@ -4526,30 +4410,30 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionsContext commandOptions() throws RecognitionException { CommandOptionsContext _localctx = new CommandOptionsContext(_ctx, getState()); - enterRule(_localctx, 94, RULE_commandOptions); + enterRule(_localctx, 90, RULE_commandOptions); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(538); + setState(526); commandOption(); - setState(543); + setState(531); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,50,_ctx); + _alt = getInterpreter().adaptivePredict(_input,49,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(539); + setState(527); match(COMMA); - setState(540); + setState(528); commandOption(); } } } - setState(545); + setState(533); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,50,_ctx); + _alt = getInterpreter().adaptivePredict(_input,49,_ctx); } } } @@ -4595,15 +4479,15 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionContext commandOption() throws RecognitionException { CommandOptionContext _localctx = new CommandOptionContext(_ctx, getState()); - enterRule(_localctx, 96, RULE_commandOption); + enterRule(_localctx, 92, RULE_commandOption); try { enterOuterAlt(_localctx, 1); { - setState(546); + setState(534); identifier(); - setState(547); + setState(535); match(ASSIGN); - setState(548); + setState(536); constant(); } } @@ -4644,12 +4528,12 @@ public T accept(ParseTreeVisitor visitor) { public final BooleanValueContext booleanValue() throws RecognitionException { BooleanValueContext _localctx = new BooleanValueContext(_ctx, getState()); - enterRule(_localctx, 98, RULE_booleanValue); + enterRule(_localctx, 94, RULE_booleanValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(550); + setState(538); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -4702,22 +4586,22 @@ public T accept(ParseTreeVisitor visitor) { public final NumericValueContext numericValue() throws RecognitionException { NumericValueContext _localctx = new NumericValueContext(_ctx, getState()); - enterRule(_localctx, 100, RULE_numericValue); + enterRule(_localctx, 96, RULE_numericValue); try { - setState(554); + setState(542); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,50,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(552); + setState(540); decimalValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(553); + setState(541); integerValue(); } break; @@ -4761,17 +4645,17 @@ public T accept(ParseTreeVisitor visitor) { public final DecimalValueContext decimalValue() throws RecognitionException { DecimalValueContext _localctx = new DecimalValueContext(_ctx, getState()); - enterRule(_localctx, 102, RULE_decimalValue); + enterRule(_localctx, 98, RULE_decimalValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(557); + setState(545); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(556); + setState(544); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4784,7 +4668,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException { } } - setState(559); + setState(547); match(DECIMAL_LITERAL); } } @@ -4826,17 +4710,17 @@ public T accept(ParseTreeVisitor visitor) { public final IntegerValueContext integerValue() throws RecognitionException { IntegerValueContext _localctx = new IntegerValueContext(_ctx, getState()); - enterRule(_localctx, 104, RULE_integerValue); + enterRule(_localctx, 100, RULE_integerValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(562); + setState(550); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(561); + setState(549); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4849,7 +4733,7 @@ public final IntegerValueContext integerValue() throws RecognitionException { } } - setState(564); + setState(552); match(INTEGER_LITERAL); } } @@ -4889,11 +4773,11 @@ public T accept(ParseTreeVisitor visitor) { public final StringContext string() throws RecognitionException { StringContext _localctx = new StringContext(_ctx, getState()); - enterRule(_localctx, 106, RULE_string); + enterRule(_localctx, 102, RULE_string); try { enterOuterAlt(_localctx, 1); { - setState(566); + setState(554); match(QUOTED_STRING); } } @@ -4938,12 +4822,12 @@ public T accept(ParseTreeVisitor visitor) { public final ComparisonOperatorContext comparisonOperator() throws RecognitionException { ComparisonOperatorContext _localctx = new ComparisonOperatorContext(_ctx, getState()); - enterRule(_localctx, 108, RULE_comparisonOperator); + enterRule(_localctx, 104, RULE_comparisonOperator); int _la; try { enterOuterAlt(_localctx, 1); { - setState(568); + setState(556); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & -432345564227567616L) != 0)) ) { _errHandler.recoverInline(this); @@ -4994,13 +4878,13 @@ public T accept(ParseTreeVisitor visitor) { public final ExplainCommandContext explainCommand() throws RecognitionException { ExplainCommandContext _localctx = new ExplainCommandContext(_ctx, getState()); - enterRule(_localctx, 110, RULE_explainCommand); + enterRule(_localctx, 106, RULE_explainCommand); try { enterOuterAlt(_localctx, 1); { - setState(570); + setState(558); match(EXPLAIN); - setState(571); + setState(559); subqueryExpression(); } } @@ -5044,15 +4928,15 @@ public T accept(ParseTreeVisitor visitor) { public final SubqueryExpressionContext subqueryExpression() throws RecognitionException { SubqueryExpressionContext _localctx = new SubqueryExpressionContext(_ctx, getState()); - enterRule(_localctx, 112, RULE_subqueryExpression); + enterRule(_localctx, 108, RULE_subqueryExpression); try { enterOuterAlt(_localctx, 1); { - setState(573); + setState(561); match(OPENING_BRACKET); - setState(574); + setState(562); query(0); - setState(575); + setState(563); match(CLOSING_BRACKET); } } @@ -5104,14 +4988,14 @@ public T accept(ParseTreeVisitor visitor) { public final ShowCommandContext showCommand() throws RecognitionException { ShowCommandContext _localctx = new ShowCommandContext(_ctx, getState()); - enterRule(_localctx, 114, RULE_showCommand); + enterRule(_localctx, 110, RULE_showCommand); try { _localctx = new ShowInfoContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(577); + setState(565); match(SHOW); - setState(578); + setState(566); match(INFO); } } @@ -5169,53 +5053,53 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichCommandContext enrichCommand() throws RecognitionException { EnrichCommandContext _localctx = new EnrichCommandContext(_ctx, getState()); - enterRule(_localctx, 116, RULE_enrichCommand); + enterRule(_localctx, 112, RULE_enrichCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(580); + setState(568); match(ENRICH); - setState(581); + setState(569); ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME); - setState(584); + setState(572); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,54,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,53,_ctx) ) { case 1: { - setState(582); + setState(570); match(ON); - setState(583); + setState(571); ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern(); } break; } - setState(595); + setState(583); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,55,_ctx) ) { case 1: { - setState(586); + setState(574); match(WITH); - setState(587); + setState(575); enrichWithClause(); - setState(592); + setState(580); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,55,_ctx); + _alt = getInterpreter().adaptivePredict(_input,54,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(588); + setState(576); match(COMMA); - setState(589); + setState(577); enrichWithClause(); } } } - setState(594); + setState(582); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,55,_ctx); + _alt = getInterpreter().adaptivePredict(_input,54,_ctx); } } break; @@ -5266,23 +5150,23 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichWithClauseContext enrichWithClause() throws RecognitionException { EnrichWithClauseContext _localctx = new EnrichWithClauseContext(_ctx, getState()); - enterRule(_localctx, 118, RULE_enrichWithClause); + enterRule(_localctx, 114, RULE_enrichWithClause); try { enterOuterAlt(_localctx, 1); { - setState(600); + setState(588); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) { case 1: { - setState(597); + setState(585); ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern(); - setState(598); + setState(586); match(ASSIGN); } break; } - setState(602); + setState(590); ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern(); } } @@ -5331,17 +5215,17 @@ public T accept(ParseTreeVisitor visitor) { public final LookupCommandContext lookupCommand() throws RecognitionException { LookupCommandContext _localctx = new LookupCommandContext(_ctx, getState()); - enterRule(_localctx, 120, RULE_lookupCommand); + enterRule(_localctx, 116, RULE_lookupCommand); try { enterOuterAlt(_localctx, 1); { - setState(604); + setState(592); match(DEV_LOOKUP); - setState(605); + setState(593); ((LookupCommandContext)_localctx).tableName = indexPattern(); - setState(606); + setState(594); match(ON); - setState(607); + setState(595); ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns(); } } @@ -5390,22 +5274,22 @@ public T accept(ParseTreeVisitor visitor) { public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionException { InlinestatsCommandContext _localctx = new InlinestatsCommandContext(_ctx, getState()); - enterRule(_localctx, 122, RULE_inlinestatsCommand); + enterRule(_localctx, 118, RULE_inlinestatsCommand); try { enterOuterAlt(_localctx, 1); { - setState(609); + setState(597); match(DEV_INLINESTATS); - setState(610); + setState(598); ((InlinestatsCommandContext)_localctx).stats = aggFields(); - setState(613); + setState(601); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,58,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) { case 1: { - setState(611); + setState(599); match(BY); - setState(612); + setState(600); ((InlinestatsCommandContext)_localctx).grouping = fields(); } break; @@ -5458,17 +5342,17 @@ public T accept(ParseTreeVisitor visitor) { public final JoinCommandContext joinCommand() throws RecognitionException { JoinCommandContext _localctx = new JoinCommandContext(_ctx, getState()); - enterRule(_localctx, 124, RULE_joinCommand); + enterRule(_localctx, 120, RULE_joinCommand); int _la; try { enterOuterAlt(_localctx, 1); { - setState(616); + setState(604); _errHandler.sync(this); _la = _input.LA(1); if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 29360128L) != 0)) { { - setState(615); + setState(603); ((JoinCommandContext)_localctx).type = _input.LT(1); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 29360128L) != 0)) ) { @@ -5482,11 +5366,11 @@ public final JoinCommandContext joinCommand() throws RecognitionException { } } - setState(618); + setState(606); match(DEV_JOIN); - setState(619); + setState(607); joinTarget(); - setState(620); + setState(608); joinCondition(); } } @@ -5534,21 +5418,21 @@ public T accept(ParseTreeVisitor visitor) { public final JoinTargetContext joinTarget() throws RecognitionException { JoinTargetContext _localctx = new JoinTargetContext(_ctx, getState()); - enterRule(_localctx, 126, RULE_joinTarget); + enterRule(_localctx, 122, RULE_joinTarget); int _la; try { enterOuterAlt(_localctx, 1); { - setState(622); + setState(610); ((JoinTargetContext)_localctx).index = identifier(); - setState(625); + setState(613); _errHandler.sync(this); _la = _input.LA(1); if (_la==AS) { { - setState(623); + setState(611); match(AS); - setState(624); + setState(612); ((JoinTargetContext)_localctx).alias = identifier(); } } @@ -5601,32 +5485,32 @@ public T accept(ParseTreeVisitor visitor) { public final JoinConditionContext joinCondition() throws RecognitionException { JoinConditionContext _localctx = new JoinConditionContext(_ctx, getState()); - enterRule(_localctx, 128, RULE_joinCondition); + enterRule(_localctx, 124, RULE_joinCondition); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(627); + setState(615); match(ON); - setState(628); + setState(616); joinPredicate(); - setState(633); + setState(621); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,61,_ctx); + _alt = getInterpreter().adaptivePredict(_input,60,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(629); + setState(617); match(COMMA); - setState(630); + setState(618); joinPredicate(); } } } - setState(635); + setState(623); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,61,_ctx); + _alt = getInterpreter().adaptivePredict(_input,60,_ctx); } } } @@ -5668,11 +5552,11 @@ public T accept(ParseTreeVisitor visitor) { public final JoinPredicateContext joinPredicate() throws RecognitionException { JoinPredicateContext _localctx = new JoinPredicateContext(_ctx, getState()); - enterRule(_localctx, 130, RULE_joinPredicate); + enterRule(_localctx, 126, RULE_joinPredicate); try { enterOuterAlt(_localctx, 1); { - setState(636); + setState(624); valueExpression(); } } @@ -5701,9 +5585,9 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { return operatorExpression_sempred((OperatorExpressionContext)_localctx, predIndex); case 10: return primaryExpression_sempred((PrimaryExpressionContext)_localctx, predIndex); - case 33: + case 31: return identifierPattern_sempred((IdentifierPatternContext)_localctx, predIndex); - case 36: + case 34: return identifierOrParameter_sempred((IdentifierOrParameterContext)_localctx, predIndex); } return true; @@ -5774,7 +5658,7 @@ private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _loca } public static final String _serializedATN = - "\u0004\u0001\u0080\u027f\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ + "\u0004\u0001\u0080\u0273\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ "\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004"+ "\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007"+ "\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b"+ @@ -5790,392 +5674,385 @@ private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _loca ",\u0002-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u0007"+ "1\u00022\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u0007"+ "6\u00027\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007"+ - ";\u0002<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0002@\u0007"+ - "@\u0002A\u0007A\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001"+ - "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0005\u0001\u008e"+ - "\b\u0001\n\u0001\f\u0001\u0091\t\u0001\u0001\u0002\u0001\u0002\u0001\u0002"+ - "\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u0099\b\u0002\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003\u0003"+ - "\u00ad\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005"+ - "\u00b9\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0005\u0005\u00c0\b\u0005\n\u0005\f\u0005\u00c3\t\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00ca\b\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00cf\b\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00d7"+ - "\b\u0005\n\u0005\f\u0005\u00da\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006"+ - "\u00de\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006"+ - "\u0003\u0006\u00e5\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006"+ - "\u00ea\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007\u00ef\b"+ - "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001"+ - "\b\u0001\b\u0003\b\u00f9\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00ff"+ - "\b\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005\t\u0107\b\t"+ - "\n\t\f\t\u010a\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ - "\n\u0001\n\u0003\n\u0114\b\n\u0001\n\u0001\n\u0001\n\u0005\n\u0119\b\n"+ - "\n\n\f\n\u011c\t\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001"+ - "\u000b\u0001\u000b\u0005\u000b\u0124\b\u000b\n\u000b\f\u000b\u0127\t\u000b"+ - "\u0003\u000b\u0129\b\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001"+ - "\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f"+ - "\u0001\u000f\u0005\u000f\u0137\b\u000f\n\u000f\f\u000f\u013a\t\u000f\u0001"+ - "\u0010\u0001\u0010\u0001\u0010\u0003\u0010\u013f\b\u0010\u0001\u0010\u0001"+ - "\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0005\u0011\u0147"+ - "\b\u0011\n\u0011\f\u0011\u014a\t\u0011\u0001\u0011\u0003\u0011\u014d\b"+ - "\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0003\u0012\u0152\b\u0012\u0001"+ - "\u0012\u0001\u0012\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001"+ - "\u0015\u0001\u0015\u0003\u0015\u015c\b\u0015\u0001\u0016\u0001\u0016\u0001"+ - "\u0016\u0001\u0016\u0005\u0016\u0162\b\u0016\n\u0016\f\u0016\u0165\t\u0016"+ - "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018"+ - "\u0001\u0018\u0001\u0018\u0005\u0018\u016f\b\u0018\n\u0018\f\u0018\u0172"+ - "\t\u0018\u0001\u0018\u0003\u0018\u0175\b\u0018\u0001\u0018\u0001\u0018"+ - "\u0003\u0018\u0179\b\u0018\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u001a"+ - "\u0001\u001a\u0003\u001a\u0180\b\u001a\u0001\u001a\u0001\u001a\u0003\u001a"+ - "\u0184\b\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0005\u001b\u0189\b"+ - "\u001b\n\u001b\f\u001b\u018c\t\u001b\u0001\u001c\u0001\u001c\u0001\u001c"+ - "\u0003\u001c\u0191\b\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d"+ - "\u0196\b\u001d\n\u001d\f\u001d\u0199\t\u001d\u0001\u001e\u0001\u001e\u0001"+ - "\u001e\u0005\u001e\u019e\b\u001e\n\u001e\f\u001e\u01a1\t\u001e\u0001\u001f"+ - "\u0001\u001f\u0001\u001f\u0005\u001f\u01a6\b\u001f\n\u001f\f\u001f\u01a9"+ - "\t\u001f\u0001 \u0001 \u0001!\u0001!\u0001!\u0003!\u01b0\b!\u0001\"\u0001"+ - "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+ - "\"\u0001\"\u0001\"\u0005\"\u01bf\b\"\n\"\f\"\u01c2\t\"\u0001\"\u0001\""+ - "\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01ca\b\"\n\"\f\"\u01cd\t\"\u0001"+ - "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01d5\b\"\n\"\f\"\u01d8"+ - "\t\"\u0001\"\u0001\"\u0003\"\u01dc\b\"\u0001#\u0001#\u0003#\u01e0\b#\u0001"+ - "$\u0001$\u0001$\u0003$\u01e5\b$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001"+ - "&\u0001&\u0005&\u01ee\b&\n&\f&\u01f1\t&\u0001\'\u0001\'\u0003\'\u01f5"+ - "\b\'\u0001\'\u0001\'\u0003\'\u01f9\b\'\u0001(\u0001(\u0001(\u0001)\u0001"+ - ")\u0001)\u0001*\u0001*\u0001*\u0001*\u0005*\u0205\b*\n*\f*\u0208\t*\u0001"+ - "+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001,\u0001,\u0003,\u0212\b,\u0001"+ - "-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001.\u0001/\u0001/\u0001/\u0005"+ - "/\u021e\b/\n/\f/\u0221\t/\u00010\u00010\u00010\u00010\u00011\u00011\u0001"+ - "2\u00012\u00032\u022b\b2\u00013\u00033\u022e\b3\u00013\u00013\u00014\u0003"+ - "4\u0233\b4\u00014\u00014\u00015\u00015\u00016\u00016\u00017\u00017\u0001"+ - "7\u00018\u00018\u00018\u00018\u00019\u00019\u00019\u0001:\u0001:\u0001"+ - ":\u0001:\u0003:\u0249\b:\u0001:\u0001:\u0001:\u0001:\u0005:\u024f\b:\n"+ - ":\f:\u0252\t:\u0003:\u0254\b:\u0001;\u0001;\u0001;\u0003;\u0259\b;\u0001"+ - ";\u0001;\u0001<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001"+ - "=\u0003=\u0266\b=\u0001>\u0003>\u0269\b>\u0001>\u0001>\u0001>\u0001>\u0001"+ - "?\u0001?\u0001?\u0003?\u0272\b?\u0001@\u0001@\u0001@\u0001@\u0005@\u0278"+ - "\b@\n@\f@\u027b\t@\u0001A\u0001A\u0001A\u0000\u0004\u0002\n\u0012\u0014"+ - "B\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a"+ - "\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\^`bdfhjlnprtvxz|~\u0080\u0082"+ - "\u0000\t\u0001\u0000@A\u0001\u0000BD\u0002\u0000\u001e\u001eQQ\u0001\u0000"+ - "HI\u0002\u0000##((\u0002\u0000++..\u0002\u0000**88\u0002\u000099;?\u0001"+ - "\u0000\u0016\u0018\u0299\u0000\u0084\u0001\u0000\u0000\u0000\u0002\u0087"+ - "\u0001\u0000\u0000\u0000\u0004\u0098\u0001\u0000\u0000\u0000\u0006\u00ac"+ - "\u0001\u0000\u0000\u0000\b\u00ae\u0001\u0000\u0000\u0000\n\u00ce\u0001"+ - "\u0000\u0000\u0000\f\u00e9\u0001\u0000\u0000\u0000\u000e\u00eb\u0001\u0000"+ - "\u0000\u0000\u0010\u00f8\u0001\u0000\u0000\u0000\u0012\u00fe\u0001\u0000"+ - "\u0000\u0000\u0014\u0113\u0001\u0000\u0000\u0000\u0016\u011d\u0001\u0000"+ - "\u0000\u0000\u0018\u012c\u0001\u0000\u0000\u0000\u001a\u012e\u0001\u0000"+ - "\u0000\u0000\u001c\u0130\u0001\u0000\u0000\u0000\u001e\u0133\u0001\u0000"+ - "\u0000\u0000 \u013e\u0001\u0000\u0000\u0000\"\u0142\u0001\u0000\u0000"+ - "\u0000$\u0151\u0001\u0000\u0000\u0000&\u0155\u0001\u0000\u0000\u0000("+ - "\u0157\u0001\u0000\u0000\u0000*\u015b\u0001\u0000\u0000\u0000,\u015d\u0001"+ - "\u0000\u0000\u0000.\u0166\u0001\u0000\u0000\u00000\u016a\u0001\u0000\u0000"+ - "\u00002\u017a\u0001\u0000\u0000\u00004\u017d\u0001\u0000\u0000\u00006"+ - "\u0185\u0001\u0000\u0000\u00008\u018d\u0001\u0000\u0000\u0000:\u0192\u0001"+ - "\u0000\u0000\u0000<\u019a\u0001\u0000\u0000\u0000>\u01a2\u0001\u0000\u0000"+ - "\u0000@\u01aa\u0001\u0000\u0000\u0000B\u01af\u0001\u0000\u0000\u0000D"+ - "\u01db\u0001\u0000\u0000\u0000F\u01df\u0001\u0000\u0000\u0000H\u01e4\u0001"+ - "\u0000\u0000\u0000J\u01e6\u0001\u0000\u0000\u0000L\u01e9\u0001\u0000\u0000"+ - "\u0000N\u01f2\u0001\u0000\u0000\u0000P\u01fa\u0001\u0000\u0000\u0000R"+ - "\u01fd\u0001\u0000\u0000\u0000T\u0200\u0001\u0000\u0000\u0000V\u0209\u0001"+ - "\u0000\u0000\u0000X\u020d\u0001\u0000\u0000\u0000Z\u0213\u0001\u0000\u0000"+ - "\u0000\\\u0217\u0001\u0000\u0000\u0000^\u021a\u0001\u0000\u0000\u0000"+ - "`\u0222\u0001\u0000\u0000\u0000b\u0226\u0001\u0000\u0000\u0000d\u022a"+ - "\u0001\u0000\u0000\u0000f\u022d\u0001\u0000\u0000\u0000h\u0232\u0001\u0000"+ - "\u0000\u0000j\u0236\u0001\u0000\u0000\u0000l\u0238\u0001\u0000\u0000\u0000"+ - "n\u023a\u0001\u0000\u0000\u0000p\u023d\u0001\u0000\u0000\u0000r\u0241"+ - "\u0001\u0000\u0000\u0000t\u0244\u0001\u0000\u0000\u0000v\u0258\u0001\u0000"+ - "\u0000\u0000x\u025c\u0001\u0000\u0000\u0000z\u0261\u0001\u0000\u0000\u0000"+ - "|\u0268\u0001\u0000\u0000\u0000~\u026e\u0001\u0000\u0000\u0000\u0080\u0273"+ - "\u0001\u0000\u0000\u0000\u0082\u027c\u0001\u0000\u0000\u0000\u0084\u0085"+ - "\u0003\u0002\u0001\u0000\u0085\u0086\u0005\u0000\u0000\u0001\u0086\u0001"+ - "\u0001\u0000\u0000\u0000\u0087\u0088\u0006\u0001\uffff\uffff\u0000\u0088"+ - "\u0089\u0003\u0004\u0002\u0000\u0089\u008f\u0001\u0000\u0000\u0000\u008a"+ - "\u008b\n\u0001\u0000\u0000\u008b\u008c\u0005\u001d\u0000\u0000\u008c\u008e"+ - "\u0003\u0006\u0003\u0000\u008d\u008a\u0001\u0000\u0000\u0000\u008e\u0091"+ - "\u0001\u0000\u0000\u0000\u008f\u008d\u0001\u0000\u0000\u0000\u008f\u0090"+ - "\u0001\u0000\u0000\u0000\u0090\u0003\u0001\u0000\u0000\u0000\u0091\u008f"+ - "\u0001\u0000\u0000\u0000\u0092\u0099\u0003n7\u0000\u0093\u0099\u0003\""+ - "\u0011\u0000\u0094\u0099\u0003\u001c\u000e\u0000\u0095\u0099\u0003r9\u0000"+ - "\u0096\u0097\u0004\u0002\u0001\u0000\u0097\u0099\u00030\u0018\u0000\u0098"+ - "\u0092\u0001\u0000\u0000\u0000\u0098\u0093\u0001\u0000\u0000\u0000\u0098"+ - "\u0094\u0001\u0000\u0000\u0000\u0098\u0095\u0001\u0000\u0000\u0000\u0098"+ - "\u0096\u0001\u0000\u0000\u0000\u0099\u0005\u0001\u0000\u0000\u0000\u009a"+ - "\u00ad\u00032\u0019\u0000\u009b\u00ad\u0003\b\u0004\u0000\u009c\u00ad"+ - "\u0003P(\u0000\u009d\u00ad\u0003J%\u0000\u009e\u00ad\u00034\u001a\u0000"+ - "\u009f\u00ad\u0003L&\u0000\u00a0\u00ad\u0003R)\u0000\u00a1\u00ad\u0003"+ - "T*\u0000\u00a2\u00ad\u0003X,\u0000\u00a3\u00ad\u0003Z-\u0000\u00a4\u00ad"+ - "\u0003t:\u0000\u00a5\u00ad\u0003\\.\u0000\u00a6\u00a7\u0004\u0003\u0002"+ - "\u0000\u00a7\u00ad\u0003z=\u0000\u00a8\u00a9\u0004\u0003\u0003\u0000\u00a9"+ - "\u00ad\u0003x<\u0000\u00aa\u00ab\u0004\u0003\u0004\u0000\u00ab\u00ad\u0003"+ - "|>\u0000\u00ac\u009a\u0001\u0000\u0000\u0000\u00ac\u009b\u0001\u0000\u0000"+ - "\u0000\u00ac\u009c\u0001\u0000\u0000\u0000\u00ac\u009d\u0001\u0000\u0000"+ - "\u0000\u00ac\u009e\u0001\u0000\u0000\u0000\u00ac\u009f\u0001\u0000\u0000"+ - "\u0000\u00ac\u00a0\u0001\u0000\u0000\u0000\u00ac\u00a1\u0001\u0000\u0000"+ - "\u0000\u00ac\u00a2\u0001\u0000\u0000\u0000\u00ac\u00a3\u0001\u0000\u0000"+ - "\u0000\u00ac\u00a4\u0001\u0000\u0000\u0000\u00ac\u00a5\u0001\u0000\u0000"+ - "\u0000\u00ac\u00a6\u0001\u0000\u0000\u0000\u00ac\u00a8\u0001\u0000\u0000"+ - "\u0000\u00ac\u00aa\u0001\u0000\u0000\u0000\u00ad\u0007\u0001\u0000\u0000"+ - "\u0000\u00ae\u00af\u0005\u0010\u0000\u0000\u00af\u00b0\u0003\n\u0005\u0000"+ - "\u00b0\t\u0001\u0000\u0000\u0000\u00b1\u00b2\u0006\u0005\uffff\uffff\u0000"+ - "\u00b2\u00b3\u00051\u0000\u0000\u00b3\u00cf\u0003\n\u0005\b\u00b4\u00cf"+ - "\u0003\u0010\b\u0000\u00b5\u00cf\u0003\f\u0006\u0000\u00b6\u00b8\u0003"+ - "\u0010\b\u0000\u00b7\u00b9\u00051\u0000\u0000\u00b8\u00b7\u0001\u0000"+ - "\u0000\u0000\u00b8\u00b9\u0001\u0000\u0000\u0000\u00b9\u00ba\u0001\u0000"+ - "\u0000\u0000\u00ba\u00bb\u0005,\u0000\u0000\u00bb\u00bc\u00050\u0000\u0000"+ - "\u00bc\u00c1\u0003\u0010\b\u0000\u00bd\u00be\u0005\'\u0000\u0000\u00be"+ - "\u00c0\u0003\u0010\b\u0000\u00bf\u00bd\u0001\u0000\u0000\u0000\u00c0\u00c3"+ - "\u0001\u0000\u0000\u0000\u00c1\u00bf\u0001\u0000\u0000\u0000\u00c1\u00c2"+ - "\u0001\u0000\u0000\u0000\u00c2\u00c4\u0001\u0000\u0000\u0000\u00c3\u00c1"+ - "\u0001\u0000\u0000\u0000\u00c4\u00c5\u00057\u0000\u0000\u00c5\u00cf\u0001"+ - "\u0000\u0000\u0000\u00c6\u00c7\u0003\u0010\b\u0000\u00c7\u00c9\u0005-"+ - "\u0000\u0000\u00c8\u00ca\u00051\u0000\u0000\u00c9\u00c8\u0001\u0000\u0000"+ - "\u0000\u00c9\u00ca\u0001\u0000\u0000\u0000\u00ca\u00cb\u0001\u0000\u0000"+ - "\u0000\u00cb\u00cc\u00052\u0000\u0000\u00cc\u00cf\u0001\u0000\u0000\u0000"+ - "\u00cd\u00cf\u0003\u000e\u0007\u0000\u00ce\u00b1\u0001\u0000\u0000\u0000"+ - "\u00ce\u00b4\u0001\u0000\u0000\u0000\u00ce\u00b5\u0001\u0000\u0000\u0000"+ - "\u00ce\u00b6\u0001\u0000\u0000\u0000\u00ce\u00c6\u0001\u0000\u0000\u0000"+ - "\u00ce\u00cd\u0001\u0000\u0000\u0000\u00cf\u00d8\u0001\u0000\u0000\u0000"+ - "\u00d0\u00d1\n\u0005\u0000\u0000\u00d1\u00d2\u0005\"\u0000\u0000\u00d2"+ - "\u00d7\u0003\n\u0005\u0006\u00d3\u00d4\n\u0004\u0000\u0000\u00d4\u00d5"+ - "\u00054\u0000\u0000\u00d5\u00d7\u0003\n\u0005\u0005\u00d6\u00d0\u0001"+ - "\u0000\u0000\u0000\u00d6\u00d3\u0001\u0000\u0000\u0000\u00d7\u00da\u0001"+ - "\u0000\u0000\u0000\u00d8\u00d6\u0001\u0000\u0000\u0000\u00d8\u00d9\u0001"+ - "\u0000\u0000\u0000\u00d9\u000b\u0001\u0000\u0000\u0000\u00da\u00d8\u0001"+ - "\u0000\u0000\u0000\u00db\u00dd\u0003\u0010\b\u0000\u00dc\u00de\u00051"+ - "\u0000\u0000\u00dd\u00dc\u0001\u0000\u0000\u0000\u00dd\u00de\u0001\u0000"+ - "\u0000\u0000\u00de\u00df\u0001\u0000\u0000\u0000\u00df\u00e0\u0005/\u0000"+ - "\u0000\u00e0\u00e1\u0003j5\u0000\u00e1\u00ea\u0001\u0000\u0000\u0000\u00e2"+ - "\u00e4\u0003\u0010\b\u0000\u00e3\u00e5\u00051\u0000\u0000\u00e4\u00e3"+ - "\u0001\u0000\u0000\u0000\u00e4\u00e5\u0001\u0000\u0000\u0000\u00e5\u00e6"+ - "\u0001\u0000\u0000\u0000\u00e6\u00e7\u00056\u0000\u0000\u00e7\u00e8\u0003"+ - "j5\u0000\u00e8\u00ea\u0001\u0000\u0000\u0000\u00e9\u00db\u0001\u0000\u0000"+ - "\u0000\u00e9\u00e2\u0001\u0000\u0000\u0000\u00ea\r\u0001\u0000\u0000\u0000"+ - "\u00eb\u00ee\u0003:\u001d\u0000\u00ec\u00ed\u0005%\u0000\u0000\u00ed\u00ef"+ - "\u0003\u001a\r\u0000\u00ee\u00ec\u0001\u0000\u0000\u0000\u00ee\u00ef\u0001"+ - "\u0000\u0000\u0000\u00ef\u00f0\u0001\u0000\u0000\u0000\u00f0\u00f1\u0005"+ - "&\u0000\u0000\u00f1\u00f2\u0003D\"\u0000\u00f2\u000f\u0001\u0000\u0000"+ - "\u0000\u00f3\u00f9\u0003\u0012\t\u0000\u00f4\u00f5\u0003\u0012\t\u0000"+ - "\u00f5\u00f6\u0003l6\u0000\u00f6\u00f7\u0003\u0012\t\u0000\u00f7\u00f9"+ - "\u0001\u0000\u0000\u0000\u00f8\u00f3\u0001\u0000\u0000\u0000\u00f8\u00f4"+ - "\u0001\u0000\u0000\u0000\u00f9\u0011\u0001\u0000\u0000\u0000\u00fa\u00fb"+ - "\u0006\t\uffff\uffff\u0000\u00fb\u00ff\u0003\u0014\n\u0000\u00fc\u00fd"+ - "\u0007\u0000\u0000\u0000\u00fd\u00ff\u0003\u0012\t\u0003\u00fe\u00fa\u0001"+ - "\u0000\u0000\u0000\u00fe\u00fc\u0001\u0000\u0000\u0000\u00ff\u0108\u0001"+ - "\u0000\u0000\u0000\u0100\u0101\n\u0002\u0000\u0000\u0101\u0102\u0007\u0001"+ - "\u0000\u0000\u0102\u0107\u0003\u0012\t\u0003\u0103\u0104\n\u0001\u0000"+ - "\u0000\u0104\u0105\u0007\u0000\u0000\u0000\u0105\u0107\u0003\u0012\t\u0002"+ - "\u0106\u0100\u0001\u0000\u0000\u0000\u0106\u0103\u0001\u0000\u0000\u0000"+ - "\u0107\u010a\u0001\u0000\u0000\u0000\u0108\u0106\u0001\u0000\u0000\u0000"+ - "\u0108\u0109\u0001\u0000\u0000\u0000\u0109\u0013\u0001\u0000\u0000\u0000"+ - "\u010a\u0108\u0001\u0000\u0000\u0000\u010b\u010c\u0006\n\uffff\uffff\u0000"+ - "\u010c\u0114\u0003D\"\u0000\u010d\u0114\u0003:\u001d\u0000\u010e\u0114"+ - "\u0003\u0016\u000b\u0000\u010f\u0110\u00050\u0000\u0000\u0110\u0111\u0003"+ - "\n\u0005\u0000\u0111\u0112\u00057\u0000\u0000\u0112\u0114\u0001\u0000"+ - "\u0000\u0000\u0113\u010b\u0001\u0000\u0000\u0000\u0113\u010d\u0001\u0000"+ - "\u0000\u0000\u0113\u010e\u0001\u0000\u0000\u0000\u0113\u010f\u0001\u0000"+ - "\u0000\u0000\u0114\u011a\u0001\u0000\u0000\u0000\u0115\u0116\n\u0001\u0000"+ - "\u0000\u0116\u0117\u0005%\u0000\u0000\u0117\u0119\u0003\u001a\r\u0000"+ - "\u0118\u0115\u0001\u0000\u0000\u0000\u0119\u011c\u0001\u0000\u0000\u0000"+ - "\u011a\u0118\u0001\u0000\u0000\u0000\u011a\u011b\u0001\u0000\u0000\u0000"+ - "\u011b\u0015\u0001\u0000\u0000\u0000\u011c\u011a\u0001\u0000\u0000\u0000"+ - "\u011d\u011e\u0003\u0018\f\u0000\u011e\u0128\u00050\u0000\u0000\u011f"+ - "\u0129\u0005B\u0000\u0000\u0120\u0125\u0003\n\u0005\u0000\u0121\u0122"+ - "\u0005\'\u0000\u0000\u0122\u0124\u0003\n\u0005\u0000\u0123\u0121\u0001"+ - "\u0000\u0000\u0000\u0124\u0127\u0001\u0000\u0000\u0000\u0125\u0123\u0001"+ - "\u0000\u0000\u0000\u0125\u0126\u0001\u0000\u0000\u0000\u0126\u0129\u0001"+ - "\u0000\u0000\u0000\u0127\u0125\u0001\u0000\u0000\u0000\u0128\u011f\u0001"+ - "\u0000\u0000\u0000\u0128\u0120\u0001\u0000\u0000\u0000\u0128\u0129\u0001"+ - "\u0000\u0000\u0000\u0129\u012a\u0001\u0000\u0000\u0000\u012a\u012b\u0005"+ - "7\u0000\u0000\u012b\u0017\u0001\u0000\u0000\u0000\u012c\u012d\u0003H$"+ - "\u0000\u012d\u0019\u0001\u0000\u0000\u0000\u012e\u012f\u0003@ \u0000\u012f"+ - "\u001b\u0001\u0000\u0000\u0000\u0130\u0131\u0005\f\u0000\u0000\u0131\u0132"+ - "\u0003\u001e\u000f\u0000\u0132\u001d\u0001\u0000\u0000\u0000\u0133\u0138"+ - "\u0003 \u0010\u0000\u0134\u0135\u0005\'\u0000\u0000\u0135\u0137\u0003"+ - " \u0010\u0000\u0136\u0134\u0001\u0000\u0000\u0000\u0137\u013a\u0001\u0000"+ - "\u0000\u0000\u0138\u0136\u0001\u0000\u0000\u0000\u0138\u0139\u0001\u0000"+ - "\u0000\u0000\u0139\u001f\u0001\u0000\u0000\u0000\u013a\u0138\u0001\u0000"+ - "\u0000\u0000\u013b\u013c\u0003:\u001d\u0000\u013c\u013d\u0005$\u0000\u0000"+ - "\u013d\u013f\u0001\u0000\u0000\u0000\u013e\u013b\u0001\u0000\u0000\u0000"+ - "\u013e\u013f\u0001\u0000\u0000\u0000\u013f\u0140\u0001\u0000\u0000\u0000"+ - "\u0140\u0141\u0003\n\u0005\u0000\u0141!\u0001\u0000\u0000\u0000\u0142"+ - "\u0143\u0005\u0006\u0000\u0000\u0143\u0148\u0003$\u0012\u0000\u0144\u0145"+ - "\u0005\'\u0000\u0000\u0145\u0147\u0003$\u0012\u0000\u0146\u0144\u0001"+ - "\u0000\u0000\u0000\u0147\u014a\u0001\u0000\u0000\u0000\u0148\u0146\u0001"+ - "\u0000\u0000\u0000\u0148\u0149\u0001\u0000\u0000\u0000\u0149\u014c\u0001"+ - "\u0000\u0000\u0000\u014a\u0148\u0001\u0000\u0000\u0000\u014b\u014d\u0003"+ - "*\u0015\u0000\u014c\u014b\u0001\u0000\u0000\u0000\u014c\u014d\u0001\u0000"+ - "\u0000\u0000\u014d#\u0001\u0000\u0000\u0000\u014e\u014f\u0003&\u0013\u0000"+ - "\u014f\u0150\u0005&\u0000\u0000\u0150\u0152\u0001\u0000\u0000\u0000\u0151"+ - "\u014e\u0001\u0000\u0000\u0000\u0151\u0152\u0001\u0000\u0000\u0000\u0152"+ - "\u0153\u0001\u0000\u0000\u0000\u0153\u0154\u0003(\u0014\u0000\u0154%\u0001"+ - "\u0000\u0000\u0000\u0155\u0156\u0005Q\u0000\u0000\u0156\'\u0001\u0000"+ - "\u0000\u0000\u0157\u0158\u0007\u0002\u0000\u0000\u0158)\u0001\u0000\u0000"+ - "\u0000\u0159\u015c\u0003,\u0016\u0000\u015a\u015c\u0003.\u0017\u0000\u015b"+ - "\u0159\u0001\u0000\u0000\u0000\u015b\u015a\u0001\u0000\u0000\u0000\u015c"+ - "+\u0001\u0000\u0000\u0000\u015d\u015e\u0005P\u0000\u0000\u015e\u0163\u0005"+ - "Q\u0000\u0000\u015f\u0160\u0005\'\u0000\u0000\u0160\u0162\u0005Q\u0000"+ - "\u0000\u0161\u015f\u0001\u0000\u0000\u0000\u0162\u0165\u0001\u0000\u0000"+ - "\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0163\u0164\u0001\u0000\u0000"+ - "\u0000\u0164-\u0001\u0000\u0000\u0000\u0165\u0163\u0001\u0000\u0000\u0000"+ - "\u0166\u0167\u0005F\u0000\u0000\u0167\u0168\u0003,\u0016\u0000\u0168\u0169"+ - "\u0005G\u0000\u0000\u0169/\u0001\u0000\u0000\u0000\u016a\u016b\u0005\u0013"+ - "\u0000\u0000\u016b\u0170\u0003$\u0012\u0000\u016c\u016d\u0005\'\u0000"+ - "\u0000\u016d\u016f\u0003$\u0012\u0000\u016e\u016c\u0001\u0000\u0000\u0000"+ - "\u016f\u0172\u0001\u0000\u0000\u0000\u0170\u016e\u0001\u0000\u0000\u0000"+ - "\u0170\u0171\u0001\u0000\u0000\u0000\u0171\u0174\u0001\u0000\u0000\u0000"+ - "\u0172\u0170\u0001\u0000\u0000\u0000\u0173\u0175\u00036\u001b\u0000\u0174"+ - "\u0173\u0001\u0000\u0000\u0000\u0174\u0175\u0001\u0000\u0000\u0000\u0175"+ - "\u0178\u0001\u0000\u0000\u0000\u0176\u0177\u0005!\u0000\u0000\u0177\u0179"+ - "\u0003\u001e\u000f\u0000\u0178\u0176\u0001\u0000\u0000\u0000\u0178\u0179"+ - "\u0001\u0000\u0000\u0000\u01791\u0001\u0000\u0000\u0000\u017a\u017b\u0005"+ - "\u0004\u0000\u0000\u017b\u017c\u0003\u001e\u000f\u0000\u017c3\u0001\u0000"+ - "\u0000\u0000\u017d\u017f\u0005\u000f\u0000\u0000\u017e\u0180\u00036\u001b"+ - "\u0000\u017f\u017e\u0001\u0000\u0000\u0000\u017f\u0180\u0001\u0000\u0000"+ - "\u0000\u0180\u0183\u0001\u0000\u0000\u0000\u0181\u0182\u0005!\u0000\u0000"+ - "\u0182\u0184\u0003\u001e\u000f\u0000\u0183\u0181\u0001\u0000\u0000\u0000"+ - "\u0183\u0184\u0001\u0000\u0000\u0000\u01845\u0001\u0000\u0000\u0000\u0185"+ - "\u018a\u00038\u001c\u0000\u0186\u0187\u0005\'\u0000\u0000\u0187\u0189"+ - "\u00038\u001c\u0000\u0188\u0186\u0001\u0000\u0000\u0000\u0189\u018c\u0001"+ - "\u0000\u0000\u0000\u018a\u0188\u0001\u0000\u0000\u0000\u018a\u018b\u0001"+ - "\u0000\u0000\u0000\u018b7\u0001\u0000\u0000\u0000\u018c\u018a\u0001\u0000"+ - "\u0000\u0000\u018d\u0190\u0003 \u0010\u0000\u018e\u018f\u0005\u0010\u0000"+ - "\u0000\u018f\u0191\u0003\n\u0005\u0000\u0190\u018e\u0001\u0000\u0000\u0000"+ - "\u0190\u0191\u0001\u0000\u0000\u0000\u01919\u0001\u0000\u0000\u0000\u0192"+ - "\u0197\u0003H$\u0000\u0193\u0194\u0005)\u0000\u0000\u0194\u0196\u0003"+ - "H$\u0000\u0195\u0193\u0001\u0000\u0000\u0000\u0196\u0199\u0001\u0000\u0000"+ - "\u0000\u0197\u0195\u0001\u0000\u0000\u0000\u0197\u0198\u0001\u0000\u0000"+ - "\u0000\u0198;\u0001\u0000\u0000\u0000\u0199\u0197\u0001\u0000\u0000\u0000"+ - "\u019a\u019f\u0003B!\u0000\u019b\u019c\u0005)\u0000\u0000\u019c\u019e"+ - "\u0003B!\u0000\u019d\u019b\u0001\u0000\u0000\u0000\u019e\u01a1\u0001\u0000"+ - "\u0000\u0000\u019f\u019d\u0001\u0000\u0000\u0000\u019f\u01a0\u0001\u0000"+ - "\u0000\u0000\u01a0=\u0001\u0000\u0000\u0000\u01a1\u019f\u0001\u0000\u0000"+ - "\u0000\u01a2\u01a7\u0003<\u001e\u0000\u01a3\u01a4\u0005\'\u0000\u0000"+ - "\u01a4\u01a6\u0003<\u001e\u0000\u01a5\u01a3\u0001\u0000\u0000\u0000\u01a6"+ - "\u01a9\u0001\u0000\u0000\u0000\u01a7\u01a5\u0001\u0000\u0000\u0000\u01a7"+ - "\u01a8\u0001\u0000\u0000\u0000\u01a8?\u0001\u0000\u0000\u0000\u01a9\u01a7"+ - "\u0001\u0000\u0000\u0000\u01aa\u01ab\u0007\u0003\u0000\u0000\u01abA\u0001"+ - "\u0000\u0000\u0000\u01ac\u01b0\u0005U\u0000\u0000\u01ad\u01ae\u0004!\n"+ - "\u0000\u01ae\u01b0\u0003F#\u0000\u01af\u01ac\u0001\u0000\u0000\u0000\u01af"+ - "\u01ad\u0001\u0000\u0000\u0000\u01b0C\u0001\u0000\u0000\u0000\u01b1\u01dc"+ - "\u00052\u0000\u0000\u01b2\u01b3\u0003h4\u0000\u01b3\u01b4\u0005H\u0000"+ - "\u0000\u01b4\u01dc\u0001\u0000\u0000\u0000\u01b5\u01dc\u0003f3\u0000\u01b6"+ - "\u01dc\u0003h4\u0000\u01b7\u01dc\u0003b1\u0000\u01b8\u01dc\u0003F#\u0000"+ - "\u01b9\u01dc\u0003j5\u0000\u01ba\u01bb\u0005F\u0000\u0000\u01bb\u01c0"+ - "\u0003d2\u0000\u01bc\u01bd\u0005\'\u0000\u0000\u01bd\u01bf\u0003d2\u0000"+ - "\u01be\u01bc\u0001\u0000\u0000\u0000\u01bf\u01c2\u0001\u0000\u0000\u0000"+ - "\u01c0\u01be\u0001\u0000\u0000\u0000\u01c0\u01c1\u0001\u0000\u0000\u0000"+ - "\u01c1\u01c3\u0001\u0000\u0000\u0000\u01c2\u01c0\u0001\u0000\u0000\u0000"+ - "\u01c3\u01c4\u0005G\u0000\u0000\u01c4\u01dc\u0001\u0000\u0000\u0000\u01c5"+ - "\u01c6\u0005F\u0000\u0000\u01c6\u01cb\u0003b1\u0000\u01c7\u01c8\u0005"+ - "\'\u0000\u0000\u01c8\u01ca\u0003b1\u0000\u01c9\u01c7\u0001\u0000\u0000"+ - "\u0000\u01ca\u01cd\u0001\u0000\u0000\u0000\u01cb\u01c9\u0001\u0000\u0000"+ - "\u0000\u01cb\u01cc\u0001\u0000\u0000\u0000\u01cc\u01ce\u0001\u0000\u0000"+ - "\u0000\u01cd\u01cb\u0001\u0000\u0000\u0000\u01ce\u01cf\u0005G\u0000\u0000"+ - "\u01cf\u01dc\u0001\u0000\u0000\u0000\u01d0\u01d1\u0005F\u0000\u0000\u01d1"+ - "\u01d6\u0003j5\u0000\u01d2\u01d3\u0005\'\u0000\u0000\u01d3\u01d5\u0003"+ - "j5\u0000\u01d4\u01d2\u0001\u0000\u0000\u0000\u01d5\u01d8\u0001\u0000\u0000"+ - "\u0000\u01d6\u01d4\u0001\u0000\u0000\u0000\u01d6\u01d7\u0001\u0000\u0000"+ - "\u0000\u01d7\u01d9\u0001\u0000\u0000\u0000\u01d8\u01d6\u0001\u0000\u0000"+ - "\u0000\u01d9\u01da\u0005G\u0000\u0000\u01da\u01dc\u0001\u0000\u0000\u0000"+ - "\u01db\u01b1\u0001\u0000\u0000\u0000\u01db\u01b2\u0001\u0000\u0000\u0000"+ - "\u01db\u01b5\u0001\u0000\u0000\u0000\u01db\u01b6\u0001\u0000\u0000\u0000"+ - "\u01db\u01b7\u0001\u0000\u0000\u0000\u01db\u01b8\u0001\u0000\u0000\u0000"+ - "\u01db\u01b9\u0001\u0000\u0000\u0000\u01db\u01ba\u0001\u0000\u0000\u0000"+ - "\u01db\u01c5\u0001\u0000\u0000\u0000\u01db\u01d0\u0001\u0000\u0000\u0000"+ - "\u01dcE\u0001\u0000\u0000\u0000\u01dd\u01e0\u00055\u0000\u0000\u01de\u01e0"+ - "\u0005E\u0000\u0000\u01df\u01dd\u0001\u0000\u0000\u0000\u01df\u01de\u0001"+ - "\u0000\u0000\u0000\u01e0G\u0001\u0000\u0000\u0000\u01e1\u01e5\u0003@ "+ - "\u0000\u01e2\u01e3\u0004$\u000b\u0000\u01e3\u01e5\u0003F#\u0000\u01e4"+ - "\u01e1\u0001\u0000\u0000\u0000\u01e4\u01e2\u0001\u0000\u0000\u0000\u01e5"+ - "I\u0001\u0000\u0000\u0000\u01e6\u01e7\u0005\t\u0000\u0000\u01e7\u01e8"+ - "\u0005\u001f\u0000\u0000\u01e8K\u0001\u0000\u0000\u0000\u01e9\u01ea\u0005"+ - "\u000e\u0000\u0000\u01ea\u01ef\u0003N\'\u0000\u01eb\u01ec\u0005\'\u0000"+ - "\u0000\u01ec\u01ee\u0003N\'\u0000\u01ed\u01eb\u0001\u0000\u0000\u0000"+ - "\u01ee\u01f1\u0001\u0000\u0000\u0000\u01ef\u01ed\u0001\u0000\u0000\u0000"+ - "\u01ef\u01f0\u0001\u0000\u0000\u0000\u01f0M\u0001\u0000\u0000\u0000\u01f1"+ - "\u01ef\u0001\u0000\u0000\u0000\u01f2\u01f4\u0003\n\u0005\u0000\u01f3\u01f5"+ - "\u0007\u0004\u0000\u0000\u01f4\u01f3\u0001\u0000\u0000\u0000\u01f4\u01f5"+ - "\u0001\u0000\u0000\u0000\u01f5\u01f8\u0001\u0000\u0000\u0000\u01f6\u01f7"+ - "\u00053\u0000\u0000\u01f7\u01f9\u0007\u0005\u0000\u0000\u01f8\u01f6\u0001"+ - "\u0000\u0000\u0000\u01f8\u01f9\u0001\u0000\u0000\u0000\u01f9O\u0001\u0000"+ - "\u0000\u0000\u01fa\u01fb\u0005\b\u0000\u0000\u01fb\u01fc\u0003>\u001f"+ - "\u0000\u01fcQ\u0001\u0000\u0000\u0000\u01fd\u01fe\u0005\u0002\u0000\u0000"+ - "\u01fe\u01ff\u0003>\u001f\u0000\u01ffS\u0001\u0000\u0000\u0000\u0200\u0201"+ - "\u0005\u000b\u0000\u0000\u0201\u0206\u0003V+\u0000\u0202\u0203\u0005\'"+ - "\u0000\u0000\u0203\u0205\u0003V+\u0000\u0204\u0202\u0001\u0000\u0000\u0000"+ - "\u0205\u0208\u0001\u0000\u0000\u0000\u0206\u0204\u0001\u0000\u0000\u0000"+ - "\u0206\u0207\u0001\u0000\u0000\u0000\u0207U\u0001\u0000\u0000\u0000\u0208"+ - "\u0206\u0001\u0000\u0000\u0000\u0209\u020a\u0003<\u001e\u0000\u020a\u020b"+ - "\u0005Y\u0000\u0000\u020b\u020c\u0003<\u001e\u0000\u020cW\u0001\u0000"+ - "\u0000\u0000\u020d\u020e\u0005\u0001\u0000\u0000\u020e\u020f\u0003\u0014"+ - "\n\u0000\u020f\u0211\u0003j5\u0000\u0210\u0212\u0003^/\u0000\u0211\u0210"+ - "\u0001\u0000\u0000\u0000\u0211\u0212\u0001\u0000\u0000\u0000\u0212Y\u0001"+ - "\u0000\u0000\u0000\u0213\u0214\u0005\u0007\u0000\u0000\u0214\u0215\u0003"+ - "\u0014\n\u0000\u0215\u0216\u0003j5\u0000\u0216[\u0001\u0000\u0000\u0000"+ - "\u0217\u0218\u0005\n\u0000\u0000\u0218\u0219\u0003:\u001d\u0000\u0219"+ - "]\u0001\u0000\u0000\u0000\u021a\u021f\u0003`0\u0000\u021b\u021c\u0005"+ - "\'\u0000\u0000\u021c\u021e\u0003`0\u0000\u021d\u021b\u0001\u0000\u0000"+ - "\u0000\u021e\u0221\u0001\u0000\u0000\u0000\u021f\u021d\u0001\u0000\u0000"+ - "\u0000\u021f\u0220\u0001\u0000\u0000\u0000\u0220_\u0001\u0000\u0000\u0000"+ - "\u0221\u021f\u0001\u0000\u0000\u0000\u0222\u0223\u0003@ \u0000\u0223\u0224"+ - "\u0005$\u0000\u0000\u0224\u0225\u0003D\"\u0000\u0225a\u0001\u0000\u0000"+ - "\u0000\u0226\u0227\u0007\u0006\u0000\u0000\u0227c\u0001\u0000\u0000\u0000"+ - "\u0228\u022b\u0003f3\u0000\u0229\u022b\u0003h4\u0000\u022a\u0228\u0001"+ - "\u0000\u0000\u0000\u022a\u0229\u0001\u0000\u0000\u0000\u022be\u0001\u0000"+ - "\u0000\u0000\u022c\u022e\u0007\u0000\u0000\u0000\u022d\u022c\u0001\u0000"+ - "\u0000\u0000\u022d\u022e\u0001\u0000\u0000\u0000\u022e\u022f\u0001\u0000"+ - "\u0000\u0000\u022f\u0230\u0005 \u0000\u0000\u0230g\u0001\u0000\u0000\u0000"+ - "\u0231\u0233\u0007\u0000\u0000\u0000\u0232\u0231\u0001\u0000\u0000\u0000"+ - "\u0232\u0233\u0001\u0000\u0000\u0000\u0233\u0234\u0001\u0000\u0000\u0000"+ - "\u0234\u0235\u0005\u001f\u0000\u0000\u0235i\u0001\u0000\u0000\u0000\u0236"+ - "\u0237\u0005\u001e\u0000\u0000\u0237k\u0001\u0000\u0000\u0000\u0238\u0239"+ - "\u0007\u0007\u0000\u0000\u0239m\u0001\u0000\u0000\u0000\u023a\u023b\u0005"+ - "\u0005\u0000\u0000\u023b\u023c\u0003p8\u0000\u023co\u0001\u0000\u0000"+ - "\u0000\u023d\u023e\u0005F\u0000\u0000\u023e\u023f\u0003\u0002\u0001\u0000"+ - "\u023f\u0240\u0005G\u0000\u0000\u0240q\u0001\u0000\u0000\u0000\u0241\u0242"+ - "\u0005\r\u0000\u0000\u0242\u0243\u0005i\u0000\u0000\u0243s\u0001\u0000"+ - "\u0000\u0000\u0244\u0245\u0005\u0003\u0000\u0000\u0245\u0248\u0005_\u0000"+ - "\u0000\u0246\u0247\u0005]\u0000\u0000\u0247\u0249\u0003<\u001e\u0000\u0248"+ - "\u0246\u0001\u0000\u0000\u0000\u0248\u0249\u0001\u0000\u0000\u0000\u0249"+ - "\u0253\u0001\u0000\u0000\u0000\u024a\u024b\u0005^\u0000\u0000\u024b\u0250"+ - "\u0003v;\u0000\u024c\u024d\u0005\'\u0000\u0000\u024d\u024f\u0003v;\u0000"+ - "\u024e\u024c\u0001\u0000\u0000\u0000\u024f\u0252\u0001\u0000\u0000\u0000"+ - "\u0250\u024e\u0001\u0000\u0000\u0000\u0250\u0251\u0001\u0000\u0000\u0000"+ - "\u0251\u0254\u0001\u0000\u0000\u0000\u0252\u0250\u0001\u0000\u0000\u0000"+ - "\u0253\u024a\u0001\u0000\u0000\u0000\u0253\u0254\u0001\u0000\u0000\u0000"+ - "\u0254u\u0001\u0000\u0000\u0000\u0255\u0256\u0003<\u001e\u0000\u0256\u0257"+ - "\u0005$\u0000\u0000\u0257\u0259\u0001\u0000\u0000\u0000\u0258\u0255\u0001"+ - "\u0000\u0000\u0000\u0258\u0259\u0001\u0000\u0000\u0000\u0259\u025a\u0001"+ - "\u0000\u0000\u0000\u025a\u025b\u0003<\u001e\u0000\u025bw\u0001\u0000\u0000"+ - "\u0000\u025c\u025d\u0005\u0012\u0000\u0000\u025d\u025e\u0003$\u0012\u0000"+ - "\u025e\u025f\u0005]\u0000\u0000\u025f\u0260\u0003>\u001f\u0000\u0260y"+ - "\u0001\u0000\u0000\u0000\u0261\u0262\u0005\u0011\u0000\u0000\u0262\u0265"+ - "\u00036\u001b\u0000\u0263\u0264\u0005!\u0000\u0000\u0264\u0266\u0003\u001e"+ - "\u000f\u0000\u0265\u0263\u0001\u0000\u0000\u0000\u0265\u0266\u0001\u0000"+ - "\u0000\u0000\u0266{\u0001\u0000\u0000\u0000\u0267\u0269\u0007\b\u0000"+ - "\u0000\u0268\u0267\u0001\u0000\u0000\u0000\u0268\u0269\u0001\u0000\u0000"+ - "\u0000\u0269\u026a\u0001\u0000\u0000\u0000\u026a\u026b\u0005\u0014\u0000"+ - "\u0000\u026b\u026c\u0003~?\u0000\u026c\u026d\u0003\u0080@\u0000\u026d"+ - "}\u0001\u0000\u0000\u0000\u026e\u0271\u0003@ \u0000\u026f\u0270\u0005"+ - "Y\u0000\u0000\u0270\u0272\u0003@ \u0000\u0271\u026f\u0001\u0000\u0000"+ - "\u0000\u0271\u0272\u0001\u0000\u0000\u0000\u0272\u007f\u0001\u0000\u0000"+ - "\u0000\u0273\u0274\u0005]\u0000\u0000\u0274\u0279\u0003\u0082A\u0000\u0275"+ - "\u0276\u0005\'\u0000\u0000\u0276\u0278\u0003\u0082A\u0000\u0277\u0275"+ - "\u0001\u0000\u0000\u0000\u0278\u027b\u0001\u0000\u0000\u0000\u0279\u0277"+ - "\u0001\u0000\u0000\u0000\u0279\u027a\u0001\u0000\u0000\u0000\u027a\u0081"+ - "\u0001\u0000\u0000\u0000\u027b\u0279\u0001\u0000\u0000\u0000\u027c\u027d"+ - "\u0003\u0010\b\u0000\u027d\u0083\u0001\u0000\u0000\u0000>\u008f\u0098"+ - "\u00ac\u00b8\u00c1\u00c9\u00ce\u00d6\u00d8\u00dd\u00e4\u00e9\u00ee\u00f8"+ - "\u00fe\u0106\u0108\u0113\u011a\u0125\u0128\u0138\u013e\u0148\u014c\u0151"+ - "\u015b\u0163\u0170\u0174\u0178\u017f\u0183\u018a\u0190\u0197\u019f\u01a7"+ - "\u01af\u01c0\u01cb\u01d6\u01db\u01df\u01e4\u01ef\u01f4\u01f8\u0206\u0211"+ - "\u021f\u022a\u022d\u0232\u0248\u0250\u0253\u0258\u0265\u0268\u0271\u0279"; + ";\u0002<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0001\u0000"+ + "\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0001\u0001\u0005\u0001\u008a\b\u0001\n\u0001\f\u0001\u008d"+ + "\t\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ + "\u0002\u0003\u0002\u0095\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0003\u0003\u00a9\b\u0003\u0001\u0004\u0001"+ + "\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00b5\b\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00bc\b\u0005\n"+ + "\u0005\f\u0005\u00bf\t\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0003\u0005\u00c6\b\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0003\u0005\u00cb\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00d3\b\u0005\n\u0005\f\u0005"+ + "\u00d6\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006\u00da\b\u0006\u0001"+ + "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00e1"+ + "\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00e6\b\u0006"+ + "\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007\u00eb\b\u0007\u0001\u0007"+ + "\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0003"+ + "\b\u00f5\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00fb\b\t\u0001\t"+ + "\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005\t\u0103\b\t\n\t\f\t\u0106"+ + "\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0003"+ + "\n\u0110\b\n\u0001\n\u0001\n\u0001\n\u0005\n\u0115\b\n\n\n\f\n\u0118\t"+ + "\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ + "\u0005\u000b\u0120\b\u000b\n\u000b\f\u000b\u0123\t\u000b\u0003\u000b\u0125"+ + "\b\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\r\u0001\r\u0001"+ + "\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0005"+ + "\u000f\u0133\b\u000f\n\u000f\f\u000f\u0136\t\u000f\u0001\u0010\u0001\u0010"+ + "\u0001\u0010\u0003\u0010\u013b\b\u0010\u0001\u0010\u0001\u0010\u0001\u0011"+ + "\u0001\u0011\u0001\u0011\u0001\u0011\u0005\u0011\u0143\b\u0011\n\u0011"+ + "\f\u0011\u0146\t\u0011\u0001\u0011\u0003\u0011\u0149\b\u0011\u0001\u0012"+ + "\u0001\u0012\u0001\u0012\u0003\u0012\u014e\b\u0012\u0001\u0012\u0001\u0012"+ + "\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015"+ + "\u0001\u0015\u0001\u0015\u0005\u0015\u015a\b\u0015\n\u0015\f\u0015\u015d"+ + "\t\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0005\u0016\u0163"+ + "\b\u0016\n\u0016\f\u0016\u0166\t\u0016\u0001\u0016\u0003\u0016\u0169\b"+ + "\u0016\u0001\u0016\u0001\u0016\u0003\u0016\u016d\b\u0016\u0001\u0017\u0001"+ + "\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0003\u0018\u0174\b\u0018\u0001"+ + "\u0018\u0001\u0018\u0003\u0018\u0178\b\u0018\u0001\u0019\u0001\u0019\u0001"+ + "\u0019\u0005\u0019\u017d\b\u0019\n\u0019\f\u0019\u0180\t\u0019\u0001\u001a"+ + "\u0001\u001a\u0001\u001a\u0003\u001a\u0185\b\u001a\u0001\u001b\u0001\u001b"+ + "\u0001\u001b\u0005\u001b\u018a\b\u001b\n\u001b\f\u001b\u018d\t\u001b\u0001"+ + "\u001c\u0001\u001c\u0001\u001c\u0005\u001c\u0192\b\u001c\n\u001c\f\u001c"+ + "\u0195\t\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d\u019a\b"+ + "\u001d\n\u001d\f\u001d\u019d\t\u001d\u0001\u001e\u0001\u001e\u0001\u001f"+ + "\u0001\u001f\u0001\u001f\u0003\u001f\u01a4\b\u001f\u0001 \u0001 \u0001"+ + " \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001"+ + " \u0005 \u01b3\b \n \f \u01b6\t \u0001 \u0001 \u0001 \u0001 \u0001 \u0001"+ + " \u0005 \u01be\b \n \f \u01c1\t \u0001 \u0001 \u0001 \u0001 \u0001 \u0001"+ + " \u0005 \u01c9\b \n \f \u01cc\t \u0001 \u0001 \u0003 \u01d0\b \u0001!"+ + "\u0001!\u0003!\u01d4\b!\u0001\"\u0001\"\u0001\"\u0003\"\u01d9\b\"\u0001"+ + "#\u0001#\u0001#\u0001$\u0001$\u0001$\u0001$\u0005$\u01e2\b$\n$\f$\u01e5"+ + "\t$\u0001%\u0001%\u0003%\u01e9\b%\u0001%\u0001%\u0003%\u01ed\b%\u0001"+ + "&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001(\u0001(\u0001(\u0001(\u0005"+ + "(\u01f9\b(\n(\f(\u01fc\t(\u0001)\u0001)\u0001)\u0001)\u0001*\u0001*\u0001"+ + "*\u0001*\u0003*\u0206\b*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001"+ + ",\u0001-\u0001-\u0001-\u0005-\u0212\b-\n-\f-\u0215\t-\u0001.\u0001.\u0001"+ + ".\u0001.\u0001/\u0001/\u00010\u00010\u00030\u021f\b0\u00011\u00031\u0222"+ + "\b1\u00011\u00011\u00012\u00032\u0227\b2\u00012\u00012\u00013\u00013\u0001"+ + "4\u00014\u00015\u00015\u00015\u00016\u00016\u00016\u00016\u00017\u0001"+ + "7\u00017\u00018\u00018\u00018\u00018\u00038\u023d\b8\u00018\u00018\u0001"+ + "8\u00018\u00058\u0243\b8\n8\f8\u0246\t8\u00038\u0248\b8\u00019\u00019"+ + "\u00019\u00039\u024d\b9\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0001"+ + ":\u0001;\u0001;\u0001;\u0001;\u0003;\u025a\b;\u0001<\u0003<\u025d\b<\u0001"+ + "<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0003=\u0266\b=\u0001>\u0001"+ + ">\u0001>\u0001>\u0005>\u026c\b>\n>\f>\u026f\t>\u0001?\u0001?\u0001?\u0000"+ + "\u0004\u0002\n\u0012\u0014@\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+ + "\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPR"+ + "TVXZ\\^`bdfhjlnprtvxz|~\u0000\t\u0001\u0000@A\u0001\u0000BD\u0002\u0000"+ + "\u001e\u001eQQ\u0001\u0000HI\u0002\u0000##((\u0002\u0000++..\u0002\u0000"+ + "**88\u0002\u000099;?\u0001\u0000\u0016\u0018\u028e\u0000\u0080\u0001\u0000"+ + "\u0000\u0000\u0002\u0083\u0001\u0000\u0000\u0000\u0004\u0094\u0001\u0000"+ + "\u0000\u0000\u0006\u00a8\u0001\u0000\u0000\u0000\b\u00aa\u0001\u0000\u0000"+ + "\u0000\n\u00ca\u0001\u0000\u0000\u0000\f\u00e5\u0001\u0000\u0000\u0000"+ + "\u000e\u00e7\u0001\u0000\u0000\u0000\u0010\u00f4\u0001\u0000\u0000\u0000"+ + "\u0012\u00fa\u0001\u0000\u0000\u0000\u0014\u010f\u0001\u0000\u0000\u0000"+ + "\u0016\u0119\u0001\u0000\u0000\u0000\u0018\u0128\u0001\u0000\u0000\u0000"+ + "\u001a\u012a\u0001\u0000\u0000\u0000\u001c\u012c\u0001\u0000\u0000\u0000"+ + "\u001e\u012f\u0001\u0000\u0000\u0000 \u013a\u0001\u0000\u0000\u0000\""+ + "\u013e\u0001\u0000\u0000\u0000$\u014d\u0001\u0000\u0000\u0000&\u0151\u0001"+ + "\u0000\u0000\u0000(\u0153\u0001\u0000\u0000\u0000*\u0155\u0001\u0000\u0000"+ + "\u0000,\u015e\u0001\u0000\u0000\u0000.\u016e\u0001\u0000\u0000\u00000"+ + "\u0171\u0001\u0000\u0000\u00002\u0179\u0001\u0000\u0000\u00004\u0181\u0001"+ + "\u0000\u0000\u00006\u0186\u0001\u0000\u0000\u00008\u018e\u0001\u0000\u0000"+ + "\u0000:\u0196\u0001\u0000\u0000\u0000<\u019e\u0001\u0000\u0000\u0000>"+ + "\u01a3\u0001\u0000\u0000\u0000@\u01cf\u0001\u0000\u0000\u0000B\u01d3\u0001"+ + "\u0000\u0000\u0000D\u01d8\u0001\u0000\u0000\u0000F\u01da\u0001\u0000\u0000"+ + "\u0000H\u01dd\u0001\u0000\u0000\u0000J\u01e6\u0001\u0000\u0000\u0000L"+ + "\u01ee\u0001\u0000\u0000\u0000N\u01f1\u0001\u0000\u0000\u0000P\u01f4\u0001"+ + "\u0000\u0000\u0000R\u01fd\u0001\u0000\u0000\u0000T\u0201\u0001\u0000\u0000"+ + "\u0000V\u0207\u0001\u0000\u0000\u0000X\u020b\u0001\u0000\u0000\u0000Z"+ + "\u020e\u0001\u0000\u0000\u0000\\\u0216\u0001\u0000\u0000\u0000^\u021a"+ + "\u0001\u0000\u0000\u0000`\u021e\u0001\u0000\u0000\u0000b\u0221\u0001\u0000"+ + "\u0000\u0000d\u0226\u0001\u0000\u0000\u0000f\u022a\u0001\u0000\u0000\u0000"+ + "h\u022c\u0001\u0000\u0000\u0000j\u022e\u0001\u0000\u0000\u0000l\u0231"+ + "\u0001\u0000\u0000\u0000n\u0235\u0001\u0000\u0000\u0000p\u0238\u0001\u0000"+ + "\u0000\u0000r\u024c\u0001\u0000\u0000\u0000t\u0250\u0001\u0000\u0000\u0000"+ + "v\u0255\u0001\u0000\u0000\u0000x\u025c\u0001\u0000\u0000\u0000z\u0262"+ + "\u0001\u0000\u0000\u0000|\u0267\u0001\u0000\u0000\u0000~\u0270\u0001\u0000"+ + "\u0000\u0000\u0080\u0081\u0003\u0002\u0001\u0000\u0081\u0082\u0005\u0000"+ + "\u0000\u0001\u0082\u0001\u0001\u0000\u0000\u0000\u0083\u0084\u0006\u0001"+ + "\uffff\uffff\u0000\u0084\u0085\u0003\u0004\u0002\u0000\u0085\u008b\u0001"+ + "\u0000\u0000\u0000\u0086\u0087\n\u0001\u0000\u0000\u0087\u0088\u0005\u001d"+ + "\u0000\u0000\u0088\u008a\u0003\u0006\u0003\u0000\u0089\u0086\u0001\u0000"+ + "\u0000\u0000\u008a\u008d\u0001\u0000\u0000\u0000\u008b\u0089\u0001\u0000"+ + "\u0000\u0000\u008b\u008c\u0001\u0000\u0000\u0000\u008c\u0003\u0001\u0000"+ + "\u0000\u0000\u008d\u008b\u0001\u0000\u0000\u0000\u008e\u0095\u0003j5\u0000"+ + "\u008f\u0095\u0003\"\u0011\u0000\u0090\u0095\u0003\u001c\u000e\u0000\u0091"+ + "\u0095\u0003n7\u0000\u0092\u0093\u0004\u0002\u0001\u0000\u0093\u0095\u0003"+ + ",\u0016\u0000\u0094\u008e\u0001\u0000\u0000\u0000\u0094\u008f\u0001\u0000"+ + "\u0000\u0000\u0094\u0090\u0001\u0000\u0000\u0000\u0094\u0091\u0001\u0000"+ + "\u0000\u0000\u0094\u0092\u0001\u0000\u0000\u0000\u0095\u0005\u0001\u0000"+ + "\u0000\u0000\u0096\u00a9\u0003.\u0017\u0000\u0097\u00a9\u0003\b\u0004"+ + "\u0000\u0098\u00a9\u0003L&\u0000\u0099\u00a9\u0003F#\u0000\u009a\u00a9"+ + "\u00030\u0018\u0000\u009b\u00a9\u0003H$\u0000\u009c\u00a9\u0003N\'\u0000"+ + "\u009d\u00a9\u0003P(\u0000\u009e\u00a9\u0003T*\u0000\u009f\u00a9\u0003"+ + "V+\u0000\u00a0\u00a9\u0003p8\u0000\u00a1\u00a9\u0003X,\u0000\u00a2\u00a3"+ + "\u0004\u0003\u0002\u0000\u00a3\u00a9\u0003v;\u0000\u00a4\u00a5\u0004\u0003"+ + "\u0003\u0000\u00a5\u00a9\u0003t:\u0000\u00a6\u00a7\u0004\u0003\u0004\u0000"+ + "\u00a7\u00a9\u0003x<\u0000\u00a8\u0096\u0001\u0000\u0000\u0000\u00a8\u0097"+ + "\u0001\u0000\u0000\u0000\u00a8\u0098\u0001\u0000\u0000\u0000\u00a8\u0099"+ + "\u0001\u0000\u0000\u0000\u00a8\u009a\u0001\u0000\u0000\u0000\u00a8\u009b"+ + "\u0001\u0000\u0000\u0000\u00a8\u009c\u0001\u0000\u0000\u0000\u00a8\u009d"+ + "\u0001\u0000\u0000\u0000\u00a8\u009e\u0001\u0000\u0000\u0000\u00a8\u009f"+ + "\u0001\u0000\u0000\u0000\u00a8\u00a0\u0001\u0000\u0000\u0000\u00a8\u00a1"+ + "\u0001\u0000\u0000\u0000\u00a8\u00a2\u0001\u0000\u0000\u0000\u00a8\u00a4"+ + "\u0001\u0000\u0000\u0000\u00a8\u00a6\u0001\u0000\u0000\u0000\u00a9\u0007"+ + "\u0001\u0000\u0000\u0000\u00aa\u00ab\u0005\u0010\u0000\u0000\u00ab\u00ac"+ + "\u0003\n\u0005\u0000\u00ac\t\u0001\u0000\u0000\u0000\u00ad\u00ae\u0006"+ + "\u0005\uffff\uffff\u0000\u00ae\u00af\u00051\u0000\u0000\u00af\u00cb\u0003"+ + "\n\u0005\b\u00b0\u00cb\u0003\u0010\b\u0000\u00b1\u00cb\u0003\f\u0006\u0000"+ + "\u00b2\u00b4\u0003\u0010\b\u0000\u00b3\u00b5\u00051\u0000\u0000\u00b4"+ + "\u00b3\u0001\u0000\u0000\u0000\u00b4\u00b5\u0001\u0000\u0000\u0000\u00b5"+ + "\u00b6\u0001\u0000\u0000\u0000\u00b6\u00b7\u0005,\u0000\u0000\u00b7\u00b8"+ + "\u00050\u0000\u0000\u00b8\u00bd\u0003\u0010\b\u0000\u00b9\u00ba\u0005"+ + "\'\u0000\u0000\u00ba\u00bc\u0003\u0010\b\u0000\u00bb\u00b9\u0001\u0000"+ + "\u0000\u0000\u00bc\u00bf\u0001\u0000\u0000\u0000\u00bd\u00bb\u0001\u0000"+ + "\u0000\u0000\u00bd\u00be\u0001\u0000\u0000\u0000\u00be\u00c0\u0001\u0000"+ + "\u0000\u0000\u00bf\u00bd\u0001\u0000\u0000\u0000\u00c0\u00c1\u00057\u0000"+ + "\u0000\u00c1\u00cb\u0001\u0000\u0000\u0000\u00c2\u00c3\u0003\u0010\b\u0000"+ + "\u00c3\u00c5\u0005-\u0000\u0000\u00c4\u00c6\u00051\u0000\u0000\u00c5\u00c4"+ + "\u0001\u0000\u0000\u0000\u00c5\u00c6\u0001\u0000\u0000\u0000\u00c6\u00c7"+ + "\u0001\u0000\u0000\u0000\u00c7\u00c8\u00052\u0000\u0000\u00c8\u00cb\u0001"+ + "\u0000\u0000\u0000\u00c9\u00cb\u0003\u000e\u0007\u0000\u00ca\u00ad\u0001"+ + "\u0000\u0000\u0000\u00ca\u00b0\u0001\u0000\u0000\u0000\u00ca\u00b1\u0001"+ + "\u0000\u0000\u0000\u00ca\u00b2\u0001\u0000\u0000\u0000\u00ca\u00c2\u0001"+ + "\u0000\u0000\u0000\u00ca\u00c9\u0001\u0000\u0000\u0000\u00cb\u00d4\u0001"+ + "\u0000\u0000\u0000\u00cc\u00cd\n\u0005\u0000\u0000\u00cd\u00ce\u0005\""+ + "\u0000\u0000\u00ce\u00d3\u0003\n\u0005\u0006\u00cf\u00d0\n\u0004\u0000"+ + "\u0000\u00d0\u00d1\u00054\u0000\u0000\u00d1\u00d3\u0003\n\u0005\u0005"+ + "\u00d2\u00cc\u0001\u0000\u0000\u0000\u00d2\u00cf\u0001\u0000\u0000\u0000"+ + "\u00d3\u00d6\u0001\u0000\u0000\u0000\u00d4\u00d2\u0001\u0000\u0000\u0000"+ + "\u00d4\u00d5\u0001\u0000\u0000\u0000\u00d5\u000b\u0001\u0000\u0000\u0000"+ + "\u00d6\u00d4\u0001\u0000\u0000\u0000\u00d7\u00d9\u0003\u0010\b\u0000\u00d8"+ + "\u00da\u00051\u0000\u0000\u00d9\u00d8\u0001\u0000\u0000\u0000\u00d9\u00da"+ + "\u0001\u0000\u0000\u0000\u00da\u00db\u0001\u0000\u0000\u0000\u00db\u00dc"+ + "\u0005/\u0000\u0000\u00dc\u00dd\u0003f3\u0000\u00dd\u00e6\u0001\u0000"+ + "\u0000\u0000\u00de\u00e0\u0003\u0010\b\u0000\u00df\u00e1\u00051\u0000"+ + "\u0000\u00e0\u00df\u0001\u0000\u0000\u0000\u00e0\u00e1\u0001\u0000\u0000"+ + "\u0000\u00e1\u00e2\u0001\u0000\u0000\u0000\u00e2\u00e3\u00056\u0000\u0000"+ + "\u00e3\u00e4\u0003f3\u0000\u00e4\u00e6\u0001\u0000\u0000\u0000\u00e5\u00d7"+ + "\u0001\u0000\u0000\u0000\u00e5\u00de\u0001\u0000\u0000\u0000\u00e6\r\u0001"+ + "\u0000\u0000\u0000\u00e7\u00ea\u00036\u001b\u0000\u00e8\u00e9\u0005%\u0000"+ + "\u0000\u00e9\u00eb\u0003\u001a\r\u0000\u00ea\u00e8\u0001\u0000\u0000\u0000"+ + "\u00ea\u00eb\u0001\u0000\u0000\u0000\u00eb\u00ec\u0001\u0000\u0000\u0000"+ + "\u00ec\u00ed\u0005&\u0000\u0000\u00ed\u00ee\u0003@ \u0000\u00ee\u000f"+ + "\u0001\u0000\u0000\u0000\u00ef\u00f5\u0003\u0012\t\u0000\u00f0\u00f1\u0003"+ + "\u0012\t\u0000\u00f1\u00f2\u0003h4\u0000\u00f2\u00f3\u0003\u0012\t\u0000"+ + "\u00f3\u00f5\u0001\u0000\u0000\u0000\u00f4\u00ef\u0001\u0000\u0000\u0000"+ + "\u00f4\u00f0\u0001\u0000\u0000\u0000\u00f5\u0011\u0001\u0000\u0000\u0000"+ + "\u00f6\u00f7\u0006\t\uffff\uffff\u0000\u00f7\u00fb\u0003\u0014\n\u0000"+ + "\u00f8\u00f9\u0007\u0000\u0000\u0000\u00f9\u00fb\u0003\u0012\t\u0003\u00fa"+ + "\u00f6\u0001\u0000\u0000\u0000\u00fa\u00f8\u0001\u0000\u0000\u0000\u00fb"+ + "\u0104\u0001\u0000\u0000\u0000\u00fc\u00fd\n\u0002\u0000\u0000\u00fd\u00fe"+ + "\u0007\u0001\u0000\u0000\u00fe\u0103\u0003\u0012\t\u0003\u00ff\u0100\n"+ + "\u0001\u0000\u0000\u0100\u0101\u0007\u0000\u0000\u0000\u0101\u0103\u0003"+ + "\u0012\t\u0002\u0102\u00fc\u0001\u0000\u0000\u0000\u0102\u00ff\u0001\u0000"+ + "\u0000\u0000\u0103\u0106\u0001\u0000\u0000\u0000\u0104\u0102\u0001\u0000"+ + "\u0000\u0000\u0104\u0105\u0001\u0000\u0000\u0000\u0105\u0013\u0001\u0000"+ + "\u0000\u0000\u0106\u0104\u0001\u0000\u0000\u0000\u0107\u0108\u0006\n\uffff"+ + "\uffff\u0000\u0108\u0110\u0003@ \u0000\u0109\u0110\u00036\u001b\u0000"+ + "\u010a\u0110\u0003\u0016\u000b\u0000\u010b\u010c\u00050\u0000\u0000\u010c"+ + "\u010d\u0003\n\u0005\u0000\u010d\u010e\u00057\u0000\u0000\u010e\u0110"+ + "\u0001\u0000\u0000\u0000\u010f\u0107\u0001\u0000\u0000\u0000\u010f\u0109"+ + "\u0001\u0000\u0000\u0000\u010f\u010a\u0001\u0000\u0000\u0000\u010f\u010b"+ + "\u0001\u0000\u0000\u0000\u0110\u0116\u0001\u0000\u0000\u0000\u0111\u0112"+ + "\n\u0001\u0000\u0000\u0112\u0113\u0005%\u0000\u0000\u0113\u0115\u0003"+ + "\u001a\r\u0000\u0114\u0111\u0001\u0000\u0000\u0000\u0115\u0118\u0001\u0000"+ + "\u0000\u0000\u0116\u0114\u0001\u0000\u0000\u0000\u0116\u0117\u0001\u0000"+ + "\u0000\u0000\u0117\u0015\u0001\u0000\u0000\u0000\u0118\u0116\u0001\u0000"+ + "\u0000\u0000\u0119\u011a\u0003\u0018\f\u0000\u011a\u0124\u00050\u0000"+ + "\u0000\u011b\u0125\u0005B\u0000\u0000\u011c\u0121\u0003\n\u0005\u0000"+ + "\u011d\u011e\u0005\'\u0000\u0000\u011e\u0120\u0003\n\u0005\u0000\u011f"+ + "\u011d\u0001\u0000\u0000\u0000\u0120\u0123\u0001\u0000\u0000\u0000\u0121"+ + "\u011f\u0001\u0000\u0000\u0000\u0121\u0122\u0001\u0000\u0000\u0000\u0122"+ + "\u0125\u0001\u0000\u0000\u0000\u0123\u0121\u0001\u0000\u0000\u0000\u0124"+ + "\u011b\u0001\u0000\u0000\u0000\u0124\u011c\u0001\u0000\u0000\u0000\u0124"+ + "\u0125\u0001\u0000\u0000\u0000\u0125\u0126\u0001\u0000\u0000\u0000\u0126"+ + "\u0127\u00057\u0000\u0000\u0127\u0017\u0001\u0000\u0000\u0000\u0128\u0129"+ + "\u0003D\"\u0000\u0129\u0019\u0001\u0000\u0000\u0000\u012a\u012b\u0003"+ + "<\u001e\u0000\u012b\u001b\u0001\u0000\u0000\u0000\u012c\u012d\u0005\f"+ + "\u0000\u0000\u012d\u012e\u0003\u001e\u000f\u0000\u012e\u001d\u0001\u0000"+ + "\u0000\u0000\u012f\u0134\u0003 \u0010\u0000\u0130\u0131\u0005\'\u0000"+ + "\u0000\u0131\u0133\u0003 \u0010\u0000\u0132\u0130\u0001\u0000\u0000\u0000"+ + "\u0133\u0136\u0001\u0000\u0000\u0000\u0134\u0132\u0001\u0000\u0000\u0000"+ + "\u0134\u0135\u0001\u0000\u0000\u0000\u0135\u001f\u0001\u0000\u0000\u0000"+ + "\u0136\u0134\u0001\u0000\u0000\u0000\u0137\u0138\u00036\u001b\u0000\u0138"+ + "\u0139\u0005$\u0000\u0000\u0139\u013b\u0001\u0000\u0000\u0000\u013a\u0137"+ + "\u0001\u0000\u0000\u0000\u013a\u013b\u0001\u0000\u0000\u0000\u013b\u013c"+ + "\u0001\u0000\u0000\u0000\u013c\u013d\u0003\n\u0005\u0000\u013d!\u0001"+ + "\u0000\u0000\u0000\u013e\u013f\u0005\u0006\u0000\u0000\u013f\u0144\u0003"+ + "$\u0012\u0000\u0140\u0141\u0005\'\u0000\u0000\u0141\u0143\u0003$\u0012"+ + "\u0000\u0142\u0140\u0001\u0000\u0000\u0000\u0143\u0146\u0001\u0000\u0000"+ + "\u0000\u0144\u0142\u0001\u0000\u0000\u0000\u0144\u0145\u0001\u0000\u0000"+ + "\u0000\u0145\u0148\u0001\u0000\u0000\u0000\u0146\u0144\u0001\u0000\u0000"+ + "\u0000\u0147\u0149\u0003*\u0015\u0000\u0148\u0147\u0001\u0000\u0000\u0000"+ + "\u0148\u0149\u0001\u0000\u0000\u0000\u0149#\u0001\u0000\u0000\u0000\u014a"+ + "\u014b\u0003&\u0013\u0000\u014b\u014c\u0005&\u0000\u0000\u014c\u014e\u0001"+ + "\u0000\u0000\u0000\u014d\u014a\u0001\u0000\u0000\u0000\u014d\u014e\u0001"+ + "\u0000\u0000\u0000\u014e\u014f\u0001\u0000\u0000\u0000\u014f\u0150\u0003"+ + "(\u0014\u0000\u0150%\u0001\u0000\u0000\u0000\u0151\u0152\u0005Q\u0000"+ + "\u0000\u0152\'\u0001\u0000\u0000\u0000\u0153\u0154\u0007\u0002\u0000\u0000"+ + "\u0154)\u0001\u0000\u0000\u0000\u0155\u0156\u0005P\u0000\u0000\u0156\u015b"+ + "\u0005Q\u0000\u0000\u0157\u0158\u0005\'\u0000\u0000\u0158\u015a\u0005"+ + "Q\u0000\u0000\u0159\u0157\u0001\u0000\u0000\u0000\u015a\u015d\u0001\u0000"+ + "\u0000\u0000\u015b\u0159\u0001\u0000\u0000\u0000\u015b\u015c\u0001\u0000"+ + "\u0000\u0000\u015c+\u0001\u0000\u0000\u0000\u015d\u015b\u0001\u0000\u0000"+ + "\u0000\u015e\u015f\u0005\u0013\u0000\u0000\u015f\u0164\u0003$\u0012\u0000"+ + "\u0160\u0161\u0005\'\u0000\u0000\u0161\u0163\u0003$\u0012\u0000\u0162"+ + "\u0160\u0001\u0000\u0000\u0000\u0163\u0166\u0001\u0000\u0000\u0000\u0164"+ + "\u0162\u0001\u0000\u0000\u0000\u0164\u0165\u0001\u0000\u0000\u0000\u0165"+ + "\u0168\u0001\u0000\u0000\u0000\u0166\u0164\u0001\u0000\u0000\u0000\u0167"+ + "\u0169\u00032\u0019\u0000\u0168\u0167\u0001\u0000\u0000\u0000\u0168\u0169"+ + "\u0001\u0000\u0000\u0000\u0169\u016c\u0001\u0000\u0000\u0000\u016a\u016b"+ + "\u0005!\u0000\u0000\u016b\u016d\u0003\u001e\u000f\u0000\u016c\u016a\u0001"+ + "\u0000\u0000\u0000\u016c\u016d\u0001\u0000\u0000\u0000\u016d-\u0001\u0000"+ + "\u0000\u0000\u016e\u016f\u0005\u0004\u0000\u0000\u016f\u0170\u0003\u001e"+ + "\u000f\u0000\u0170/\u0001\u0000\u0000\u0000\u0171\u0173\u0005\u000f\u0000"+ + "\u0000\u0172\u0174\u00032\u0019\u0000\u0173\u0172\u0001\u0000\u0000\u0000"+ + "\u0173\u0174\u0001\u0000\u0000\u0000\u0174\u0177\u0001\u0000\u0000\u0000"+ + "\u0175\u0176\u0005!\u0000\u0000\u0176\u0178\u0003\u001e\u000f\u0000\u0177"+ + "\u0175\u0001\u0000\u0000\u0000\u0177\u0178\u0001\u0000\u0000\u0000\u0178"+ + "1\u0001\u0000\u0000\u0000\u0179\u017e\u00034\u001a\u0000\u017a\u017b\u0005"+ + "\'\u0000\u0000\u017b\u017d\u00034\u001a\u0000\u017c\u017a\u0001\u0000"+ + "\u0000\u0000\u017d\u0180\u0001\u0000\u0000\u0000\u017e\u017c\u0001\u0000"+ + "\u0000\u0000\u017e\u017f\u0001\u0000\u0000\u0000\u017f3\u0001\u0000\u0000"+ + "\u0000\u0180\u017e\u0001\u0000\u0000\u0000\u0181\u0184\u0003 \u0010\u0000"+ + "\u0182\u0183\u0005\u0010\u0000\u0000\u0183\u0185\u0003\n\u0005\u0000\u0184"+ + "\u0182\u0001\u0000\u0000\u0000\u0184\u0185\u0001\u0000\u0000\u0000\u0185"+ + "5\u0001\u0000\u0000\u0000\u0186\u018b\u0003D\"\u0000\u0187\u0188\u0005"+ + ")\u0000\u0000\u0188\u018a\u0003D\"\u0000\u0189\u0187\u0001\u0000\u0000"+ + "\u0000\u018a\u018d\u0001\u0000\u0000\u0000\u018b\u0189\u0001\u0000\u0000"+ + "\u0000\u018b\u018c\u0001\u0000\u0000\u0000\u018c7\u0001\u0000\u0000\u0000"+ + "\u018d\u018b\u0001\u0000\u0000\u0000\u018e\u0193\u0003>\u001f\u0000\u018f"+ + "\u0190\u0005)\u0000\u0000\u0190\u0192\u0003>\u001f\u0000\u0191\u018f\u0001"+ + "\u0000\u0000\u0000\u0192\u0195\u0001\u0000\u0000\u0000\u0193\u0191\u0001"+ + "\u0000\u0000\u0000\u0193\u0194\u0001\u0000\u0000\u0000\u01949\u0001\u0000"+ + "\u0000\u0000\u0195\u0193\u0001\u0000\u0000\u0000\u0196\u019b\u00038\u001c"+ + "\u0000\u0197\u0198\u0005\'\u0000\u0000\u0198\u019a\u00038\u001c\u0000"+ + "\u0199\u0197\u0001\u0000\u0000\u0000\u019a\u019d\u0001\u0000\u0000\u0000"+ + "\u019b\u0199\u0001\u0000\u0000\u0000\u019b\u019c\u0001\u0000\u0000\u0000"+ + "\u019c;\u0001\u0000\u0000\u0000\u019d\u019b\u0001\u0000\u0000\u0000\u019e"+ + "\u019f\u0007\u0003\u0000\u0000\u019f=\u0001\u0000\u0000\u0000\u01a0\u01a4"+ + "\u0005U\u0000\u0000\u01a1\u01a2\u0004\u001f\n\u0000\u01a2\u01a4\u0003"+ + "B!\u0000\u01a3\u01a0\u0001\u0000\u0000\u0000\u01a3\u01a1\u0001\u0000\u0000"+ + "\u0000\u01a4?\u0001\u0000\u0000\u0000\u01a5\u01d0\u00052\u0000\u0000\u01a6"+ + "\u01a7\u0003d2\u0000\u01a7\u01a8\u0005H\u0000\u0000\u01a8\u01d0\u0001"+ + "\u0000\u0000\u0000\u01a9\u01d0\u0003b1\u0000\u01aa\u01d0\u0003d2\u0000"+ + "\u01ab\u01d0\u0003^/\u0000\u01ac\u01d0\u0003B!\u0000\u01ad\u01d0\u0003"+ + "f3\u0000\u01ae\u01af\u0005F\u0000\u0000\u01af\u01b4\u0003`0\u0000\u01b0"+ + "\u01b1\u0005\'\u0000\u0000\u01b1\u01b3\u0003`0\u0000\u01b2\u01b0\u0001"+ + "\u0000\u0000\u0000\u01b3\u01b6\u0001\u0000\u0000\u0000\u01b4\u01b2\u0001"+ + "\u0000\u0000\u0000\u01b4\u01b5\u0001\u0000\u0000\u0000\u01b5\u01b7\u0001"+ + "\u0000\u0000\u0000\u01b6\u01b4\u0001\u0000\u0000\u0000\u01b7\u01b8\u0005"+ + "G\u0000\u0000\u01b8\u01d0\u0001\u0000\u0000\u0000\u01b9\u01ba\u0005F\u0000"+ + "\u0000\u01ba\u01bf\u0003^/\u0000\u01bb\u01bc\u0005\'\u0000\u0000\u01bc"+ + "\u01be\u0003^/\u0000\u01bd\u01bb\u0001\u0000\u0000\u0000\u01be\u01c1\u0001"+ + "\u0000\u0000\u0000\u01bf\u01bd\u0001\u0000\u0000\u0000\u01bf\u01c0\u0001"+ + "\u0000\u0000\u0000\u01c0\u01c2\u0001\u0000\u0000\u0000\u01c1\u01bf\u0001"+ + "\u0000\u0000\u0000\u01c2\u01c3\u0005G\u0000\u0000\u01c3\u01d0\u0001\u0000"+ + "\u0000\u0000\u01c4\u01c5\u0005F\u0000\u0000\u01c5\u01ca\u0003f3\u0000"+ + "\u01c6\u01c7\u0005\'\u0000\u0000\u01c7\u01c9\u0003f3\u0000\u01c8\u01c6"+ + "\u0001\u0000\u0000\u0000\u01c9\u01cc\u0001\u0000\u0000\u0000\u01ca\u01c8"+ + "\u0001\u0000\u0000\u0000\u01ca\u01cb\u0001\u0000\u0000\u0000\u01cb\u01cd"+ + "\u0001\u0000\u0000\u0000\u01cc\u01ca\u0001\u0000\u0000\u0000\u01cd\u01ce"+ + "\u0005G\u0000\u0000\u01ce\u01d0\u0001\u0000\u0000\u0000\u01cf\u01a5\u0001"+ + "\u0000\u0000\u0000\u01cf\u01a6\u0001\u0000\u0000\u0000\u01cf\u01a9\u0001"+ + "\u0000\u0000\u0000\u01cf\u01aa\u0001\u0000\u0000\u0000\u01cf\u01ab\u0001"+ + "\u0000\u0000\u0000\u01cf\u01ac\u0001\u0000\u0000\u0000\u01cf\u01ad\u0001"+ + "\u0000\u0000\u0000\u01cf\u01ae\u0001\u0000\u0000\u0000\u01cf\u01b9\u0001"+ + "\u0000\u0000\u0000\u01cf\u01c4\u0001\u0000\u0000\u0000\u01d0A\u0001\u0000"+ + "\u0000\u0000\u01d1\u01d4\u00055\u0000\u0000\u01d2\u01d4\u0005E\u0000\u0000"+ + "\u01d3\u01d1\u0001\u0000\u0000\u0000\u01d3\u01d2\u0001\u0000\u0000\u0000"+ + "\u01d4C\u0001\u0000\u0000\u0000\u01d5\u01d9\u0003<\u001e\u0000\u01d6\u01d7"+ + "\u0004\"\u000b\u0000\u01d7\u01d9\u0003B!\u0000\u01d8\u01d5\u0001\u0000"+ + "\u0000\u0000\u01d8\u01d6\u0001\u0000\u0000\u0000\u01d9E\u0001\u0000\u0000"+ + "\u0000\u01da\u01db\u0005\t\u0000\u0000\u01db\u01dc\u0005\u001f\u0000\u0000"+ + "\u01dcG\u0001\u0000\u0000\u0000\u01dd\u01de\u0005\u000e\u0000\u0000\u01de"+ + "\u01e3\u0003J%\u0000\u01df\u01e0\u0005\'\u0000\u0000\u01e0\u01e2\u0003"+ + "J%\u0000\u01e1\u01df\u0001\u0000\u0000\u0000\u01e2\u01e5\u0001\u0000\u0000"+ + "\u0000\u01e3\u01e1\u0001\u0000\u0000\u0000\u01e3\u01e4\u0001\u0000\u0000"+ + "\u0000\u01e4I\u0001\u0000\u0000\u0000\u01e5\u01e3\u0001\u0000\u0000\u0000"+ + "\u01e6\u01e8\u0003\n\u0005\u0000\u01e7\u01e9\u0007\u0004\u0000\u0000\u01e8"+ + "\u01e7\u0001\u0000\u0000\u0000\u01e8\u01e9\u0001\u0000\u0000\u0000\u01e9"+ + "\u01ec\u0001\u0000\u0000\u0000\u01ea\u01eb\u00053\u0000\u0000\u01eb\u01ed"+ + "\u0007\u0005\u0000\u0000\u01ec\u01ea\u0001\u0000\u0000\u0000\u01ec\u01ed"+ + "\u0001\u0000\u0000\u0000\u01edK\u0001\u0000\u0000\u0000\u01ee\u01ef\u0005"+ + "\b\u0000\u0000\u01ef\u01f0\u0003:\u001d\u0000\u01f0M\u0001\u0000\u0000"+ + "\u0000\u01f1\u01f2\u0005\u0002\u0000\u0000\u01f2\u01f3\u0003:\u001d\u0000"+ + "\u01f3O\u0001\u0000\u0000\u0000\u01f4\u01f5\u0005\u000b\u0000\u0000\u01f5"+ + "\u01fa\u0003R)\u0000\u01f6\u01f7\u0005\'\u0000\u0000\u01f7\u01f9\u0003"+ + "R)\u0000\u01f8\u01f6\u0001\u0000\u0000\u0000\u01f9\u01fc\u0001\u0000\u0000"+ + "\u0000\u01fa\u01f8\u0001\u0000\u0000\u0000\u01fa\u01fb\u0001\u0000\u0000"+ + "\u0000\u01fbQ\u0001\u0000\u0000\u0000\u01fc\u01fa\u0001\u0000\u0000\u0000"+ + "\u01fd\u01fe\u00038\u001c\u0000\u01fe\u01ff\u0005Y\u0000\u0000\u01ff\u0200"+ + "\u00038\u001c\u0000\u0200S\u0001\u0000\u0000\u0000\u0201\u0202\u0005\u0001"+ + "\u0000\u0000\u0202\u0203\u0003\u0014\n\u0000\u0203\u0205\u0003f3\u0000"+ + "\u0204\u0206\u0003Z-\u0000\u0205\u0204\u0001\u0000\u0000\u0000\u0205\u0206"+ + "\u0001\u0000\u0000\u0000\u0206U\u0001\u0000\u0000\u0000\u0207\u0208\u0005"+ + "\u0007\u0000\u0000\u0208\u0209\u0003\u0014\n\u0000\u0209\u020a\u0003f"+ + "3\u0000\u020aW\u0001\u0000\u0000\u0000\u020b\u020c\u0005\n\u0000\u0000"+ + "\u020c\u020d\u00036\u001b\u0000\u020dY\u0001\u0000\u0000\u0000\u020e\u0213"+ + "\u0003\\.\u0000\u020f\u0210\u0005\'\u0000\u0000\u0210\u0212\u0003\\.\u0000"+ + "\u0211\u020f\u0001\u0000\u0000\u0000\u0212\u0215\u0001\u0000\u0000\u0000"+ + "\u0213\u0211\u0001\u0000\u0000\u0000\u0213\u0214\u0001\u0000\u0000\u0000"+ + "\u0214[\u0001\u0000\u0000\u0000\u0215\u0213\u0001\u0000\u0000\u0000\u0216"+ + "\u0217\u0003<\u001e\u0000\u0217\u0218\u0005$\u0000\u0000\u0218\u0219\u0003"+ + "@ \u0000\u0219]\u0001\u0000\u0000\u0000\u021a\u021b\u0007\u0006\u0000"+ + "\u0000\u021b_\u0001\u0000\u0000\u0000\u021c\u021f\u0003b1\u0000\u021d"+ + "\u021f\u0003d2\u0000\u021e\u021c\u0001\u0000\u0000\u0000\u021e\u021d\u0001"+ + "\u0000\u0000\u0000\u021fa\u0001\u0000\u0000\u0000\u0220\u0222\u0007\u0000"+ + "\u0000\u0000\u0221\u0220\u0001\u0000\u0000\u0000\u0221\u0222\u0001\u0000"+ + "\u0000\u0000\u0222\u0223\u0001\u0000\u0000\u0000\u0223\u0224\u0005 \u0000"+ + "\u0000\u0224c\u0001\u0000\u0000\u0000\u0225\u0227\u0007\u0000\u0000\u0000"+ + "\u0226\u0225\u0001\u0000\u0000\u0000\u0226\u0227\u0001\u0000\u0000\u0000"+ + "\u0227\u0228\u0001\u0000\u0000\u0000\u0228\u0229\u0005\u001f\u0000\u0000"+ + "\u0229e\u0001\u0000\u0000\u0000\u022a\u022b\u0005\u001e\u0000\u0000\u022b"+ + "g\u0001\u0000\u0000\u0000\u022c\u022d\u0007\u0007\u0000\u0000\u022di\u0001"+ + "\u0000\u0000\u0000\u022e\u022f\u0005\u0005\u0000\u0000\u022f\u0230\u0003"+ + "l6\u0000\u0230k\u0001\u0000\u0000\u0000\u0231\u0232\u0005F\u0000\u0000"+ + "\u0232\u0233\u0003\u0002\u0001\u0000\u0233\u0234\u0005G\u0000\u0000\u0234"+ + "m\u0001\u0000\u0000\u0000\u0235\u0236\u0005\r\u0000\u0000\u0236\u0237"+ + "\u0005i\u0000\u0000\u0237o\u0001\u0000\u0000\u0000\u0238\u0239\u0005\u0003"+ + "\u0000\u0000\u0239\u023c\u0005_\u0000\u0000\u023a\u023b\u0005]\u0000\u0000"+ + "\u023b\u023d\u00038\u001c\u0000\u023c\u023a\u0001\u0000\u0000\u0000\u023c"+ + "\u023d\u0001\u0000\u0000\u0000\u023d\u0247\u0001\u0000\u0000\u0000\u023e"+ + "\u023f\u0005^\u0000\u0000\u023f\u0244\u0003r9\u0000\u0240\u0241\u0005"+ + "\'\u0000\u0000\u0241\u0243\u0003r9\u0000\u0242\u0240\u0001\u0000\u0000"+ + "\u0000\u0243\u0246\u0001\u0000\u0000\u0000\u0244\u0242\u0001\u0000\u0000"+ + "\u0000\u0244\u0245\u0001\u0000\u0000\u0000\u0245\u0248\u0001\u0000\u0000"+ + "\u0000\u0246\u0244\u0001\u0000\u0000\u0000\u0247\u023e\u0001\u0000\u0000"+ + "\u0000\u0247\u0248\u0001\u0000\u0000\u0000\u0248q\u0001\u0000\u0000\u0000"+ + "\u0249\u024a\u00038\u001c\u0000\u024a\u024b\u0005$\u0000\u0000\u024b\u024d"+ + "\u0001\u0000\u0000\u0000\u024c\u0249\u0001\u0000\u0000\u0000\u024c\u024d"+ + "\u0001\u0000\u0000\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e\u024f"+ + "\u00038\u001c\u0000\u024fs\u0001\u0000\u0000\u0000\u0250\u0251\u0005\u0012"+ + "\u0000\u0000\u0251\u0252\u0003$\u0012\u0000\u0252\u0253\u0005]\u0000\u0000"+ + "\u0253\u0254\u0003:\u001d\u0000\u0254u\u0001\u0000\u0000\u0000\u0255\u0256"+ + "\u0005\u0011\u0000\u0000\u0256\u0259\u00032\u0019\u0000\u0257\u0258\u0005"+ + "!\u0000\u0000\u0258\u025a\u0003\u001e\u000f\u0000\u0259\u0257\u0001\u0000"+ + "\u0000\u0000\u0259\u025a\u0001\u0000\u0000\u0000\u025aw\u0001\u0000\u0000"+ + "\u0000\u025b\u025d\u0007\b\u0000\u0000\u025c\u025b\u0001\u0000\u0000\u0000"+ + "\u025c\u025d\u0001\u0000\u0000\u0000\u025d\u025e\u0001\u0000\u0000\u0000"+ + "\u025e\u025f\u0005\u0014\u0000\u0000\u025f\u0260\u0003z=\u0000\u0260\u0261"+ + "\u0003|>\u0000\u0261y\u0001\u0000\u0000\u0000\u0262\u0265\u0003<\u001e"+ + "\u0000\u0263\u0264\u0005Y\u0000\u0000\u0264\u0266\u0003<\u001e\u0000\u0265"+ + "\u0263\u0001\u0000\u0000\u0000\u0265\u0266\u0001\u0000\u0000\u0000\u0266"+ + "{\u0001\u0000\u0000\u0000\u0267\u0268\u0005]\u0000\u0000\u0268\u026d\u0003"+ + "~?\u0000\u0269\u026a\u0005\'\u0000\u0000\u026a\u026c\u0003~?\u0000\u026b"+ + "\u0269\u0001\u0000\u0000\u0000\u026c\u026f\u0001\u0000\u0000\u0000\u026d"+ + "\u026b\u0001\u0000\u0000\u0000\u026d\u026e\u0001\u0000\u0000\u0000\u026e"+ + "}\u0001\u0000\u0000\u0000\u026f\u026d\u0001\u0000\u0000\u0000\u0270\u0271"+ + "\u0003\u0010\b\u0000\u0271\u007f\u0001\u0000\u0000\u0000=\u008b\u0094"+ + "\u00a8\u00b4\u00bd\u00c5\u00ca\u00d2\u00d4\u00d9\u00e0\u00e5\u00ea\u00f4"+ + "\u00fa\u0102\u0104\u010f\u0116\u0121\u0124\u0134\u013a\u0144\u0148\u014d"+ + "\u015b\u0164\u0168\u016c\u0173\u0177\u017e\u0184\u018b\u0193\u019b\u01a3"+ + "\u01b4\u01bf\u01ca\u01cf\u01d3\u01d8\u01e3\u01e8\u01ec\u01fa\u0205\u0213"+ + "\u021e\u0221\u0226\u023c\u0244\u0247\u024c\u0259\u025c\u0265\u026d"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java index 6071219839bab..2ee0efe52dfbe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java @@ -452,30 +452,6 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { *

The default implementation does nothing.

*/ @Override public void exitMetadata(EsqlBaseParser.MetadataContext ctx) { } - /** - * {@inheritDoc} - * - *

The default implementation does nothing.

- */ - @Override public void enterMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { } - /** - * {@inheritDoc} - * - *

The default implementation does nothing.

- */ - @Override public void exitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { } - /** - * {@inheritDoc} - * - *

The default implementation does nothing.

- */ - @Override public void enterDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { } - /** - * {@inheritDoc} - * - *

The default implementation does nothing.

- */ - @Override public void exitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java index afe7146923791..c5c1b0dfa7d7d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java @@ -272,20 +272,6 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitMetadata(EsqlBaseParser.MetadataContext ctx) { return visitChildren(ctx); } - /** - * {@inheritDoc} - * - *

The default implementation returns the result of calling - * {@link #visitChildren} on {@code ctx}.

- */ - @Override public T visitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { return visitChildren(ctx); } - /** - * {@inheritDoc} - * - *

The default implementation returns the result of calling - * {@link #visitChildren} on {@code ctx}.

- */ - @Override public T visitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java index 0faca2541c9ad..f45184e920658 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java @@ -415,26 +415,6 @@ public interface EsqlBaseParserListener extends ParseTreeListener { * @param ctx the parse tree */ void exitMetadata(EsqlBaseParser.MetadataContext ctx); - /** - * Enter a parse tree produced by {@link EsqlBaseParser#metadataOption}. - * @param ctx the parse tree - */ - void enterMetadataOption(EsqlBaseParser.MetadataOptionContext ctx); - /** - * Exit a parse tree produced by {@link EsqlBaseParser#metadataOption}. - * @param ctx the parse tree - */ - void exitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx); - /** - * Enter a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}. - * @param ctx the parse tree - */ - void enterDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx); - /** - * Exit a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}. - * @param ctx the parse tree - */ - void exitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx); /** * Enter a parse tree produced by {@link EsqlBaseParser#metricsCommand}. * @param ctx the parse tree diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java index e91cd6670e971..30c5e0ce78092 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java @@ -254,18 +254,6 @@ public interface EsqlBaseParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitMetadata(EsqlBaseParser.MetadataContext ctx); - /** - * Visit a parse tree produced by {@link EsqlBaseParser#metadataOption}. - * @param ctx the parse tree - * @return the visitor result - */ - T visitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx); - /** - * Visit a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}. - * @param ctx the parse tree - * @return the visitor result - */ - T visitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx); /** * Visit a parse tree produced by {@link EsqlBaseParser#metricsCommand}. * @param ctx the parse tree diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index 4edd0470058db..4b7c0118acda3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -35,7 +35,6 @@ import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern; import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; -import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.MetadataOptionContext; import org.elasticsearch.xpack.esql.plan.TableIdentifier; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Dissect; @@ -72,7 +71,6 @@ import java.util.function.Function; import static java.util.Collections.emptyList; -import static org.elasticsearch.common.logging.HeaderWarning.addWarning; import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions; import static org.elasticsearch.xpack.esql.parser.ParserUtils.source; @@ -261,21 +259,7 @@ public LogicalPlan visitFromCommand(EsqlBaseParser.FromCommandContext ctx) { TableIdentifier table = new TableIdentifier(source, null, visitIndexPattern(ctx.indexPattern())); Map metadataMap = new LinkedHashMap<>(); if (ctx.metadata() != null) { - var deprecatedContext = ctx.metadata().deprecated_metadata(); - MetadataOptionContext metadataOptionContext = null; - if (deprecatedContext != null) { - var s = source(deprecatedContext).source(); - addWarning( - "Line {}:{}: Square brackets '[]' need to be removed in FROM METADATA declaration", - s.getLineNumber(), - s.getColumnNumber() - ); - metadataOptionContext = deprecatedContext.metadataOption(); - } else { - metadataOptionContext = ctx.metadata().metadataOption(); - - } - for (var c : metadataOptionContext.UNQUOTED_SOURCE()) { + for (var c : ctx.metadata().UNQUOTED_SOURCE()) { String id = c.getText(); Source src = source(c); if (MetadataAttribute.isSupported(id) == false // TODO: drop check below once METADATA_SCORE is no longer snapshot-only diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index a4712ae77b5d8..a6243d25ba579 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -625,11 +625,8 @@ public void testInvalidQuotingAsFromIndexPattern() { expectError("FROM \"foo\"bar\"", ": token recognition error at: '\"'"); expectError("FROM \"foo\"\"bar\"", ": extraneous input '\"bar\"' expecting "); - expectError("FROM \"\"\"foo\"\"\"bar\"\"\"", ": mismatched input 'bar' expecting {, '|', ',', OPENING_BRACKET, 'metadata'}"); - expectError( - "FROM \"\"\"foo\"\"\"\"\"\"bar\"\"\"", - ": mismatched input '\"bar\"' expecting {, '|', ',', OPENING_BRACKET, 'metadata'}" - ); + expectError("FROM \"\"\"foo\"\"\"bar\"\"\"", ": mismatched input 'bar' expecting {, '|', ',', 'metadata'}"); + expectError("FROM \"\"\"foo\"\"\"\"\"\"bar\"\"\"", ": mismatched input '\"bar\"' expecting {, '|', ',', 'metadata'}"); } public void testInvalidQuotingAsMetricsIndexPattern() { @@ -909,10 +906,7 @@ public void testDeprecatedIsNullFunction() { public void testMetadataFieldOnOtherSources() { expectError("row a = 1 metadata _index", "line 1:20: extraneous input '_index' expecting "); expectError("show info metadata _index", "line 1:11: token recognition error at: 'm'"); - expectError( - "explain [from foo] metadata _index", - "line 1:20: mismatched input 'metadata' expecting {'|', ',', OPENING_BRACKET, ']', 'metadata'}" - ); + expectError("explain [from foo] metadata _index", "line 1:20: mismatched input 'metadata' expecting {'|', ',', ']', 'metadata'}"); } public void testMetadataFieldMultipleDeclarations() { @@ -2357,4 +2351,11 @@ public void testMatchOperatorFieldCasting() { assertThat(matchField.name(), equalTo("field")); assertThat(match.query().fold(FoldContext.small()), equalTo("value")); } + + public void testFailingMetadataWithSquareBrackets() { + expectError( + "FROM test [METADATA _index] | STATS count(*)", + "line 1:11: mismatched input '[' expecting {, '|', ',', 'metadata'}" + ); + } } diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml index 1b435c551fbe9..41bc0a96e737e 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml @@ -92,8 +92,6 @@ teardown: - do: catch: bad_request - allowed_warnings: - - "Line 1:21: Square brackets '[]' need to be removed in FROM METADATA declaration" headers: { Authorization: "Basic am9lOnMza3JpdC1wYXNzd29yZA==" } esql.query: body: @@ -104,12 +102,10 @@ teardown: - do: catch: bad_request - allowed_warnings: - - "Line 1:21: Square brackets '[]' need to be removed in FROM METADATA declaration" headers: { Authorization: "Basic am9lOnMza3JpdC1wYXNzd29yZA==" } esql.query: body: - query: 'FROM *:esql*,esql_* [METADATA _index] | sort cost | KEEP _index, tag, cost | LIMIT 10' + query: 'FROM *:esql*,esql_* METADATA _index | sort cost | KEEP _index, tag, cost | LIMIT 10' filter: range: since: From 35341330b2942a6a05af57567a1228223132526f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 14 Jan 2025 22:44:04 +1100 Subject: [PATCH 39/44] Mute org.elasticsearch.xpack.searchablesnapshots.minio.MinioSearchableSnapshotsIT org.elasticsearch.xpack.searchablesnapshots.minio.MinioSearchableSnapshotsIT #120101 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 3e69b7be0b2c3..0eb80b79b74c1 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -267,6 +267,8 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/118406 - class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT issue: https://github.com/elastic/elasticsearch/issues/120088 +- class: org.elasticsearch.xpack.searchablesnapshots.minio.MinioSearchableSnapshotsIT + issue: https://github.com/elastic/elasticsearch/issues/120101 # Examples: # From 2fe55dbc3ec83cc280ac2f791bfca1d9a8ee7d6d Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 14 Jan 2025 12:57:04 +0100 Subject: [PATCH 40/44] Minimize granted and denied fields in FLS (#119453) Instead of only determinizing, we can minimize the intermediate automata during FLS automaton construction. This brings the codebase back to its original form, before the Lucene 10 upgrade. Follow up to: https://github.com/elastic/elasticsearch/pull/119309 --- .../core/security/authz/permission/FieldPermissions.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java index 235d7419d2bf0..c46f1350776b1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.lucene.util.automaton.MinimizationOperations; import org.elasticsearch.plugins.FieldPredicate; import org.elasticsearch.xpack.core.security.authz.accesscontrol.FieldSubsetReader; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition.FieldGrantExcludeGroup; @@ -172,12 +173,8 @@ public static Automaton buildPermittedFieldsAutomaton(final String[] grantedFiel deniedFieldsAutomaton = Automatons.patterns(deniedFields); } - grantedFieldsAutomaton = Operations.removeDeadStates( - Operations.determinize(grantedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT) - ); - deniedFieldsAutomaton = Operations.removeDeadStates( - Operations.determinize(deniedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT) - ); + grantedFieldsAutomaton = MinimizationOperations.minimize(grantedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); + deniedFieldsAutomaton = MinimizationOperations.minimize(deniedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); if (Automatons.subsetOf(deniedFieldsAutomaton, grantedFieldsAutomaton) == false) { throw new ElasticsearchSecurityException( From a7d831e3009cc2fc2170e73861577e5fd18a4dd7 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 15 Jan 2025 00:45:54 +1100 Subject: [PATCH 41/44] Mute org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests #120115 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 0eb80b79b74c1..96f215b8b6815 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -269,6 +269,8 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/120088 - class: org.elasticsearch.xpack.searchablesnapshots.minio.MinioSearchableSnapshotsIT issue: https://github.com/elastic/elasticsearch/issues/120101 +- class: org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests + issue: https://github.com/elastic/elasticsearch/issues/120115 # Examples: # From c638f746763733532e3c9db018ea72916955ac3a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 15 Jan 2025 00:49:53 +1100 Subject: [PATCH 42/44] Mute org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT #120117 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 96f215b8b6815..9cf2ef065f82f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -271,6 +271,8 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/120101 - class: org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests issue: https://github.com/elastic/elasticsearch/issues/120115 +- class: org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT + issue: https://github.com/elastic/elasticsearch/issues/120117 # Examples: # From fc8e2fcc22d354ebfd37d99aa9f024e6a9989dda Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:07:48 +0100 Subject: [PATCH 43/44] Move source mode setting from SourceFieldMapper to IndexSettings (#120096) Here we move the `index.mapping.source.mode` setting to `IndexSettings` because of dependencies and because of the initialisation order of static fields for classes `IndexSettings` and `SourceFieldMapper`. Not initialising settings `index.mode`, `index.mapping.source.mode`, and `index.recovery.use_synthetic_source` in the right order results in multiple `NullPointerException`. This work is done to simplify another PR #119110 --- .../admin/cluster/stats/MappingStats.java | 4 +- .../metadata/MetadataCreateIndexService.java | 3 +- .../common/settings/IndexScopedSettings.java | 3 +- .../elasticsearch/index/IndexSettings.java | 97 +++++++++++-------- .../index/mapper/SourceFieldMapper.java | 16 +-- .../snapshots/RestoreService.java | 3 +- .../cluster/stats/MappingStatsTests.java | 8 +- ...neSyntheticSourceChangesSnapshotTests.java | 4 +- .../TranslogOperationAsserterTests.java | 8 +- ...edSourceFieldMapperConfigurationTests.java | 3 +- .../index/mapper/SourceFieldMapperTests.java | 30 +++--- .../index/mapper/MapperTestCase.java | 2 +- .../xpack/ccr/FollowIndexIT.java | 2 +- .../TransportResumeFollowActionTests.java | 3 +- .../SourceOnlySnapshotShardTests.java | 4 +- .../logsdb/LogsDBUsageTransportAction.java | 5 +- .../LogsdbIndexModeSettingsProvider.java | 6 +- .../logsdb/LegacyLicenceIntegrationTests.java | 6 +- .../LogsdbIndexModeSettingsProviderTests.java | 10 +- ...dexSettingsProviderLegacyLicenseTests.java | 16 +-- 20 files changed, 115 insertions(+), 118 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java index 1bc2e1d13c864..29a124b3d0b20 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java @@ -21,7 +21,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -71,7 +71,7 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) { } AnalysisStats.countMapping(mappingCounts, indexMetadata); - var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()); + var sourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()); sourceModeUsageCount.merge(sourceMode.toString().toLowerCase(Locale.ENGLISH), 1, Integer::sum); } final AtomicLong totalFieldCount = new AtomicLong(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index fb19a3f04fcec..e24d921ba78fd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -69,7 +69,6 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.indices.IndexCreationException; import org.elasticsearch.indices.IndicesService; @@ -1586,7 +1585,7 @@ static void validateCloneIndex( private static final Set UNMODIFIABLE_SETTINGS_DURING_RESIZE = Set.of( IndexSettings.MODE.getKey(), - SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index afee2491672ef..3c1f53ca4a2c9 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -36,7 +36,6 @@ import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; import org.elasticsearch.index.store.Store; @@ -191,7 +190,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, - SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING, + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING, IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING, InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 68f334b10ea52..766b6ecbc7b9d 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -52,7 +52,6 @@ import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; /** * This class encapsulates all index level settings and handles settings updates. @@ -655,48 +654,6 @@ public Iterator> settings() { Property.Final ); - public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( - "index.recovery.use_synthetic_source", - false, - new Setting.Validator<>() { - @Override - public void validate(Boolean value) {} - - @Override - public void validate(Boolean enabled, Map, Object> settings) { - if (enabled == false) { - return; - } - - // Verify if synthetic source is enabled on the index; fail if it is not - var indexMode = (IndexMode) settings.get(MODE); - if (indexMode.defaultSourceMode() != SourceFieldMapper.Mode.SYNTHETIC) { - var sourceMode = (SourceFieldMapper.Mode) settings.get(INDEX_MAPPER_SOURCE_MODE_SETTING); - if (sourceMode != SourceFieldMapper.Mode.SYNTHETIC) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", - RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), - INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), - SourceFieldMapper.Mode.SYNTHETIC.name(), - sourceMode.name() - ) - ); - } - } - } - - @Override - public Iterator> settings() { - List> res = List.of(INDEX_MAPPER_SOURCE_MODE_SETTING, MODE); - return res.iterator(); - } - }, - Property.IndexScope, - Property.Final - ); - /** * Returns true if TSDB encoding is enabled. The default is true */ @@ -753,6 +710,60 @@ public Iterator> settings() { Property.ServerlessPublic ); + public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting( + SourceFieldMapper.Mode.class, + settings -> { + final IndexMode indexMode = IndexSettings.MODE.get(settings); + return indexMode.defaultSourceMode().name(); + }, + "index.mapping.source.mode", + value -> {}, + Setting.Property.Final, + Setting.Property.IndexScope + ); + + public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( + "index.recovery.use_synthetic_source", + false, + new Setting.Validator<>() { + @Override + public void validate(Boolean value) {} + + @Override + public void validate(Boolean enabled, Map, Object> settings) { + if (enabled == false) { + return; + } + + // Verify if synthetic source is enabled on the index; fail if it is not + var indexMode = (IndexMode) settings.get(MODE); + if (indexMode.defaultSourceMode() != SourceFieldMapper.Mode.SYNTHETIC) { + var sourceMode = (SourceFieldMapper.Mode) settings.get(INDEX_MAPPER_SOURCE_MODE_SETTING); + if (sourceMode != SourceFieldMapper.Mode.SYNTHETIC) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", + RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), + INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), + SourceFieldMapper.Mode.SYNTHETIC.name(), + sourceMode.name() + ) + ); + } + } + } + + @Override + public Iterator> settings() { + List> res = List.of(INDEX_MAPPER_SOURCE_MODE_SETTING, MODE); + return res.iterator(); + } + }, + Property.IndexScope, + Property.Final + ); + /** * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use. * TODO: Remove in 9.0 diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 29acdf200692d..36335be58ce94 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.core.Nullable; @@ -70,11 +69,6 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final String LOSSY_PARAMETERS_ALLOWED_SETTING_NAME = "index.lossy.source-mapping-parameters"; - public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> { - final IndexMode indexMode = IndexSettings.MODE.get(settings); - return indexMode.defaultSourceMode().name(); - }, "index.mapping.source.mode", value -> {}, Setting.Property.Final, Setting.Property.IndexScope); - public static final String DEPRECATION_WARNING = "Configuring source mode in mappings is deprecated and will be removed " + "in future versions. Use [index.mapping.source.mode] index setting instead."; @@ -264,8 +258,8 @@ public SourceFieldMapper build() { private Mode resolveSourceMode() { // If the `index.mapping.source.mode` exists it takes precedence to determine the source mode for `_source` // otherwise the mode is determined according to `_source.mode`. - if (INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)) { - return INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); + if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)) { + return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); } // If `_source.mode` is not set we need to apply a default according to index mode. @@ -297,7 +291,7 @@ private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) { return DEFAULT; } - final Mode settingSourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings()); + final Mode settingSourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings()); // Needed for bwc so that "mode" is not serialized in case of standard index with stored source. if (indexMode == IndexMode.STANDARD && settingSourceMode == Mode.STORED) { return DEFAULT; @@ -482,11 +476,11 @@ public boolean isSynthetic() { } public static boolean isSynthetic(IndexSettings indexSettings) { - return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC; + return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC; } public static boolean isStored(IndexSettings indexSettings) { - return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == Mode.STORED; + return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == Mode.STORED; } public boolean isDisabled() { diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index f260c7aad30e5..debe3d6e6bd92 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -70,7 +70,6 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; @@ -158,7 +157,7 @@ public final class RestoreService implements ClusterStateApplier { SETTING_CREATION_DATE, SETTING_HISTORY_UUID, IndexSettings.MODE.getKey(), - SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java index 96954458c18c4..0617053769da0 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java @@ -17,8 +17,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.script.Script; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -586,7 +586,7 @@ public void testSourceModes() { int numDisabledIndices = randomIntBetween(1, 5); for (int i = 0; i < numSyntheticIndices; i++) { IndexMetadata.Builder indexMetadata = new IndexMetadata.Builder("foo-synthetic-" + i).settings( - indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") + indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") ); builder.put(indexMetadata); } @@ -594,7 +594,7 @@ public void testSourceModes() { IndexMetadata.Builder indexMetadata; if (randomBoolean()) { indexMetadata = new IndexMetadata.Builder("foo-stored-" + i).settings( - indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "stored") + indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "stored") ); } else { indexMetadata = new IndexMetadata.Builder("foo-stored-" + i).settings(indexSettings(IndexVersion.current(), 4, 1)); @@ -603,7 +603,7 @@ public void testSourceModes() { } for (int i = 0; i < numDisabledIndices; i++) { IndexMetadata.Builder indexMetadata = new IndexMetadata.Builder("foo-disabled-" + i).settings( - indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "disabled") + indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "disabled") ); builder.put(indexMetadata); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java b/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java index 2a6c3428d6d45..a5d5d9b210e33 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java @@ -19,14 +19,12 @@ import java.io.IOException; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; - public class LuceneSyntheticSourceChangesSnapshotTests extends SearchBasedChangesSnapshotTests { @Override protected Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) - .put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java index b764bce464d15..d0455c14bd784 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java @@ -24,8 +24,6 @@ import java.io.IOException; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; - public class TranslogOperationAsserterTests extends EngineTestCase { @Override @@ -33,7 +31,7 @@ protected Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) - .put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); } @@ -57,10 +55,10 @@ EngineConfig engineConfig(boolean useSyntheticSource) { EngineConfig config = engine.config(); Settings.Builder settings = Settings.builder().put(config.getIndexSettings().getSettings()); if (useSyntheticSource) { - settings.put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()); + settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()); settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true); } else { - settings.put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.name()); + settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.name()); settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), false); } IndexMetadata imd = IndexMetadata.builder(config.getIndexSettings().getIndexMetadata()).settings(settings).build(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java index 8646e1b66dcb0..1ba5f423d4b03 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.DirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -130,7 +131,7 @@ private MapperService mapperServiceWithCustomSettings( for (var entry : customSettings.entrySet()) { settings.put(entry.getKey(), entry.getValue()); } - settings.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC); + settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC); return createMapperService(settings.build(), mapping(mapping)); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index b7693513a434d..bc560d94b8f52 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -427,7 +427,7 @@ public void testRecoverySourceWitInvalidSettings() { { Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); IllegalArgumentException exc = expectThrows( @@ -470,7 +470,7 @@ public void testRecoverySourceWitInvalidSettings() { public void testRecoverySourceWithSyntheticSource() throws IOException { { Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) .build(); MapperService mapperService = createMapperService(settings, topMapping(b -> {})); DocumentMapper docMapper = mapperService.documentMapper(); @@ -480,7 +480,7 @@ public void testRecoverySourceWithSyntheticSource() throws IOException { } { Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); MapperService mapperService = createMapperService(settings, topMapping(b -> {})); @@ -539,7 +539,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); final MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); @@ -549,7 +549,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -559,7 +559,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -571,7 +571,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); final MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); @@ -581,7 +581,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -591,7 +591,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .build(); var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings)); assertEquals("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode", ex.getMessage()); @@ -613,7 +613,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { """; final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field") .build(); final MapperService mapperService = createMapperService(settings, mappings); @@ -635,7 +635,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { """; final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field") .build(); final MapperService mapperService = createMapperService(settings, mappings); @@ -657,7 +657,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { """; final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field") .build(); var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings)); @@ -668,7 +668,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); final MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); @@ -677,7 +677,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -686,7 +686,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index bce88a2a8d2f7..b33abc2d781e8 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1180,7 +1180,7 @@ private void assertSyntheticSourceWithTranslogSnapshot(SyntheticSourceSupport su var firstExample = support.example(1); int maxDocs = randomIntBetween(20, 50); var settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); var mapperService = createMapperService(getVersion(), settings, () -> true, mapping(b -> { diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java index 0bb4afe51b85a..e8e19bad2a7ef 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java @@ -368,7 +368,7 @@ public void testSyntheticSource() throws Exception { if ("leader".equals(targetCluster)) { logger.info("Running against leader cluster"); Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); createIndex(adminClient(), leaderIndexName, settings, """ "properties": {"kwd": {"type": "keyword"}}}""", null); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 357e1bca38e8f..6bc25215855ee 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrSettings; @@ -335,7 +334,7 @@ public void testDynamicIndexSettingsAreClassified() { replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); - replicatedSettings.add(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING); + replicatedSettings.add(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING); for (Setting setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) { // removed settings have no effect, they are only there for BWC diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java index 81d194ce84131..a2cdfff2e2c80 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java @@ -46,6 +46,7 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; @@ -53,7 +54,6 @@ import org.elasticsearch.index.engine.InternalEngineFactory; import org.elasticsearch.index.fieldvisitor.FieldsVisitor; import org.elasticsearch.index.mapper.SeqNoFieldMapper; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.seqno.RetentionLeaseSyncer; import org.elasticsearch.index.seqno.SeqNoStats; @@ -162,7 +162,7 @@ public void testSourceIncompleteSyntheticSourceNoDoc() throws IOException { .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") .build(); IndexMetadata metadata = IndexMetadata.builder(shardRouting.getIndexName()).settings(settings).primaryTerm(0, primaryTerm).build(); IndexShard shard = newShard(shardRouting, metadata, null, new InternalEngineFactory()); diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java index f4fa2a29d79a0..cdf02273b9df5 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.features.FeatureService; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.monitor.metrics.IndexModeStatsActionType; @@ -29,8 +30,6 @@ import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; import org.elasticsearch.xpack.core.application.LogsDBFeatureSetUsage; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; - public class LogsDBUsageTransportAction extends XPackUsageFeatureTransportAction { private final ClusterService clusterService; private final FeatureService featureService; @@ -71,7 +70,7 @@ protected void masterOperation( for (IndexMetadata indexMetadata : state.metadata()) { if (indexMetadata.getIndexMode() == IndexMode.LOGSDB) { numIndices++; - if (INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC) { + if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC) { numIndicesWithSyntheticSources++; } } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index 3316ac3603e9e..633dd6123c088 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -117,7 +117,7 @@ && matchesLogsPattern(dataStreamName)) { if (settingsBuilder == null) { settingsBuilder = Settings.builder(); } - settingsBuilder.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()); + settingsBuilder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()); } } @@ -206,7 +206,7 @@ MappingHints getMappingHints( var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings); var indexMode = tmpIndexMetadata.getIndexMode(); boolean hasSyntheticSourceUsage = false; - if (SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings()) + if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings()) || indexMode == IndexMode.LOGSDB || indexMode == IndexMode.TIME_SERIES) { // In case when index mode is tsdb or logsdb and only _source.mode mapping attribute is specified, then the default @@ -214,7 +214,7 @@ MappingHints getMappingHints( // then configuring the index.mapping.source.mode setting to stored has no effect. Additionally _source.mode can't be set // to disabled, because that isn't allowed with logsdb/tsdb. In other words setting index.mapping.source.mode setting to // stored when _source.mode mapping attribute is stored is fine as it has no effect, but avoids creating MapperService. - var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings()); + var sourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings()); hasSyntheticSourceUsage = sourceMode == SourceFieldMapper.Mode.SYNTHETIC; if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(indexTemplateAndCreateRequestSettings).isEmpty() == false) { // Custom sort config, no point for further checks on [host.name] field. diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java index f8f307b572f33..b2a533f6b76c9 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java @@ -9,7 +9,7 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.license.AbstractLicensesIntegrationTestCase; import org.elasticsearch.license.GetFeatureUsageRequest; import org.elasticsearch.license.GetFeatureUsageResponse; @@ -104,11 +104,11 @@ public void testSyntheticSourceUsageTracksBothLegacyAndRegularFeature() throws E } private void createIndexWithSyntheticSourceAndAssertExpectedType(String indexName, String expectedType) { - var settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic").build(); + var settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic").build(); createIndex(indexName, settings); var response = admin().indices().getSettings(new GetSettingsRequest().indices(indexName)).actionGet(); assertThat( - response.getIndexToSettings().get(indexName).get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), + response.getIndexToSettings().get(indexName).get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo(expectedType) ); } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 71d52842ecbae..77319a881f1e5 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -640,7 +640,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws ); Metadata metadata = mb.build(); LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); - Settings settings = builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + Settings settings = builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); Settings result = provider.getAdditionalIndexSettings( @@ -666,7 +666,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws List.of() ); assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertThat(newMapperServiceCounter.get(), equalTo(2)); result = provider.getAdditionalIndexSettings( @@ -679,7 +679,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws List.of() ); assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertThat(newMapperServiceCounter.get(), equalTo(3)); result = provider.getAdditionalIndexSettings( @@ -692,7 +692,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws List.of() ); assertThat(result.size(), equalTo(3)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertThat(newMapperServiceCounter.get(), equalTo(4)); @@ -747,7 +747,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch( List.of() ); assertThat(result.size(), equalTo(4)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertEquals(IndexMode.LOGSDB, IndexSettings.MODE.get(result)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index 4921344278845..7fa2f11880f40 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -10,9 +10,9 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.MapperTestUtils; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseService; import org.elasticsearch.license.XPackLicenseState; @@ -58,16 +58,16 @@ public void setup() throws Exception { } public void testGetAdditionalIndexSettingsDefault() { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of()); assertThat(result.size(), equalTo(1)); - assertThat(result.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); + assertThat(result.get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); } public void testGetAdditionalIndexSettingsApm() throws IOException { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-apm.app.test"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of()); @@ -75,7 +75,7 @@ public void testGetAdditionalIndexSettingsApm() throws IOException { } public void testGetAdditionalIndexSettingsProfiling() throws IOException { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); for (String dataStreamName : new String[] { "profiling-metrics", "profiling-events" }) { String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of()); @@ -89,7 +89,7 @@ public void testGetAdditionalIndexSettingsProfiling() throws IOException { } public void testGetAdditionalIndexSettingsTsdb() throws IOException { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, IndexMode.TIME_SERIES, null, null, settings, List.of()); @@ -118,11 +118,11 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception true ); - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, IndexMode.TIME_SERIES, null, null, settings, List.of()); assertThat(result.size(), equalTo(1)); - assertThat(result.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); + assertThat(result.get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); } } From 8114f85bec046400a0f0a8f957f07186e7f0d9c4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 15 Jan 2025 01:35:19 +1100 Subject: [PATCH 44/44] Mute org.elasticsearch.repositories.blobstore.testkit.analyze.MinioRepositoryAnalysisRestIT org.elasticsearch.repositories.blobstore.testkit.analyze.MinioRepositoryAnalysisRestIT #118548 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 9cf2ef065f82f..993fa849dcd31 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -273,6 +273,8 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/120115 - class: org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT issue: https://github.com/elastic/elasticsearch/issues/120117 +- class: org.elasticsearch.repositories.blobstore.testkit.analyze.MinioRepositoryAnalysisRestIT + issue: https://github.com/elastic/elasticsearch/issues/118548 # Examples: #