From ec4f4e72df895713b42cc2aafd5d87017390a4ed Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 9 Jan 2025 17:49:24 +0100 Subject: [PATCH 01/16] Support extracting extent from doc values as int[] for both cartesian and geo shapes --- .../utils/SpatialEnvelopeVisitor.java | 50 +++- .../mapper/LegacyGeoShapeFieldMapper.java | 3 +- .../AbstractShapeGeometryFieldMapper.java | 32 ++- .../compute/gen/AggregatorImplementer.java | 101 +++++--- .../gen/GroupingAggregatorImplementer.java | 135 ++++++---- ...esianShapeDocValuesAggregatorFunction.java | 182 ++++++++++++++ ...peDocValuesAggregatorFunctionSupplier.java | 41 ++++ ...peDocValuesGroupingAggregatorFunction.java | 219 +++++++++++++++++ ...nShapeSourceValuesAggregatorFunction.java} | 24 +- ...ourceValuesAggregatorFunctionSupplier.java | 41 ++++ ...urceValuesGroupingAggregatorFunction.java} | 28 +-- ...ntGeoShapeDocValuesAggregatorFunction.java | 196 +++++++++++++++ ...eDocValuesAggregatorFunctionSupplier.java} | 16 +- ...peDocValuesGroupingAggregatorFunction.java | 231 ++++++++++++++++++ ...oShapeSourceValuesAggregatorFunction.java} | 24 +- ...urceValuesAggregatorFunctionSupplier.java} | 17 +- ...urceValuesGroupingAggregatorFunction.java} | 28 +-- .../spatial/GeoPointEnvelopeVisitor.java | 63 ----- .../spatial/SpatialAggregationUtils.java | 30 +-- ...tentCartesianShapeDocValuesAggregator.java | 42 ++++ ...CartesianShapeSourceValuesAggregator.java} | 2 +- ...tialExtentGeoShapeDocValuesAggregator.java | 44 ++++ ...ExtentGeoShapeSourceValuesAggregator.java} | 3 +- .../spatial/SpatialExtentGroupingState.java | 31 +++ ...entGroupingStateWrappedLongitudeState.java | 49 +++- .../spatial/SpatialExtentState.java | 31 +++ ...atialExtentStateWrappedLongitudeState.java | 65 +++-- .../src/main/resources/spatial.csv-spec | 140 ++++++++++- .../aggregate/SpatialAggregateFunction.java | 2 +- .../function/aggregate/SpatialCentroid.java | 4 +- .../function/aggregate/SpatialExtent.java | 28 ++- .../local/SpatialDocValuesExtraction.java | 5 +- .../local/SpatialShapeBoundsExtraction.java | 39 ++- .../xpack/esql/planner/AggregateMapper.java | 6 +- .../xpack/esql/planner/PlannerUtils.java | 10 +- .../optimizer/PhysicalPlanOptimizerTests.java | 190 ++++++++------ .../TestPhysicalOperationProviders.java | 122 +++++++-- .../GeoShapeWithDocValuesFieldMapper.java | 3 +- 38 files changed, 1851 insertions(+), 426 deletions(-) create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction.java rename x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/{SpatialExtentCartesianShapeAggregatorFunction.java => SpatialExtentCartesianShapeSourceValuesAggregatorFunction.java} (81%) create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier.java rename x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/{SpatialExtentCartesianShapeGroupingAggregatorFunction.java => SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction.java} (82%) create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunction.java rename x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/{SpatialExtentGeoShapeAggregatorFunctionSupplier.java => SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier.java} (55%) create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction.java rename x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/{SpatialExtentGeoShapeAggregatorFunction.java => SpatialExtentGeoShapeSourceValuesAggregatorFunction.java} (83%) rename x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/{SpatialExtentCartesianShapeAggregatorFunctionSupplier.java => SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier.java} (54%) rename x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/{SpatialExtentGeoShapeGroupingAggregatorFunction.java => SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction.java} (83%) delete mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/GeoPointEnvelopeVisitor.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregator.java rename x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/{SpatialExtentCartesianShapeAggregator.java => SpatialExtentCartesianShapeSourceValuesAggregator.java} (94%) create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregator.java rename x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/{SpatialExtentGeoShapeAggregator.java => SpatialExtentGeoShapeSourceValuesAggregator.java} (92%) diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java index 696be2808ed1f..6e647bfd7f9eb 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java @@ -116,6 +116,8 @@ public interface PointVisitor { boolean isValid(); Rectangle getResult(); + + void reset(); } /** @@ -174,6 +176,14 @@ public boolean isValid() { public Rectangle getResult() { return new Rectangle(minX, maxX, maxY, minY); } + + @Override + public void reset() { + minY = Double.POSITIVE_INFINITY; + maxY = Double.NEGATIVE_INFINITY; + minX = Double.POSITIVE_INFINITY; + maxX = Double.NEGATIVE_INFINITY; + } } /** @@ -199,6 +209,30 @@ public GeoPointVisitor(WrapLongitude wrapLongitude) { this.wrapLongitude = wrapLongitude; } + public double getMinNegX() { + return minNegX; + } + + public double getMinPosX() { + return minPosX; + } + + public double getMaxNegX() { + return maxNegX; + } + + public double getMaxPosX() { + return maxPosX; + } + + public double getMaxY() { + return maxY; + } + + public double getMinY() { + return minY; + } + @Override public void visitPoint(double x, double y) { minY = Math.min(minY, y); @@ -234,7 +268,17 @@ public Rectangle getResult() { return getResult(minNegX, minPosX, maxNegX, maxPosX, maxY, minY, wrapLongitude); } - protected static Rectangle getResult( + @Override + public void reset() { + minY = Double.POSITIVE_INFINITY; + maxY = Double.NEGATIVE_INFINITY; + minNegX = Double.POSITIVE_INFINITY; + maxNegX = Double.NEGATIVE_INFINITY; + minPosX = Double.POSITIVE_INFINITY; + maxPosX = Double.NEGATIVE_INFINITY; + } + + public static Rectangle getResult( double minNegX, double minPosX, double maxNegX, @@ -244,7 +288,9 @@ protected static Rectangle getResult( WrapLongitude wrapLongitude ) { assert Double.isFinite(maxY); - if (Double.isInfinite(minPosX)) { + // Due to this data coming through Extent (and aggs that use the same approach), which saves values as integers, + // we must use maxPosX==-Inf for all-neg check, and minNegX==+Inf for all-pos check. + if (Double.isInfinite(maxPosX)) { return new Rectangle(minNegX, maxNegX, maxY, minY); } else if (Double.isInfinite(minNegX)) { return new Rectangle(minPosX, maxPosX, maxY, minY); diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java index 6127b4beb71ff..a7a3dccdadd03 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java @@ -84,6 +84,7 @@ * "field" : "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)) * * @deprecated use the field mapper in the spatial module + * TODO: Remove this class once we no longer need to supported reading 7.x indices that might have this field type */ @Deprecated public class LegacyGeoShapeFieldMapper extends AbstractShapeGeometryFieldMapper> { @@ -534,7 +535,7 @@ public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) { @Override protected boolean isBoundsExtractionSupported() { - // Extracting bounds for geo shapes is not implemented yet. + // Legacy geo-shapes do not support doc-values, so extracting bounds is not possible. return false; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java index 318e877c7ebb9..57de2e5b497fe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java @@ -10,16 +10,12 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.Orientation; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.geometry.utils.WellKnownBinary; import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import java.io.IOException; -import java.nio.ByteOrder; import java.util.Map; import java.util.function.Function; @@ -107,7 +103,7 @@ public BlockLoader.AllReader reader(LeafReaderContext context) throws IOExceptio public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { var binaryDocValues = context.reader().getBinaryDocValues(fieldName); var reader = new GeometryDocValueReader(); - try (var builder = factory.bytesRefs(docs.count())) { + try (var builder = factory.ints(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(binaryDocValues, docs.get(i), reader, builder); } @@ -119,10 +115,10 @@ public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { var binaryDocValues = context.reader().getBinaryDocValues(fieldName); var reader = new GeometryDocValueReader(); - read(binaryDocValues, docId, reader, (BytesRefBuilder) builder); + read(binaryDocValues, docId, reader, (IntBuilder) builder); } - private void read(BinaryDocValues binaryDocValues, int doc, GeometryDocValueReader reader, BytesRefBuilder builder) + private void read(BinaryDocValues binaryDocValues, int doc, GeometryDocValueReader reader, IntBuilder builder) throws IOException { if (binaryDocValues.advanceExact(doc) == false) { builder.appendNull(); @@ -130,16 +126,16 @@ private void read(BinaryDocValues binaryDocValues, int doc, GeometryDocValueRead } reader.reset(binaryDocValues.binaryValue()); var extent = reader.getExtent(); - // This is rather silly: an extent is already encoded as ints, but we convert it to Rectangle to - // preserve its properties as a WKB shape, only to convert it back to ints when we compute the - // aggregation. An obvious optimization would be to avoid this back-and-forth conversion. - var rectangle = new Rectangle( - encoder.decodeX(extent.minX()), - encoder.decodeX(extent.maxX()), - encoder.decodeY(extent.maxY()), - encoder.decodeY(extent.minY()) - ); - builder.appendBytesRef(new BytesRef(WellKnownBinary.toWKB(rectangle, ByteOrder.LITTLE_ENDIAN))); + // We store the 6 values as a single multi-valued field, in the same order as the fields in the Extent class + // This requires that consumers also know the meaning of the values, which they can learn from the Extent class + builder.beginPositionEntry(); + builder.appendInt(extent.top); + builder.appendInt(extent.bottom); + builder.appendInt(extent.negLeft); + builder.appendInt(extent.negRight); + builder.appendInt(extent.posLeft); + builder.appendInt(extent.posRight); + builder.endPositionEntry(); } @Override @@ -151,7 +147,7 @@ public boolean canReuse(int startingDocID) { @Override public BlockLoader.Builder builder(BlockLoader.BlockFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); + return factory.ints(expectedCount); } } } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java index 4aee9ea517d8f..ea64d1660dbe7 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java @@ -29,6 +29,7 @@ import javax.lang.model.element.ExecutableElement; import javax.lang.model.element.Modifier; import javax.lang.model.element.TypeElement; +import javax.lang.model.type.TypeKind; import javax.lang.model.type.TypeMirror; import javax.lang.model.util.Elements; @@ -85,6 +86,7 @@ public class AggregatorImplementer { private final boolean stateTypeHasSeen; private final boolean stateTypeHasFailed; private final boolean valuesIsBytesRef; + private final boolean valuesIsArray; private final List intermediateState; private final List createParameters; @@ -126,7 +128,8 @@ public AggregatorImplementer( elements.getPackageOf(declarationType).toString(), (declarationType.getSimpleName() + "AggregatorFunction").replace("AggregatorAggregator", "Aggregator") ); - this.valuesIsBytesRef = BYTES_REF.equals(TypeName.get(combine.getParameters().get(combine.getParameters().size() - 1).asType())); + this.valuesIsBytesRef = BYTES_REF.equals(valueTypeName()); + this.valuesIsArray = TypeKind.ARRAY.equals(valueTypeKind()); intermediateState = Arrays.stream(interStateAnno).map(IntermediateStateDesc::newIntermediateStateDesc).toList(); } @@ -143,10 +146,11 @@ private TypeName choseStateType() { if (false == initReturn.isPrimitive()) { return initReturn; } + String simpleName = firstUpper(initReturn.toString()); if (warnExceptions.isEmpty()) { - return ClassName.get("org.elasticsearch.compute.aggregation", firstUpper(initReturn.toString()) + "State"); + return ClassName.get("org.elasticsearch.compute.aggregation", simpleName + "State"); } - return ClassName.get("org.elasticsearch.compute.aggregation", firstUpper(initReturn.toString()) + "FallibleState"); + return ClassName.get("org.elasticsearch.compute.aggregation", simpleName + "FallibleState"); } static String valueType(ExecutableElement init, ExecutableElement combine) { @@ -177,7 +181,7 @@ static ClassName valueBlockType(ExecutableElement init, ExecutableElement combin case "double" -> DOUBLE_BLOCK; case "float" -> FLOAT_BLOCK; case "long" -> LONG_BLOCK; - case "int" -> INT_BLOCK; + case "int", "int[]" -> INT_BLOCK; case "org.apache.lucene.util.BytesRef" -> BYTES_REF_BLOCK; default -> throw new IllegalArgumentException("unknown block type for " + valueType(init, combine)); }; @@ -189,7 +193,7 @@ static ClassName valueVectorType(ExecutableElement init, ExecutableElement combi case "double" -> DOUBLE_VECTOR; case "float" -> FLOAT_VECTOR; case "long" -> LONG_VECTOR; - case "int" -> INT_VECTOR; + case "int", "int[]" -> INT_VECTOR; case "org.apache.lucene.util.BytesRef" -> BYTES_REF_VECTOR; default -> throw new IllegalArgumentException("unknown vector type for " + valueType(init, combine)); }; @@ -390,6 +394,10 @@ private MethodSpec addRawVector(boolean masked) { if (masked) { builder.addParameter(BOOLEAN_VECTOR, "mask"); } + if (valuesIsArray) { + builder.addComment("This type does not support vectors because all values are multi-valued"); + return builder.build(); + } if (stateTypeHasSeen) { builder.addStatement("state.seen(true)"); @@ -437,9 +445,18 @@ private MethodSpec addRawBlock(boolean masked) { } builder.addStatement("int start = block.getFirstValueIndex(p)"); builder.addStatement("int end = start + block.getValueCount(p)"); - builder.beginControlFlow("for (int i = start; i < end; i++)"); - combineRawInput(builder, "block"); - builder.endControlFlow(); + if (valuesIsArray) { + String arrayType = valueTypeString(); + builder.addStatement("$L[] valuesArray = new $L[end - start]", arrayType, arrayType); + builder.beginControlFlow("for (int i = start; i < end; i++)"); + builder.addStatement("valuesArray[i-start] = $L.get$L(i)", "block", firstUpper(arrayType)); + builder.endControlFlow(); + combineRawInputForArray(builder, "valuesArray"); + } else { + builder.beginControlFlow("for (int i = start; i < end; i++)"); + combineRawInput(builder, "block"); + builder.endControlFlow(); + } } builder.endControlFlow(); if (combineValueCount != null) { @@ -450,9 +467,7 @@ private MethodSpec addRawBlock(boolean masked) { private void combineRawInput(MethodSpec.Builder builder, String blockVariable) { TypeName returnType = TypeName.get(combine.getReturnType()); - if (warnExceptions.isEmpty() == false) { - builder.beginControlFlow("try"); - } + startWarningsBlock(builder); if (valuesIsBytesRef) { combineRawInputForBytesRef(builder, blockVariable); } else if (returnType.isPrimitive()) { @@ -462,14 +477,7 @@ private void combineRawInput(MethodSpec.Builder builder, String blockVariable) { } else { throw new IllegalArgumentException("combine must return void or a primitive"); } - if (warnExceptions.isEmpty() == false) { - String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; - builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); - builder.addStatement("warnings.registerException(e)"); - builder.addStatement("state.failed(true)"); - builder.addStatement("return"); - builder.endControlFlow(); - } + endWarningsBlock(builder); } private void combineRawInputForPrimitive(TypeName returnType, MethodSpec.Builder builder, String blockVariable) { @@ -483,6 +491,12 @@ private void combineRawInputForPrimitive(TypeName returnType, MethodSpec.Builder ); } + private void combineRawInputForArray(MethodSpec.Builder builder, String arrayVariable) { + startWarningsBlock(builder); + builder.addStatement("$T.combine(state, $L)", declarationType, arrayVariable); + endWarningsBlock(builder); + } + private void combineRawInputForVoid(MethodSpec.Builder builder, String blockVariable) { builder.addStatement( "$T.combine(state, $L.get$L(i))", @@ -497,6 +511,23 @@ private void combineRawInputForBytesRef(MethodSpec.Builder builder, String block builder.addStatement("$T.combine(state, $L.getBytesRef(i, scratch))", declarationType, blockVariable); } + private void startWarningsBlock(MethodSpec.Builder builder) { + if (warnExceptions.isEmpty() == false) { + builder.beginControlFlow("try"); + } + } + + private void endWarningsBlock(MethodSpec.Builder builder) { + if (warnExceptions.isEmpty() == false) { + String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; + builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); + builder.addStatement("warnings.registerException(e)"); + builder.addStatement("state.failed(true)"); + builder.addStatement("return"); + builder.endControlFlow(); + } + } + private MethodSpec addIntermediateInput() { MethodSpec.Builder builder = MethodSpec.methodBuilder("addIntermediateInput"); builder.addAnnotation(Override.class).addModifiers(Modifier.PUBLIC).addParameter(PAGE, "page"); @@ -529,20 +560,12 @@ private MethodSpec addIntermediateInput() { builder.nextControlFlow("else if (seen.getBoolean(0))"); } - if (warnExceptions.isEmpty() == false) { - builder.beginControlFlow("try"); - } + startWarningsBlock(builder); var state = intermediateState.get(0); var s = "state.$L($T.combine(state.$L(), " + state.name() + "." + vectorAccessorName(state.elementType()) + "(0)))"; builder.addStatement(s, primitiveStateMethod(), declarationType, primitiveStateMethod()); builder.addStatement("state.seen(true)"); - if (warnExceptions.isEmpty() == false) { - String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; - builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); - builder.addStatement("warnings.registerException(e)"); - builder.addStatement("state.failed(true)"); - builder.endControlFlow(); - } + endWarningsBlock(builder); builder.endControlFlow(); } else { throw new IllegalArgumentException("Don't know how to combine intermediate input. Define combineIntermediate"); @@ -693,4 +716,24 @@ public void assignToVariable(MethodSpec.Builder builder, int offset) { } } } + + private TypeMirror valueTypeMirror() { + return combine.getParameters().get(combine.getParameters().size() - 1).asType(); + } + + private TypeName valueTypeName() { + return TypeName.get(valueTypeMirror()); + } + + private TypeKind valueTypeKind() { + return valueTypeMirror().getKind(); + } + + private String valueTypeString() { + String valueTypeString = TypeName.get(valueTypeMirror()).toString(); + if (valuesIsArray) { + valueTypeString = valueTypeString.substring(0, valueTypeString.length() - 2); + } + return valueTypeString; + } } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java index 9e8112e10f878..eaef32c8ce395 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java @@ -20,7 +20,6 @@ import java.util.Arrays; import java.util.List; -import java.util.Locale; import java.util.function.Consumer; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -28,10 +27,12 @@ import javax.lang.model.element.ExecutableElement; import javax.lang.model.element.Modifier; import javax.lang.model.element.TypeElement; +import javax.lang.model.type.TypeKind; import javax.lang.model.type.TypeMirror; import javax.lang.model.util.Elements; import static java.util.stream.Collectors.joining; +import static org.elasticsearch.compute.gen.AggregatorImplementer.firstUpper; import static org.elasticsearch.compute.gen.AggregatorImplementer.valueBlockType; import static org.elasticsearch.compute.gen.AggregatorImplementer.valueVectorType; import static org.elasticsearch.compute.gen.Methods.findMethod; @@ -74,6 +75,7 @@ public class GroupingAggregatorImplementer { private final ExecutableElement combineIntermediate; private final TypeName stateType; private final boolean valuesIsBytesRef; + private final boolean valuesIsArray; private final List createParameters; private final ClassName implementation; private final List intermediateState; @@ -102,7 +104,8 @@ public GroupingAggregatorImplementer( this.combineStates = findMethod(declarationType, "combineStates"); this.combineIntermediate = findMethod(declarationType, "combineIntermediate"); this.evaluateFinal = findMethod(declarationType, "evaluateFinal"); - this.valuesIsBytesRef = BYTES_REF.equals(TypeName.get(combine.getParameters().get(combine.getParameters().size() - 1).asType())); + this.valuesIsBytesRef = BYTES_REF.equals(valueTypeName()); + this.valuesIsArray = TypeKind.ARRAY.equals(valueTypeKind()); this.createParameters = init.getParameters() .stream() .map(Parameter::from) @@ -133,12 +136,11 @@ private TypeName choseStateType() { if (false == initReturn.isPrimitive()) { return initReturn; } - String head = initReturn.toString().substring(0, 1).toUpperCase(Locale.ROOT); - String tail = initReturn.toString().substring(1); + String simpleName = firstUpper(initReturn.toString()); if (warnExceptions.isEmpty()) { - return ClassName.get("org.elasticsearch.compute.aggregation", head + tail + "ArrayState"); + return ClassName.get("org.elasticsearch.compute.aggregation", simpleName + "ArrayState"); } - return ClassName.get("org.elasticsearch.compute.aggregation", head + tail + "FallibleArrayState"); + return ClassName.get("org.elasticsearch.compute.aggregation", simpleName + "FallibleArrayState"); } public JavaFile sourceFile() { @@ -364,6 +366,10 @@ private MethodSpec addRawInputLoop(TypeName groupsType, TypeName valuesType) { // Add bytes_ref scratch var that will be used for bytes_ref blocks/vectors builder.addStatement("$T scratch = new $T()", BYTES_REF, BYTES_REF); } + if (valuesIsArray && valuesIsBlock == false) { + builder.addComment("This type does not support vectors because all values are multi-valued"); + return builder.build(); + } builder.beginControlFlow("for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++)"); { @@ -391,9 +397,18 @@ private MethodSpec addRawInputLoop(TypeName groupsType, TypeName valuesType) { builder.endControlFlow(); builder.addStatement("int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset)"); builder.addStatement("int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset)"); - builder.beginControlFlow("for (int v = valuesStart; v < valuesEnd; v++)"); - combineRawInput(builder, "values", "v"); - builder.endControlFlow(); + if (valuesIsArray) { + String arrayType = valueTypeString(); + builder.addStatement("$L[] valuesArray = new $L[valuesEnd - valuesStart]", arrayType, arrayType); + builder.beginControlFlow("for (int v = valuesStart; v < valuesEnd; v++)"); + builder.addStatement("valuesArray[v-valuesStart] = $L.get$L(v)", "values", firstUpper(arrayType)); + builder.endControlFlow(); + combineRawInputForArray(builder, "valuesArray"); + } else { + builder.beginControlFlow("for (int v = valuesStart; v < valuesEnd; v++)"); + combineRawInput(builder, "values", "v"); + builder.endControlFlow(); + } } else { combineRawInput(builder, "values", "groupPosition + positionOffset"); } @@ -407,70 +422,54 @@ private MethodSpec addRawInputLoop(TypeName groupsType, TypeName valuesType) { } private void combineRawInput(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { - TypeName valueType = TypeName.get(combine.getParameters().get(combine.getParameters().size() - 1).asType()); - String secondParameterGetter = "get" - + valueType.toString().substring(0, 1).toUpperCase(Locale.ROOT) - + valueType.toString().substring(1); + TypeName valueType = valueTypeName(); TypeName returnType = TypeName.get(combine.getReturnType()); - if (warnExceptions.isEmpty() == false) { - builder.beginControlFlow("try"); - } + startWarningsBlock(builder); if (valuesIsBytesRef) { combineRawInputForBytesRef(builder, blockVariable, offsetVariable); } else if (includeTimestampVector) { combineRawInputWithTimestamp(builder, offsetVariable); } else if (valueType.isPrimitive() == false) { - throw new IllegalArgumentException("second parameter to combine must be a primitive"); + throw new IllegalArgumentException("second parameter to combine must be a primitive, array or BytesRef: " + valueType); } else if (returnType.isPrimitive()) { - combineRawInputForPrimitive(builder, secondParameterGetter, blockVariable, offsetVariable); + combineRawInputForPrimitive(builder, blockVariable, offsetVariable); } else if (returnType == TypeName.VOID) { - combineRawInputForVoid(builder, secondParameterGetter, blockVariable, offsetVariable); + combineRawInputForVoid(builder, blockVariable, offsetVariable); } else { throw new IllegalArgumentException("combine must return void or a primitive"); } - if (warnExceptions.isEmpty() == false) { - String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; - builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); - builder.addStatement("warnings.registerException(e)"); - builder.addStatement("state.setFailed(groupId)"); - builder.endControlFlow(); - } + endWarningsBlock(builder); } - private void combineRawInputForPrimitive( - MethodSpec.Builder builder, - String secondParameterGetter, - String blockVariable, - String offsetVariable - ) { + private void combineRawInputForPrimitive(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { builder.addStatement( - "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.$L($L)))", + "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.get$L($L)))", declarationType, blockVariable, - secondParameterGetter, + firstUpper(valueTypeName().toString()), offsetVariable ); } - private void combineRawInputForVoid( - MethodSpec.Builder builder, - String secondParameterGetter, - String blockVariable, - String offsetVariable - ) { + private void combineRawInputForArray(MethodSpec.Builder builder, String arrayVariable) { + startWarningsBlock(builder); + builder.addStatement("$T.combine(state, groupId, $L)", declarationType, arrayVariable); + endWarningsBlock(builder); + } + + private void combineRawInputForVoid(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { builder.addStatement( - "$T.combine(state, groupId, $L.$L($L))", + "$T.combine(state, groupId, $L.get$L($L))", declarationType, blockVariable, - secondParameterGetter, + firstUpper(valueTypeName().toString()), offsetVariable ); } private void combineRawInputWithTimestamp(MethodSpec.Builder builder, String offsetVariable) { - TypeName valueType = TypeName.get(combine.getParameters().get(combine.getParameters().size() - 1).asType()); - String blockType = valueType.toString().substring(0, 1).toUpperCase(Locale.ROOT) + valueType.toString().substring(1); + String blockType = firstUpper(valueTypeName().toString()); if (offsetVariable.contains(" + ")) { builder.addStatement("var valuePosition = $L", offsetVariable); offsetVariable = "valuePosition"; @@ -489,6 +488,22 @@ private void combineRawInputForBytesRef(MethodSpec.Builder builder, String block builder.addStatement("$T.combine(state, groupId, $L.getBytesRef($L, scratch))", declarationType, blockVariable, offsetVariable); } + private void startWarningsBlock(MethodSpec.Builder builder) { + if (warnExceptions.isEmpty() == false) { + builder.beginControlFlow("try"); + } + } + + private void endWarningsBlock(MethodSpec.Builder builder) { + if (warnExceptions.isEmpty() == false) { + String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; + builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); + builder.addStatement("warnings.registerException(e)"); + builder.addStatement("state.setFailed(groupId)"); + builder.endControlFlow(); + } + } + private MethodSpec selectedMayContainUnseenGroups() { MethodSpec.Builder builder = MethodSpec.methodBuilder("selectedMayContainUnseenGroups"); builder.addAnnotation(Override.class).addModifiers(Modifier.PUBLIC); @@ -544,9 +559,7 @@ private MethodSpec addIntermediateInput() { builder.nextControlFlow("else if (seen.getBoolean(groupPosition + positionOffset))"); } - if (warnExceptions.isEmpty() == false) { - builder.beginControlFlow("try"); - } + startWarningsBlock(builder); var name = intermediateState.get(0).name(); var vectorAccessor = vectorAccessorName(intermediateState.get(0).elementType()); builder.addStatement( @@ -555,13 +568,7 @@ private MethodSpec addIntermediateInput() { name, vectorAccessor ); - if (warnExceptions.isEmpty() == false) { - String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; - builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); - builder.addStatement("warnings.registerException(e)"); - builder.addStatement("state.setFailed(groupId)"); - builder.endControlFlow(); - } + endWarningsBlock(builder); builder.endControlFlow(); } else { builder.addStatement("$T.combineIntermediate(state, groupId, " + intermediateStateRowAccess() + ")", declarationType); @@ -657,4 +664,24 @@ private MethodSpec close() { private boolean hasPrimitiveState() { return PRIMITIVE_STATE_PATTERN.matcher(stateType.toString()).matches(); } + + private TypeMirror valueTypeMirror() { + return combine.getParameters().get(combine.getParameters().size() - 1).asType(); + } + + private TypeName valueTypeName() { + return TypeName.get(valueTypeMirror()); + } + + private TypeKind valueTypeKind() { + return valueTypeMirror().getKind(); + } + + private String valueTypeString() { + String valueTypeString = TypeName.get(valueTypeMirror()).toString(); + if (valuesIsArray) { + valueTypeString = valueTypeString.substring(0, valueTypeString.length() - 2); + } + return valueTypeString; + } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunction.java new file mode 100644 index 0000000000000..3471aafc3a53b --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunction.java @@ -0,0 +1,182 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation.spatial; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.aggregation.AggregatorFunction; +import org.elasticsearch.compute.aggregation.IntermediateStateDesc; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link SpatialExtentCartesianShapeDocValuesAggregator}. + * This class is generated. Do not edit it. + */ +public final class SpatialExtentCartesianShapeDocValuesAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("minX", ElementType.INT), + new IntermediateStateDesc("maxX", ElementType.INT), + new IntermediateStateDesc("maxY", ElementType.INT), + new IntermediateStateDesc("minY", ElementType.INT) ); + + private final DriverContext driverContext; + + private final SpatialExtentState state; + + private final List channels; + + public SpatialExtentCartesianShapeDocValuesAggregatorFunction(DriverContext driverContext, + List channels, SpatialExtentState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static SpatialExtentCartesianShapeDocValuesAggregatorFunction create( + DriverContext driverContext, List channels) { + return new SpatialExtentCartesianShapeDocValuesAggregatorFunction(driverContext, channels, SpatialExtentCartesianShapeDocValuesAggregator.initSingle()); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page, BooleanVector mask) { + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { + // No masking + IntBlock block = page.getBlock(channels.get(0)); + IntVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + return; + } + // Some positions masked away, others kept + IntBlock block = page.getBlock(channels.get(0)); + IntVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector, mask); + } else { + addRawBlock(block, mask); + } + } + + private void addRawVector(IntVector vector) { + // This type does not support vectors because all values are multi-valued + } + + private void addRawVector(IntVector vector, BooleanVector mask) { + // This type does not support vectors because all values are multi-valued + } + + private void addRawBlock(IntBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + int[] valuesArray = new int[end - start]; + for (int i = start; i < end; i++) { + valuesArray[i-start] = block.getInt(i); + } + SpatialExtentCartesianShapeDocValuesAggregator.combine(state, valuesArray); + } + } + + private void addRawBlock(IntBlock block, BooleanVector mask) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (mask.getBoolean(p) == false) { + continue; + } + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + int[] valuesArray = new int[end - start]; + for (int i = start; i < end; i++) { + valuesArray[i-start] = block.getInt(i); + } + SpatialExtentCartesianShapeDocValuesAggregator.combine(state, valuesArray); + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block minXUncast = page.getBlock(channels.get(0)); + if (minXUncast.areAllValuesNull()) { + return; + } + IntVector minX = ((IntBlock) minXUncast).asVector(); + assert minX.getPositionCount() == 1; + Block maxXUncast = page.getBlock(channels.get(1)); + if (maxXUncast.areAllValuesNull()) { + return; + } + IntVector maxX = ((IntBlock) maxXUncast).asVector(); + assert maxX.getPositionCount() == 1; + Block maxYUncast = page.getBlock(channels.get(2)); + if (maxYUncast.areAllValuesNull()) { + return; + } + IntVector maxY = ((IntBlock) maxYUncast).asVector(); + assert maxY.getPositionCount() == 1; + Block minYUncast = page.getBlock(channels.get(3)); + if (minYUncast.areAllValuesNull()) { + return; + } + IntVector minY = ((IntBlock) minYUncast).asVector(); + assert minY.getPositionCount() == 1; + SpatialExtentCartesianShapeDocValuesAggregator.combineIntermediate(state, minX.getInt(0), maxX.getInt(0), maxY.getInt(0), minY.getInt(0)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = SpatialExtentCartesianShapeDocValuesAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..b53d779912fc9 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier.java @@ -0,0 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation.spatial; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link SpatialExtentCartesianShapeDocValuesAggregator}. + * This class is generated. Do not edit it. + */ +public final class SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public SpatialExtentCartesianShapeDocValuesAggregatorFunction aggregator( + DriverContext driverContext) { + return SpatialExtentCartesianShapeDocValuesAggregatorFunction.create(driverContext, channels); + } + + @Override + public SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction groupingAggregator( + DriverContext driverContext) { + return SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "spatial_extent_cartesian_shape_doc of valuess"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..aa3c1a7ba56ae --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction.java @@ -0,0 +1,219 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation.spatial; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction; +import org.elasticsearch.compute.aggregation.IntermediateStateDesc; +import org.elasticsearch.compute.aggregation.SeenGroupIds; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link SpatialExtentCartesianShapeDocValuesAggregator}. + * This class is generated. Do not edit it. + */ +public final class SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("minX", ElementType.INT), + new IntermediateStateDesc("maxX", ElementType.INT), + new IntermediateStateDesc("maxY", ElementType.INT), + new IntermediateStateDesc("minY", ElementType.INT) ); + + private final SpatialExtentGroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + public SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction(List channels, + SpatialExtentGroupingState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction create( + List channels, DriverContext driverContext) { + return new SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction(channels, SpatialExtentCartesianShapeDocValuesAggregator.initGrouping(), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + IntBlock valuesBlock = page.getBlock(channels.get(0)); + IntVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void close() { + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void close() { + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, IntBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + int[] valuesArray = new int[valuesEnd - valuesStart]; + for (int v = valuesStart; v < valuesEnd; v++) { + valuesArray[v-valuesStart] = values.getInt(v); + } + SpatialExtentCartesianShapeDocValuesAggregator.combine(state, groupId, valuesArray); + } + } + + private void addRawInput(int positionOffset, IntVector groups, IntVector values) { + // This type does not support vectors because all values are multi-valued + } + + private void addRawInput(int positionOffset, IntBlock groups, IntBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = groups.getInt(g); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + int[] valuesArray = new int[valuesEnd - valuesStart]; + for (int v = valuesStart; v < valuesEnd; v++) { + valuesArray[v-valuesStart] = values.getInt(v); + } + SpatialExtentCartesianShapeDocValuesAggregator.combine(state, groupId, valuesArray); + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, IntVector values) { + // This type does not support vectors because all values are multi-valued + } + + @Override + public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { + state.enableGroupIdTracking(seenGroupIds); + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block minXUncast = page.getBlock(channels.get(0)); + if (minXUncast.areAllValuesNull()) { + return; + } + IntVector minX = ((IntBlock) minXUncast).asVector(); + Block maxXUncast = page.getBlock(channels.get(1)); + if (maxXUncast.areAllValuesNull()) { + return; + } + IntVector maxX = ((IntBlock) maxXUncast).asVector(); + Block maxYUncast = page.getBlock(channels.get(2)); + if (maxYUncast.areAllValuesNull()) { + return; + } + IntVector maxY = ((IntBlock) maxYUncast).asVector(); + Block minYUncast = page.getBlock(channels.get(3)); + if (minYUncast.areAllValuesNull()) { + return; + } + IntVector minY = ((IntBlock) minYUncast).asVector(); + assert minX.getPositionCount() == maxX.getPositionCount() && minX.getPositionCount() == maxY.getPositionCount() && minX.getPositionCount() == minY.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + SpatialExtentCartesianShapeDocValuesAggregator.combineIntermediate(state, groupId, minX.getInt(groupPosition + positionOffset), maxX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + SpatialExtentGroupingState inState = ((SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + SpatialExtentCartesianShapeDocValuesAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = SpatialExtentCartesianShapeDocValuesAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregatorFunction.java similarity index 81% rename from x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregatorFunction.java rename to x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregatorFunction.java index 19aa4f7ca78a2..014a2d454f576 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregatorFunction.java @@ -23,10 +23,10 @@ import org.elasticsearch.compute.operator.DriverContext; /** - * {@link AggregatorFunction} implementation for {@link SpatialExtentCartesianShapeAggregator}. + * {@link AggregatorFunction} implementation for {@link SpatialExtentCartesianShapeSourceValuesAggregator}. * This class is generated. Do not edit it. */ -public final class SpatialExtentCartesianShapeAggregatorFunction implements AggregatorFunction { +public final class SpatialExtentCartesianShapeSourceValuesAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( new IntermediateStateDesc("minX", ElementType.INT), new IntermediateStateDesc("maxX", ElementType.INT), @@ -39,16 +39,16 @@ public final class SpatialExtentCartesianShapeAggregatorFunction implements Aggr private final List channels; - public SpatialExtentCartesianShapeAggregatorFunction(DriverContext driverContext, + public SpatialExtentCartesianShapeSourceValuesAggregatorFunction(DriverContext driverContext, List channels, SpatialExtentState state) { this.driverContext = driverContext; this.channels = channels; this.state = state; } - public static SpatialExtentCartesianShapeAggregatorFunction create(DriverContext driverContext, - List channels) { - return new SpatialExtentCartesianShapeAggregatorFunction(driverContext, channels, SpatialExtentCartesianShapeAggregator.initSingle()); + public static SpatialExtentCartesianShapeSourceValuesAggregatorFunction create( + DriverContext driverContext, List channels) { + return new SpatialExtentCartesianShapeSourceValuesAggregatorFunction(driverContext, channels, SpatialExtentCartesianShapeSourceValuesAggregator.initSingle()); } public static List intermediateStateDesc() { @@ -90,7 +90,7 @@ public void addRawInput(Page page, BooleanVector mask) { private void addRawVector(BytesRefVector vector) { BytesRef scratch = new BytesRef(); for (int i = 0; i < vector.getPositionCount(); i++) { - SpatialExtentCartesianShapeAggregator.combine(state, vector.getBytesRef(i, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, vector.getBytesRef(i, scratch)); } } @@ -100,7 +100,7 @@ private void addRawVector(BytesRefVector vector, BooleanVector mask) { if (mask.getBoolean(i) == false) { continue; } - SpatialExtentCartesianShapeAggregator.combine(state, vector.getBytesRef(i, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, vector.getBytesRef(i, scratch)); } } @@ -113,7 +113,7 @@ private void addRawBlock(BytesRefBlock block) { int start = block.getFirstValueIndex(p); int end = start + block.getValueCount(p); for (int i = start; i < end; i++) { - SpatialExtentCartesianShapeAggregator.combine(state, block.getBytesRef(i, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, block.getBytesRef(i, scratch)); } } } @@ -130,7 +130,7 @@ private void addRawBlock(BytesRefBlock block, BooleanVector mask) { int start = block.getFirstValueIndex(p); int end = start + block.getValueCount(p); for (int i = start; i < end; i++) { - SpatialExtentCartesianShapeAggregator.combine(state, block.getBytesRef(i, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, block.getBytesRef(i, scratch)); } } } @@ -163,7 +163,7 @@ public void addIntermediateInput(Page page) { } IntVector minY = ((IntBlock) minYUncast).asVector(); assert minY.getPositionCount() == 1; - SpatialExtentCartesianShapeAggregator.combineIntermediate(state, minX.getInt(0), maxX.getInt(0), maxY.getInt(0), minY.getInt(0)); + SpatialExtentCartesianShapeSourceValuesAggregator.combineIntermediate(state, minX.getInt(0), maxX.getInt(0), maxY.getInt(0), minY.getInt(0)); } @Override @@ -173,7 +173,7 @@ public void evaluateIntermediate(Block[] blocks, int offset, DriverContext drive @Override public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { - blocks[offset] = SpatialExtentCartesianShapeAggregator.evaluateFinal(state, driverContext); + blocks[offset] = SpatialExtentCartesianShapeSourceValuesAggregator.evaluateFinal(state, driverContext); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..c8b1372d44b68 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier.java @@ -0,0 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation.spatial; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link SpatialExtentCartesianShapeSourceValuesAggregator}. + * This class is generated. Do not edit it. + */ +public final class SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public SpatialExtentCartesianShapeSourceValuesAggregatorFunction aggregator( + DriverContext driverContext) { + return SpatialExtentCartesianShapeSourceValuesAggregatorFunction.create(driverContext, channels); + } + + @Override + public SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction groupingAggregator( + DriverContext driverContext) { + return SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "spatial_extent_cartesian_shape_source of valuess"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction.java similarity index 82% rename from x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeGroupingAggregatorFunction.java rename to x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction.java index c55c3d9c66946..d932038a26ec7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction.java @@ -23,10 +23,10 @@ import org.elasticsearch.compute.operator.DriverContext; /** - * {@link GroupingAggregatorFunction} implementation for {@link SpatialExtentCartesianShapeAggregator}. + * {@link GroupingAggregatorFunction} implementation for {@link SpatialExtentCartesianShapeSourceValuesAggregator}. * This class is generated. Do not edit it. */ -public final class SpatialExtentCartesianShapeGroupingAggregatorFunction implements GroupingAggregatorFunction { +public final class SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( new IntermediateStateDesc("minX", ElementType.INT), new IntermediateStateDesc("maxX", ElementType.INT), @@ -39,16 +39,16 @@ public final class SpatialExtentCartesianShapeGroupingAggregatorFunction impleme private final DriverContext driverContext; - public SpatialExtentCartesianShapeGroupingAggregatorFunction(List channels, + public SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction(List channels, SpatialExtentGroupingState state, DriverContext driverContext) { this.channels = channels; this.state = state; this.driverContext = driverContext; } - public static SpatialExtentCartesianShapeGroupingAggregatorFunction create(List channels, - DriverContext driverContext) { - return new SpatialExtentCartesianShapeGroupingAggregatorFunction(channels, SpatialExtentCartesianShapeAggregator.initGrouping(), driverContext); + public static SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction create( + List channels, DriverContext driverContext) { + return new SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction(channels, SpatialExtentCartesianShapeSourceValuesAggregator.initGrouping(), driverContext); } public static List intermediateStateDesc() { @@ -112,7 +112,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock val int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); for (int v = valuesStart; v < valuesEnd; v++) { - SpatialExtentCartesianShapeAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); } } } @@ -121,7 +121,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefVector va BytesRef scratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentCartesianShapeAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); } } @@ -141,7 +141,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock valu int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); for (int v = valuesStart; v < valuesEnd; v++) { - SpatialExtentCartesianShapeAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); } } } @@ -157,7 +157,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector val int groupEnd = groupStart + groups.getValueCount(groupPosition); for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); - SpatialExtentCartesianShapeAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); } } } @@ -194,7 +194,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page assert minX.getPositionCount() == maxX.getPositionCount() && minX.getPositionCount() == maxY.getPositionCount() && minX.getPositionCount() == minY.getPositionCount(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentCartesianShapeAggregator.combineIntermediate(state, groupId, minX.getInt(groupPosition + positionOffset), maxX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); + SpatialExtentCartesianShapeSourceValuesAggregator.combineIntermediate(state, groupId, minX.getInt(groupPosition + positionOffset), maxX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); } } @@ -203,9 +203,9 @@ public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction inpu if (input.getClass() != getClass()) { throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); } - SpatialExtentGroupingState inState = ((SpatialExtentCartesianShapeGroupingAggregatorFunction) input).state; + SpatialExtentGroupingState inState = ((SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction) input).state; state.enableGroupIdTracking(new SeenGroupIds.Empty()); - SpatialExtentCartesianShapeAggregator.combineStates(state, groupId, inState, position); + SpatialExtentCartesianShapeSourceValuesAggregator.combineStates(state, groupId, inState, position); } @Override @@ -216,7 +216,7 @@ public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) @Override public void evaluateFinal(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { - blocks[offset] = SpatialExtentCartesianShapeAggregator.evaluateFinal(state, selected, driverContext); + blocks[offset] = SpatialExtentCartesianShapeSourceValuesAggregator.evaluateFinal(state, selected, driverContext); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunction.java new file mode 100644 index 0000000000000..11d301549bf18 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunction.java @@ -0,0 +1,196 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation.spatial; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.aggregation.AggregatorFunction; +import org.elasticsearch.compute.aggregation.IntermediateStateDesc; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link SpatialExtentGeoShapeDocValuesAggregator}. + * This class is generated. Do not edit it. + */ +public final class SpatialExtentGeoShapeDocValuesAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("minNegX", ElementType.INT), + new IntermediateStateDesc("minPosX", ElementType.INT), + new IntermediateStateDesc("maxNegX", ElementType.INT), + new IntermediateStateDesc("maxPosX", ElementType.INT), + new IntermediateStateDesc("maxY", ElementType.INT), + new IntermediateStateDesc("minY", ElementType.INT) ); + + private final DriverContext driverContext; + + private final SpatialExtentStateWrappedLongitudeState state; + + private final List channels; + + public SpatialExtentGeoShapeDocValuesAggregatorFunction(DriverContext driverContext, + List channels, SpatialExtentStateWrappedLongitudeState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static SpatialExtentGeoShapeDocValuesAggregatorFunction create(DriverContext driverContext, + List channels) { + return new SpatialExtentGeoShapeDocValuesAggregatorFunction(driverContext, channels, SpatialExtentGeoShapeDocValuesAggregator.initSingle()); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page, BooleanVector mask) { + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { + // No masking + IntBlock block = page.getBlock(channels.get(0)); + IntVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + return; + } + // Some positions masked away, others kept + IntBlock block = page.getBlock(channels.get(0)); + IntVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector, mask); + } else { + addRawBlock(block, mask); + } + } + + private void addRawVector(IntVector vector) { + // This type does not support vectors because all values are multi-valued + } + + private void addRawVector(IntVector vector, BooleanVector mask) { + // This type does not support vectors because all values are multi-valued + } + + private void addRawBlock(IntBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + int[] valuesArray = new int[end - start]; + for (int i = start; i < end; i++) { + valuesArray[i-start] = block.getInt(i); + } + SpatialExtentGeoShapeDocValuesAggregator.combine(state, valuesArray); + } + } + + private void addRawBlock(IntBlock block, BooleanVector mask) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (mask.getBoolean(p) == false) { + continue; + } + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + int[] valuesArray = new int[end - start]; + for (int i = start; i < end; i++) { + valuesArray[i-start] = block.getInt(i); + } + SpatialExtentGeoShapeDocValuesAggregator.combine(state, valuesArray); + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block minNegXUncast = page.getBlock(channels.get(0)); + if (minNegXUncast.areAllValuesNull()) { + return; + } + IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); + assert minNegX.getPositionCount() == 1; + Block minPosXUncast = page.getBlock(channels.get(1)); + if (minPosXUncast.areAllValuesNull()) { + return; + } + IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); + assert minPosX.getPositionCount() == 1; + Block maxNegXUncast = page.getBlock(channels.get(2)); + if (maxNegXUncast.areAllValuesNull()) { + return; + } + IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); + assert maxNegX.getPositionCount() == 1; + Block maxPosXUncast = page.getBlock(channels.get(3)); + if (maxPosXUncast.areAllValuesNull()) { + return; + } + IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); + assert maxPosX.getPositionCount() == 1; + Block maxYUncast = page.getBlock(channels.get(4)); + if (maxYUncast.areAllValuesNull()) { + return; + } + IntVector maxY = ((IntBlock) maxYUncast).asVector(); + assert maxY.getPositionCount() == 1; + Block minYUncast = page.getBlock(channels.get(5)); + if (minYUncast.areAllValuesNull()) { + return; + } + IntVector minY = ((IntBlock) minYUncast).asVector(); + assert minY.getPositionCount() == 1; + SpatialExtentGeoShapeDocValuesAggregator.combineIntermediate(state, minNegX.getInt(0), minPosX.getInt(0), maxNegX.getInt(0), maxPosX.getInt(0), maxY.getInt(0), minY.getInt(0)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = SpatialExtentGeoShapeDocValuesAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier.java similarity index 55% rename from x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregatorFunctionSupplier.java rename to x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier.java index 09f210c7085f8..d104c74bc5072 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregatorFunctionSupplier.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier.java @@ -12,29 +12,29 @@ import org.elasticsearch.compute.operator.DriverContext; /** - * {@link AggregatorFunctionSupplier} implementation for {@link SpatialExtentGeoShapeAggregator}. + * {@link AggregatorFunctionSupplier} implementation for {@link SpatialExtentGeoShapeDocValuesAggregator}. * This class is generated. Do not edit it. */ -public final class SpatialExtentGeoShapeAggregatorFunctionSupplier implements AggregatorFunctionSupplier { +public final class SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier implements AggregatorFunctionSupplier { private final List channels; - public SpatialExtentGeoShapeAggregatorFunctionSupplier(List channels) { + public SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier(List channels) { this.channels = channels; } @Override - public SpatialExtentGeoShapeAggregatorFunction aggregator(DriverContext driverContext) { - return SpatialExtentGeoShapeAggregatorFunction.create(driverContext, channels); + public SpatialExtentGeoShapeDocValuesAggregatorFunction aggregator(DriverContext driverContext) { + return SpatialExtentGeoShapeDocValuesAggregatorFunction.create(driverContext, channels); } @Override - public SpatialExtentGeoShapeGroupingAggregatorFunction groupingAggregator( + public SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction groupingAggregator( DriverContext driverContext) { - return SpatialExtentGeoShapeGroupingAggregatorFunction.create(channels, driverContext); + return SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction.create(channels, driverContext); } @Override public String describe() { - return "spatial_extent_geo of shapes"; + return "spatial_extent_geo_shape_doc of valuess"; } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..17300e5ee1a41 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction.java @@ -0,0 +1,231 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation.spatial; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction; +import org.elasticsearch.compute.aggregation.IntermediateStateDesc; +import org.elasticsearch.compute.aggregation.SeenGroupIds; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link SpatialExtentGeoShapeDocValuesAggregator}. + * This class is generated. Do not edit it. + */ +public final class SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("minNegX", ElementType.INT), + new IntermediateStateDesc("minPosX", ElementType.INT), + new IntermediateStateDesc("maxNegX", ElementType.INT), + new IntermediateStateDesc("maxPosX", ElementType.INT), + new IntermediateStateDesc("maxY", ElementType.INT), + new IntermediateStateDesc("minY", ElementType.INT) ); + + private final SpatialExtentGroupingStateWrappedLongitudeState state; + + private final List channels; + + private final DriverContext driverContext; + + public SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction(List channels, + SpatialExtentGroupingStateWrappedLongitudeState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction create( + List channels, DriverContext driverContext) { + return new SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction(channels, SpatialExtentGeoShapeDocValuesAggregator.initGrouping(), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + IntBlock valuesBlock = page.getBlock(channels.get(0)); + IntVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void close() { + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void close() { + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, IntBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + int[] valuesArray = new int[valuesEnd - valuesStart]; + for (int v = valuesStart; v < valuesEnd; v++) { + valuesArray[v-valuesStart] = values.getInt(v); + } + SpatialExtentGeoShapeDocValuesAggregator.combine(state, groupId, valuesArray); + } + } + + private void addRawInput(int positionOffset, IntVector groups, IntVector values) { + // This type does not support vectors because all values are multi-valued + } + + private void addRawInput(int positionOffset, IntBlock groups, IntBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = groups.getInt(g); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + int[] valuesArray = new int[valuesEnd - valuesStart]; + for (int v = valuesStart; v < valuesEnd; v++) { + valuesArray[v-valuesStart] = values.getInt(v); + } + SpatialExtentGeoShapeDocValuesAggregator.combine(state, groupId, valuesArray); + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, IntVector values) { + // This type does not support vectors because all values are multi-valued + } + + @Override + public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { + state.enableGroupIdTracking(seenGroupIds); + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block minNegXUncast = page.getBlock(channels.get(0)); + if (minNegXUncast.areAllValuesNull()) { + return; + } + IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); + Block minPosXUncast = page.getBlock(channels.get(1)); + if (minPosXUncast.areAllValuesNull()) { + return; + } + IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); + Block maxNegXUncast = page.getBlock(channels.get(2)); + if (maxNegXUncast.areAllValuesNull()) { + return; + } + IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); + Block maxPosXUncast = page.getBlock(channels.get(3)); + if (maxPosXUncast.areAllValuesNull()) { + return; + } + IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); + Block maxYUncast = page.getBlock(channels.get(4)); + if (maxYUncast.areAllValuesNull()) { + return; + } + IntVector maxY = ((IntBlock) maxYUncast).asVector(); + Block minYUncast = page.getBlock(channels.get(5)); + if (minYUncast.areAllValuesNull()) { + return; + } + IntVector minY = ((IntBlock) minYUncast).asVector(); + assert minNegX.getPositionCount() == minPosX.getPositionCount() && minNegX.getPositionCount() == maxNegX.getPositionCount() && minNegX.getPositionCount() == maxPosX.getPositionCount() && minNegX.getPositionCount() == maxY.getPositionCount() && minNegX.getPositionCount() == minY.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + SpatialExtentGeoShapeDocValuesAggregator.combineIntermediate(state, groupId, minNegX.getInt(groupPosition + positionOffset), minPosX.getInt(groupPosition + positionOffset), maxNegX.getInt(groupPosition + positionOffset), maxPosX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + SpatialExtentGroupingStateWrappedLongitudeState inState = ((SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + SpatialExtentGeoShapeDocValuesAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = SpatialExtentGeoShapeDocValuesAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunction.java similarity index 83% rename from x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregatorFunction.java rename to x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunction.java index abee9a1cee284..e23fbcd1b4032 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunction.java @@ -23,10 +23,10 @@ import org.elasticsearch.compute.operator.DriverContext; /** - * {@link AggregatorFunction} implementation for {@link SpatialExtentGeoShapeAggregator}. + * {@link AggregatorFunction} implementation for {@link SpatialExtentGeoShapeSourceValuesAggregator}. * This class is generated. Do not edit it. */ -public final class SpatialExtentGeoShapeAggregatorFunction implements AggregatorFunction { +public final class SpatialExtentGeoShapeSourceValuesAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( new IntermediateStateDesc("minNegX", ElementType.INT), new IntermediateStateDesc("minPosX", ElementType.INT), @@ -41,16 +41,16 @@ public final class SpatialExtentGeoShapeAggregatorFunction implements Aggregator private final List channels; - public SpatialExtentGeoShapeAggregatorFunction(DriverContext driverContext, + public SpatialExtentGeoShapeSourceValuesAggregatorFunction(DriverContext driverContext, List channels, SpatialExtentStateWrappedLongitudeState state) { this.driverContext = driverContext; this.channels = channels; this.state = state; } - public static SpatialExtentGeoShapeAggregatorFunction create(DriverContext driverContext, - List channels) { - return new SpatialExtentGeoShapeAggregatorFunction(driverContext, channels, SpatialExtentGeoShapeAggregator.initSingle()); + public static SpatialExtentGeoShapeSourceValuesAggregatorFunction create( + DriverContext driverContext, List channels) { + return new SpatialExtentGeoShapeSourceValuesAggregatorFunction(driverContext, channels, SpatialExtentGeoShapeSourceValuesAggregator.initSingle()); } public static List intermediateStateDesc() { @@ -92,7 +92,7 @@ public void addRawInput(Page page, BooleanVector mask) { private void addRawVector(BytesRefVector vector) { BytesRef scratch = new BytesRef(); for (int i = 0; i < vector.getPositionCount(); i++) { - SpatialExtentGeoShapeAggregator.combine(state, vector.getBytesRef(i, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, vector.getBytesRef(i, scratch)); } } @@ -102,7 +102,7 @@ private void addRawVector(BytesRefVector vector, BooleanVector mask) { if (mask.getBoolean(i) == false) { continue; } - SpatialExtentGeoShapeAggregator.combine(state, vector.getBytesRef(i, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, vector.getBytesRef(i, scratch)); } } @@ -115,7 +115,7 @@ private void addRawBlock(BytesRefBlock block) { int start = block.getFirstValueIndex(p); int end = start + block.getValueCount(p); for (int i = start; i < end; i++) { - SpatialExtentGeoShapeAggregator.combine(state, block.getBytesRef(i, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, block.getBytesRef(i, scratch)); } } } @@ -132,7 +132,7 @@ private void addRawBlock(BytesRefBlock block, BooleanVector mask) { int start = block.getFirstValueIndex(p); int end = start + block.getValueCount(p); for (int i = start; i < end; i++) { - SpatialExtentGeoShapeAggregator.combine(state, block.getBytesRef(i, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, block.getBytesRef(i, scratch)); } } } @@ -177,7 +177,7 @@ public void addIntermediateInput(Page page) { } IntVector minY = ((IntBlock) minYUncast).asVector(); assert minY.getPositionCount() == 1; - SpatialExtentGeoShapeAggregator.combineIntermediate(state, minNegX.getInt(0), minPosX.getInt(0), maxNegX.getInt(0), maxPosX.getInt(0), maxY.getInt(0), minY.getInt(0)); + SpatialExtentGeoShapeSourceValuesAggregator.combineIntermediate(state, minNegX.getInt(0), minPosX.getInt(0), maxNegX.getInt(0), maxPosX.getInt(0), maxY.getInt(0), minY.getInt(0)); } @Override @@ -187,7 +187,7 @@ public void evaluateIntermediate(Block[] blocks, int offset, DriverContext drive @Override public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { - blocks[offset] = SpatialExtentGeoShapeAggregator.evaluateFinal(state, driverContext); + blocks[offset] = SpatialExtentGeoShapeSourceValuesAggregator.evaluateFinal(state, driverContext); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier.java similarity index 54% rename from x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregatorFunctionSupplier.java rename to x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier.java index 9e4b292a0ea29..1eeb17367d852 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregatorFunctionSupplier.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier.java @@ -12,29 +12,30 @@ import org.elasticsearch.compute.operator.DriverContext; /** - * {@link AggregatorFunctionSupplier} implementation for {@link SpatialExtentCartesianShapeAggregator}. + * {@link AggregatorFunctionSupplier} implementation for {@link SpatialExtentGeoShapeSourceValuesAggregator}. * This class is generated. Do not edit it. */ -public final class SpatialExtentCartesianShapeAggregatorFunctionSupplier implements AggregatorFunctionSupplier { +public final class SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier implements AggregatorFunctionSupplier { private final List channels; - public SpatialExtentCartesianShapeAggregatorFunctionSupplier(List channels) { + public SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier(List channels) { this.channels = channels; } @Override - public SpatialExtentCartesianShapeAggregatorFunction aggregator(DriverContext driverContext) { - return SpatialExtentCartesianShapeAggregatorFunction.create(driverContext, channels); + public SpatialExtentGeoShapeSourceValuesAggregatorFunction aggregator( + DriverContext driverContext) { + return SpatialExtentGeoShapeSourceValuesAggregatorFunction.create(driverContext, channels); } @Override - public SpatialExtentCartesianShapeGroupingAggregatorFunction groupingAggregator( + public SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction groupingAggregator( DriverContext driverContext) { - return SpatialExtentCartesianShapeGroupingAggregatorFunction.create(channels, driverContext); + return SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction.create(channels, driverContext); } @Override public String describe() { - return "spatial_extent_cartesian of shapes"; + return "spatial_extent_geo_shape_source of valuess"; } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction.java similarity index 83% rename from x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeGroupingAggregatorFunction.java rename to x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction.java index 1200259ea6c41..b2b865b9f8a33 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction.java @@ -23,10 +23,10 @@ import org.elasticsearch.compute.operator.DriverContext; /** - * {@link GroupingAggregatorFunction} implementation for {@link SpatialExtentGeoShapeAggregator}. + * {@link GroupingAggregatorFunction} implementation for {@link SpatialExtentGeoShapeSourceValuesAggregator}. * This class is generated. Do not edit it. */ -public final class SpatialExtentGeoShapeGroupingAggregatorFunction implements GroupingAggregatorFunction { +public final class SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( new IntermediateStateDesc("minNegX", ElementType.INT), new IntermediateStateDesc("minPosX", ElementType.INT), @@ -41,16 +41,16 @@ public final class SpatialExtentGeoShapeGroupingAggregatorFunction implements Gr private final DriverContext driverContext; - public SpatialExtentGeoShapeGroupingAggregatorFunction(List channels, + public SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction(List channels, SpatialExtentGroupingStateWrappedLongitudeState state, DriverContext driverContext) { this.channels = channels; this.state = state; this.driverContext = driverContext; } - public static SpatialExtentGeoShapeGroupingAggregatorFunction create(List channels, - DriverContext driverContext) { - return new SpatialExtentGeoShapeGroupingAggregatorFunction(channels, SpatialExtentGeoShapeAggregator.initGrouping(), driverContext); + public static SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction create( + List channels, DriverContext driverContext) { + return new SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction(channels, SpatialExtentGeoShapeSourceValuesAggregator.initGrouping(), driverContext); } public static List intermediateStateDesc() { @@ -114,7 +114,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock val int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); for (int v = valuesStart; v < valuesEnd; v++) { - SpatialExtentGeoShapeAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); } } } @@ -123,7 +123,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefVector va BytesRef scratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentGeoShapeAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); } } @@ -143,7 +143,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock valu int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); for (int v = valuesStart; v < valuesEnd; v++) { - SpatialExtentGeoShapeAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); } } } @@ -159,7 +159,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector val int groupEnd = groupStart + groups.getValueCount(groupPosition); for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); - SpatialExtentGeoShapeAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); } } } @@ -206,7 +206,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page assert minNegX.getPositionCount() == minPosX.getPositionCount() && minNegX.getPositionCount() == maxNegX.getPositionCount() && minNegX.getPositionCount() == maxPosX.getPositionCount() && minNegX.getPositionCount() == maxY.getPositionCount() && minNegX.getPositionCount() == minY.getPositionCount(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentGeoShapeAggregator.combineIntermediate(state, groupId, minNegX.getInt(groupPosition + positionOffset), minPosX.getInt(groupPosition + positionOffset), maxNegX.getInt(groupPosition + positionOffset), maxPosX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); + SpatialExtentGeoShapeSourceValuesAggregator.combineIntermediate(state, groupId, minNegX.getInt(groupPosition + positionOffset), minPosX.getInt(groupPosition + positionOffset), maxNegX.getInt(groupPosition + positionOffset), maxPosX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); } } @@ -215,9 +215,9 @@ public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction inpu if (input.getClass() != getClass()) { throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); } - SpatialExtentGroupingStateWrappedLongitudeState inState = ((SpatialExtentGeoShapeGroupingAggregatorFunction) input).state; + SpatialExtentGroupingStateWrappedLongitudeState inState = ((SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction) input).state; state.enableGroupIdTracking(new SeenGroupIds.Empty()); - SpatialExtentGeoShapeAggregator.combineStates(state, groupId, inState, position); + SpatialExtentGeoShapeSourceValuesAggregator.combineStates(state, groupId, inState, position); } @Override @@ -228,7 +228,7 @@ public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) @Override public void evaluateFinal(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { - blocks[offset] = SpatialExtentGeoShapeAggregator.evaluateFinal(state, selected, driverContext); + blocks[offset] = SpatialExtentGeoShapeSourceValuesAggregator.evaluateFinal(state, selected, driverContext); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/GeoPointEnvelopeVisitor.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/GeoPointEnvelopeVisitor.java deleted file mode 100644 index 6bdd028f3d6ee..0000000000000 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/GeoPointEnvelopeVisitor.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.compute.aggregation.spatial; - -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; -import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor.WrapLongitude; - -class GeoPointEnvelopeVisitor extends SpatialEnvelopeVisitor.GeoPointVisitor { - GeoPointEnvelopeVisitor() { - super(WrapLongitude.WRAP); - } - - void reset() { - minY = Double.POSITIVE_INFINITY; - maxY = Double.NEGATIVE_INFINITY; - minNegX = Double.POSITIVE_INFINITY; - maxNegX = Double.NEGATIVE_INFINITY; - minPosX = Double.POSITIVE_INFINITY; - maxPosX = Double.NEGATIVE_INFINITY; - } - - double getMinNegX() { - return minNegX; - } - - double getMinPosX() { - return minPosX; - } - - double getMaxNegX() { - return maxNegX; - } - - double getMaxPosX() { - return maxPosX; - } - - double getMaxY() { - return maxY; - } - - double getMinY() { - return minY; - } - - static Rectangle asRectangle( - double minNegX, - double minPosX, - double maxNegX, - double maxPosX, - double maxY, - double minY, - WrapLongitude wrapLongitude - ) { - return SpatialEnvelopeVisitor.GeoPointVisitor.getResult(minNegX, minPosX, maxNegX, maxPosX, maxY, minY, wrapLongitude); - } -} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialAggregationUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialAggregationUtils.java index 6b29b20601dae..671ef6116ae6d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialAggregationUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialAggregationUtils.java @@ -12,12 +12,10 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.GeometryValidator; -import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor.WrapLongitude; import org.elasticsearch.geometry.utils.WellKnownBinary; -class SpatialAggregationUtils { +public class SpatialAggregationUtils { private SpatialAggregationUtils() { /* Utility class */ } public static Geometry decode(BytesRef wkb) { @@ -52,26 +50,12 @@ public static double decodeLatitude(long encoded) { return GeoEncodingUtils.decodeLatitude((int) (encoded >>> 32)); } - public static int encodeNegativeLongitude(double d) { - return Double.isFinite(d) ? GeoEncodingUtils.encodeLongitude(d) : DEFAULT_NEG; + public static int encodeLongitude(double d) { + return Double.isFinite(d) ? GeoEncodingUtils.encodeLongitude(d) : encodeInfinity(d); } - public static int encodePositiveLongitude(double d) { - return Double.isFinite(d) ? GeoEncodingUtils.encodeLongitude(d) : DEFAULT_POS; - } - - public static Rectangle asRectangle(int minNegX, int minPosX, int maxNegX, int maxPosX, int maxY, int minY) { - assert minNegX <= 0 == maxNegX <= 0; - assert minPosX >= 0 == maxPosX >= 0; - return GeoPointEnvelopeVisitor.asRectangle( - minNegX <= 0 ? decodeLongitude(minNegX) : Double.POSITIVE_INFINITY, - minPosX >= 0 ? decodeLongitude(minPosX) : Double.POSITIVE_INFINITY, - maxNegX <= 0 ? decodeLongitude(maxNegX) : Double.NEGATIVE_INFINITY, - maxPosX >= 0 ? decodeLongitude(maxPosX) : Double.NEGATIVE_INFINITY, - GeoEncodingUtils.decodeLatitude(maxY), - GeoEncodingUtils.decodeLatitude(minY), - WrapLongitude.WRAP - ); + private static int encodeInfinity(double d) { + return d == Double.NEGATIVE_INFINITY ? Integer.MIN_VALUE : Integer.MAX_VALUE; } public static int maxNeg(int a, int b) { @@ -81,8 +65,4 @@ public static int maxNeg(int a, int b) { public static int minPos(int a, int b) { return a >= 0 && b >= 0 ? Math.min(a, b) : Math.max(a, b); } - - // The default values are intentionally non-negative/non-positive, so we can mark unassigned values. - public static final int DEFAULT_POS = -1; - public static final int DEFAULT_NEG = 1; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregator.java new file mode 100644 index 0000000000000..2c2c3013a2510 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregator.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation.spatial; + +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; + +/** + * Computes the extent of a set of cartesian shapes. It is assumed that the cartesian shapes are encoded as WKB BytesRef. + * We do not currently support reading shape values or extents from doc values. + */ +@Aggregator( + { + @IntermediateState(name = "minX", type = "INT"), + @IntermediateState(name = "maxX", type = "INT"), + @IntermediateState(name = "maxY", type = "INT"), + @IntermediateState(name = "minY", type = "INT") } +) +@GroupingAggregator +class SpatialExtentCartesianShapeDocValuesAggregator extends SpatialExtentAggregator { + public static SpatialExtentState initSingle() { + return new SpatialExtentState(PointType.CARTESIAN); + } + + public static SpatialExtentGroupingState initGrouping() { + return new SpatialExtentGroupingState(PointType.CARTESIAN); + } + + public static void combine(SpatialExtentState current, int[] values) { + current.add(values); + } + + public static void combine(SpatialExtentGroupingState current, int groupId, int[] values) { + current.add(groupId, values); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregator.java similarity index 94% rename from x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregator.java rename to x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregator.java index 6d50d27aa5a2d..e90a8fe2bb00a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregator.java @@ -24,7 +24,7 @@ @IntermediateState(name = "minY", type = "INT") } ) @GroupingAggregator -class SpatialExtentCartesianShapeAggregator extends SpatialExtentAggregator { +class SpatialExtentCartesianShapeSourceValuesAggregator extends SpatialExtentAggregator { public static SpatialExtentState initSingle() { return new SpatialExtentState(PointType.CARTESIAN); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregator.java new file mode 100644 index 0000000000000..14d2d88325209 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregator.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation.spatial; + +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; + +/** + * Computes the extent of a set of geo shapes. It is assumed that the geo shapes are encoded as WKB BytesRef. + * We do not currently support reading shape values or extents from doc values. + */ +@Aggregator( + { + @IntermediateState(name = "minNegX", type = "INT"), + @IntermediateState(name = "minPosX", type = "INT"), + @IntermediateState(name = "maxNegX", type = "INT"), + @IntermediateState(name = "maxPosX", type = "INT"), + @IntermediateState(name = "maxY", type = "INT"), + @IntermediateState(name = "minY", type = "INT") } +) +@GroupingAggregator +class SpatialExtentGeoShapeDocValuesAggregator extends SpatialExtentLongitudeWrappingAggregator { + public static SpatialExtentStateWrappedLongitudeState initSingle() { + return new SpatialExtentStateWrappedLongitudeState(); + } + + public static SpatialExtentGroupingStateWrappedLongitudeState initGrouping() { + return new SpatialExtentGroupingStateWrappedLongitudeState(); + } + + public static void combine(SpatialExtentStateWrappedLongitudeState current, int[] values) { + current.add(values); + } + + public static void combine(SpatialExtentGroupingStateWrappedLongitudeState current, int groupId, int[] values) { + current.add(groupId, values); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregator.java similarity index 92% rename from x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregator.java rename to x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregator.java index 3d1b9b6300c9d..2bda4f5ddd9c1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregator.java @@ -26,8 +26,7 @@ @IntermediateState(name = "minY", type = "INT") } ) @GroupingAggregator -class SpatialExtentGeoShapeAggregator extends SpatialExtentLongitudeWrappingAggregator { - // TODO support non-longitude wrapped geo shapes. +class SpatialExtentGeoShapeSourceValuesAggregator extends SpatialExtentLongitudeWrappingAggregator { public static SpatialExtentStateWrappedLongitudeState initSingle() { return new SpatialExtentStateWrappedLongitudeState(); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java index cb765e4d6757e..c9ebc23773e0f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java @@ -18,6 +18,7 @@ import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import java.nio.ByteOrder; @@ -66,6 +67,32 @@ public void toIntermediate(Block[] blocks, int offset, IntVector selected, Drive } } + /** + * This method is used when extents are extracted from the doc-values field by the {@link GeometryDocValueReader}. + * This optimization is enabled when the field has doc-values and is only used in the ST_EXTENT aggregation. + */ + public void add(int groupId, int[] values) { + if (values.length == 6) { + // Values are stored according to the order defined in the Extent class + int top = values[0]; + int bottom = values[1]; + int negLeft = values[2]; + int negRight = values[3]; + int posLeft = values[4]; + int posRight = values[5]; + add(groupId, Math.min(negLeft, posLeft), Math.max(negRight, posRight), top, bottom); + } else if (values.length == 4) { + // Values are stored according to the order defined in the Rectangle class + int minX = values[0]; + int maxX = values[1]; + int maxY = values[2]; + int minY = values[3]; + add(groupId, minX, maxX, maxY, minY); + } else { + throw new IllegalArgumentException("Expected 4 or 6 values, got " + values.length); + } + } + public void add(int groupId, Geometry geometry) { ensureCapacity(groupId); pointType.computeEnvelope(geometry) @@ -80,6 +107,10 @@ public void add(int groupId, Geometry geometry) { ); } + /** + * This method is used when the field is a geo_point or cartesian_point and is loaded from doc-values. + * This optimization is enabled when the field has doc-values and is only used in a spatial aggregation. + */ public void add(int groupId, long encoded) { int x = pointType.extractX(encoded); int y = pointType.extractY(encoded); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java index 41bc50abcf6bc..cdbf82d56efc8 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java @@ -19,20 +19,23 @@ import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import java.nio.ByteOrder; +import static org.elasticsearch.compute.aggregation.spatial.SpatialExtentStateWrappedLongitudeState.asRectangle; + final class SpatialExtentGroupingStateWrappedLongitudeState extends AbstractArrayState implements GroupingAggregatorState { // Only geo points support longitude wrapping. private static final PointType POINT_TYPE = PointType.GEO; + private IntArray maxYs; + private IntArray minYs; private IntArray minNegXs; - private IntArray minPosXs; private IntArray maxNegXs; + private IntArray minPosXs; private IntArray maxPosXs; - private IntArray maxYs; - private IntArray minYs; - private GeoPointEnvelopeVisitor geoPointVisitor = new GeoPointEnvelopeVisitor(); + private final SpatialEnvelopeVisitor.GeoPointVisitor geoPointVisitor; SpatialExtentGroupingStateWrappedLongitudeState() { this(BigArrays.NON_RECYCLING_INSTANCE); @@ -47,6 +50,7 @@ final class SpatialExtentGroupingStateWrappedLongitudeState extends AbstractArra this.maxYs = bigArrays.newIntArray(0, false); this.minYs = bigArrays.newIntArray(0, false); enableGroupIdTracking(new SeenGroupIds.Empty()); + this.geoPointVisitor = new SpatialEnvelopeVisitor.GeoPointVisitor(SpatialEnvelopeVisitor.WrapLongitude.WRAP); } @Override @@ -63,8 +67,6 @@ public void toIntermediate(Block[] blocks, int offset, IntVector selected, Drive for (int i = 0; i < selected.getPositionCount(); i++) { int group = selected.getInt(i); assert hasValue(group); - assert minNegXs.get(group) <= 0 == maxNegXs.get(group) <= 0; - assert minPosXs.get(group) >= 0 == maxPosXs.get(group) >= 0; minNegXsBuilder.appendInt(minNegXs.get(group)); minPosXsBuilder.appendInt(minPosXs.get(group)); maxNegXsBuilder.appendInt(maxNegXs.get(group)); @@ -87,10 +89,10 @@ public void add(int groupId, Geometry geo) { if (geo.visit(new SpatialEnvelopeVisitor(geoPointVisitor))) { add( groupId, - SpatialAggregationUtils.encodeNegativeLongitude(geoPointVisitor.getMinNegX()), - SpatialAggregationUtils.encodePositiveLongitude(geoPointVisitor.getMinPosX()), - SpatialAggregationUtils.encodeNegativeLongitude(geoPointVisitor.getMaxNegX()), - SpatialAggregationUtils.encodePositiveLongitude(geoPointVisitor.getMaxPosX()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMinNegX()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMinPosX()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMaxNegX()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMaxPosX()), POINT_TYPE.encoder().encodeY(geoPointVisitor.getMaxY()), POINT_TYPE.encoder().encodeY(geoPointVisitor.getMinY()) ); @@ -112,12 +114,35 @@ public void add(int groupId, SpatialExtentGroupingStateWrappedLongitudeState inS } } + /** + * This method is used when the field is a geo_point or cartesian_point and is loaded from doc-values. + * This optimization is enabled when the field has doc-values and is only used in a spatial aggregation. + */ public void add(int groupId, long encoded) { int x = POINT_TYPE.extractX(encoded); int y = POINT_TYPE.extractY(encoded); add(groupId, x, x, x, x, y, y); } + /** + * This method is used when extents are extracted from the doc-values field by the {@link GeometryDocValueReader}. + * This optimization is enabled when the field has doc-values and is only used in the ST_EXTENT aggregation. + */ + public void add(int groupId, int[] values) { + if (values.length == 6) { + // Values are stored according to the order defined in the Extent class + int top = values[0]; + int bottom = values[1]; + int negLeft = values[2]; + int negRight = values[3]; + int posLeft = values[4]; + int posRight = values[5]; + add(groupId, negLeft, posLeft, negRight, posRight, top, bottom); + } else { + throw new IllegalArgumentException("Expected 6 values, got " + values.length); + } + } + public void add(int groupId, int minNegX, int minPosX, int maxNegX, int maxPosX, int maxY, int minY) { ensureCapacity(groupId); if (hasValue(groupId)) { @@ -135,8 +160,6 @@ public void add(int groupId, int minNegX, int minPosX, int maxNegX, int maxPosX, maxYs.set(groupId, maxY); minYs.set(groupId, minY); } - assert minNegX <= 0 == maxNegX <= 0 : "minNegX=" + minNegX + " maxNegX=" + maxNegX; - assert minPosX >= 0 == maxPosX >= 0 : "minPosX=" + minPosX + " maxPosX=" + maxPosX; trackGroupId(groupId); } @@ -160,7 +183,7 @@ public Block toBlock(IntVector selected, DriverContext driverContext) { builder.appendBytesRef( new BytesRef( WellKnownBinary.toWKB( - SpatialAggregationUtils.asRectangle( + asRectangle( minNegXs.get(si), minPosXs.get(si), maxNegXs.get(si), diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentState.java index 3dc150d1702a2..cd52d346b09f4 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentState.java @@ -15,6 +15,7 @@ import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.WellKnownBinary; import org.elasticsearch.lucene.spatial.CoordinateEncoder; +import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import java.nio.ByteOrder; @@ -55,6 +56,32 @@ public void add(Geometry geo) { ); } + /** + * This method is used when extents are extracted from the doc-values field by the {@link GeometryDocValueReader}. + * This optimization is enabled when the field has doc-values and is only used in the ST_EXTENT aggregation. + */ + public void add(int[] values) { + if (values.length == 6) { + // Values are stored according to the order defined in the Extent class + int top = values[0]; + int bottom = values[1]; + int negLeft = values[2]; + int negRight = values[3]; + int posLeft = values[4]; + int posRight = values[5]; + add(Math.min(negLeft, posLeft), Math.max(negRight, posRight), top, bottom); + } else if (values.length == 4) { + // Values are stored according to the order defined in the Rectangle class + int minX = values[0]; + int maxX = values[1]; + int maxY = values[2]; + int minY = values[3]; + add(minX, maxX, maxY, minY); + } else { + throw new IllegalArgumentException("Expected 4 or 6 values, got " + values.length); + } + } + public void add(int minX, int maxX, int maxY, int minY) { seen = true; this.minX = Math.min(this.minX, minX); @@ -63,6 +90,10 @@ public void add(int minX, int maxX, int maxY, int minY) { this.minY = Math.min(this.minY, minY); } + /** + * This method is used when the field is a geo_point or cartesian_point and is loaded from doc-values. + * This optimization is enabled when the field has doc-values and is only used in a spatial aggregation. + */ public void add(long encoded) { int x = pointType.extractX(encoded); int y = pointType.extractY(encoded); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java index 0d6163636fcde..043b632f64ee5 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java @@ -7,28 +7,35 @@ package org.elasticsearch.compute.aggregation.spatial; +import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.aggregation.AggregatorState; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import java.nio.ByteOrder; +import static org.elasticsearch.compute.aggregation.spatial.SpatialAggregationUtils.decodeLongitude; + final class SpatialExtentStateWrappedLongitudeState implements AggregatorState { // Only geo points support longitude wrapping. private static final PointType POINT_TYPE = PointType.GEO; private boolean seen = false; - private int minNegX = SpatialAggregationUtils.DEFAULT_NEG; - private int minPosX = SpatialAggregationUtils.DEFAULT_POS; - private int maxNegX = SpatialAggregationUtils.DEFAULT_NEG; - private int maxPosX = SpatialAggregationUtils.DEFAULT_POS; private int maxY = Integer.MIN_VALUE; private int minY = Integer.MAX_VALUE; + private int minNegX = Integer.MAX_VALUE; + private int maxNegX = Integer.MIN_VALUE; + private int minPosX = Integer.MAX_VALUE; + private int maxPosX = Integer.MIN_VALUE; - private GeoPointEnvelopeVisitor geoPointVisitor = new GeoPointEnvelopeVisitor(); + private final SpatialEnvelopeVisitor.GeoPointVisitor geoPointVisitor = new SpatialEnvelopeVisitor.GeoPointVisitor( + SpatialEnvelopeVisitor.WrapLongitude.WRAP + ); @Override public void close() {} @@ -49,16 +56,35 @@ public void add(Geometry geo) { geoPointVisitor.reset(); if (geo.visit(new SpatialEnvelopeVisitor(geoPointVisitor))) { add( - SpatialAggregationUtils.encodeNegativeLongitude(geoPointVisitor.getMinNegX()), - SpatialAggregationUtils.encodePositiveLongitude(geoPointVisitor.getMinPosX()), - SpatialAggregationUtils.encodeNegativeLongitude(geoPointVisitor.getMaxNegX()), - SpatialAggregationUtils.encodePositiveLongitude(geoPointVisitor.getMaxPosX()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMinNegX()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMinPosX()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMaxNegX()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMaxPosX()), POINT_TYPE.encoder().encodeY(geoPointVisitor.getMaxY()), POINT_TYPE.encoder().encodeY(geoPointVisitor.getMinY()) ); } } + /** + * This method is used when extents are extracted from the doc-values field by the {@link GeometryDocValueReader}. + * This optimization is enabled when the field has doc-values and is only used in the ST_EXTENT aggregation. + */ + public void add(int[] values) { + if (values.length == 6) { + // Values are stored according to the order defined in the Extent class + int top = values[0]; + int bottom = values[1]; + int negLeft = values[2]; + int negRight = values[3]; + int posLeft = values[4]; + int posRight = values[5]; + add(negLeft, posLeft, negRight, posRight, top, bottom); + } else { + throw new IllegalArgumentException("Expected 6 values, got " + values.length); + } + } + public void add(int minNegX, int minPosX, int maxNegX, int maxPosX, int maxY, int minY) { seen = true; this.minNegX = Math.min(this.minNegX, minNegX); @@ -67,10 +93,12 @@ public void add(int minNegX, int minPosX, int maxNegX, int maxPosX, int maxY, in this.maxPosX = Math.max(this.maxPosX, maxPosX); this.maxY = Math.max(this.maxY, maxY); this.minY = Math.min(this.minY, minY); - assert this.minNegX <= 0 == this.maxNegX <= 0 : "minNegX=" + this.minNegX + " maxNegX=" + this.maxNegX; - assert this.minPosX >= 0 == this.maxPosX >= 0 : "minPosX=" + this.minPosX + " maxPosX=" + this.maxPosX; } + /** + * This method is used when the field is a geo_point or cartesian_point and is loaded from doc-values. + * This optimization is enabled when the field has doc-values and is only used in a spatial aggregation. + */ public void add(long encoded) { int x = POINT_TYPE.extractX(encoded); int y = POINT_TYPE.extractY(encoded); @@ -83,9 +111,18 @@ public Block toBlock(DriverContext driverContext) { } private byte[] toWKB() { - return WellKnownBinary.toWKB( - SpatialAggregationUtils.asRectangle(minNegX, minPosX, maxNegX, maxPosX, maxY, minY), - ByteOrder.LITTLE_ENDIAN + return WellKnownBinary.toWKB(asRectangle(minNegX, minPosX, maxNegX, maxPosX, maxY, minY), ByteOrder.LITTLE_ENDIAN); + } + + static Rectangle asRectangle(int minNegX, int minPosX, int maxNegX, int maxPosX, int maxY, int minY) { + return SpatialEnvelopeVisitor.GeoPointVisitor.getResult( + minNegX <= 0 ? decodeLongitude(minNegX) : Double.POSITIVE_INFINITY, + minPosX >= 0 ? decodeLongitude(minPosX) : Double.POSITIVE_INFINITY, + maxNegX <= 0 ? decodeLongitude(maxNegX) : Double.NEGATIVE_INFINITY, + maxPosX >= 0 ? decodeLongitude(maxPosX) : Double.NEGATIVE_INFINITY, + GeoEncodingUtils.decodeLatitude(maxY), + GeoEncodingUtils.decodeLatitude(minY), + SpatialEnvelopeVisitor.WrapLongitude.WRAP ); } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index 689c36197ee70..868d5c50ec2b0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -558,7 +558,10 @@ BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) stExtentMultipleGeoPointGrouping required_capability: st_extent_agg -FROM airports | STATS extent = ST_EXTENT_AGG(location) BY country | SORT country | LIMIT 3 +FROM airports +| STATS extent = ST_EXTENT_AGG(location) BY country +| SORT country +| LIMIT 3 ; extent:geo_shape | country:keyword @@ -569,11 +572,140 @@ BBOX (-0.6067969836294651, 6.621946580708027, 36.69972063973546, 35.620274716056 stExtentGeoShapes required_capability: st_extent_agg -FROM airport_city_boundaries | WHERE region == "City of New York" | STATS extent = ST_EXTENT_AGG(city_boundary) + +FROM airport_city_boundaries +| WHERE region == "City of New York" +| STATS extent = ST_EXTENT_AGG(city_boundary), count = COUNT() ; -extent:geo_shape -BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) +extent:geo_shape | count:long +BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | 3 +; + +stExtentGeoPoints +required_capability: st_extent_agg + +FROM airport_city_boundaries +| WHERE region == "City of New York" +| STATS extent = ST_EXTENT_AGG(city_location), count = COUNT() +; + +extent:geo_shape | count:long +BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) | 3 +; + +stExtentGeoShapesAndPoints +required_capability: st_extent_agg + +FROM airport_city_boundaries +| WHERE region == "City of New York" +| STATS extent_shapes = ST_EXTENT_AGG(city_boundary), extent_points = ST_EXTENT_AGG(city_location), count = COUNT() +; + +extent_shapes:geo_shape | extent_points:geo_shape | count:long +BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) | 3 +; + +stExtentGeoShapesGrouped +required_capability: st_extent_agg + +FROM airport_city_boundaries +| WHERE region == "City of New York" +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent = ST_EXTENT_AGG(city_boundary), count = COUNT() BY prefix +| KEEP prefix, count, extent +| SORT prefix ASC +; + +prefix:keyword | count:long | extent:geo_shape +E | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) +J | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) +L | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) +; + +stExtentGeoPointsGrouped +required_capability: st_extent_agg + +FROM airport_city_boundaries +| WHERE region == "City of New York" +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent = ST_EXTENT_AGG(city_location), count = COUNT() BY prefix +| KEEP prefix, count, extent +| SORT prefix ASC +; + +prefix:keyword | count:long | extent:geo_shape +E | 1 | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +J | 1 | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +L | 1 | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +; + +stExtentGeoShapesAndPointsGrouped +required_capability: st_extent_agg + +FROM airport_city_boundaries +| WHERE region == "City of New York" +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent_shapes = ST_EXTENT_AGG(city_boundary), extent_points = ST_EXTENT_AGG(city_location), count = COUNT() BY prefix +| KEEP prefix, count, extent_shapes, extent_points +| SORT prefix ASC +; + +prefix:keyword | count:long | extent_shapes:geo_shape | extent_points:geo_shape +E | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +J | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +L | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +; + +stExtentManyGeoShapesGrouped +required_capability: st_extent_agg + +FROM airport_city_boundaries +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent = ST_EXTENT_AGG(city_boundary), count = COUNT() BY prefix +| KEEP prefix, count, extent +| SORT count DESC, prefix ASC +| LIMIT 3 +; + +prefix:keyword | count:long | extent:geo_shape +S | 70 | BBOX (-136.45440001040697, 178.8686999771744, 61.38089996762574, -33.92440003808588) +C | 64 | BBOX (-107.51820000819862, 172.6055999379605, 55.732699991203845, -43.90400002710521) +B | 61 | BBOX (-116.51340007781982, 153.2021999359131, 60.631899973377585, -41.20620000176132) +; + +stExtentManyGeoPointsGrouped +required_capability: st_extent_agg + +FROM airport_city_boundaries +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent = ST_EXTENT_AGG(city_location), count = COUNT() BY prefix +| KEEP prefix, count, extent +| SORT count DESC, prefix ASC +| LIMIT 3 +; + +prefix:keyword | count:long | extent:geo_shape +S | 70 | BBOX (-135.3152000438422, 178.54539999738336, 61.24999999534339, -33.8678000215441) +C | 64 | BBOX (-107.39390007220209, 172.38329996354878, 55.676099974662066, -43.58330002985895) +B | 61 | BBOX (-116.23080002143979, 153.02809992805123, 60.46669999603182, -41.1500000115484) +; + +stExtentManyGeoShapesAndPointsGrouped +required_capability: st_extent_agg + +FROM airport_city_boundaries +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent_shapes = ST_EXTENT_AGG(city_boundary), extent_points = ST_EXTENT_AGG(city_location), count = COUNT() BY prefix +| KEEP prefix, count, extent_shapes, extent_points +| SORT count DESC, prefix ASC +| LIMIT 3 +; + +prefix:keyword | count:long | extent_shapes:geo_shape | extent_points:geo_shape +S | 70 | BBOX (-136.45440001040697, 178.8686999771744, 61.38089996762574, -33.92440003808588) | BBOX (-135.3152000438422, 178.54539999738336, 61.24999999534339, -33.8678000215441) +C | 64 | BBOX (-107.51820000819862, 172.6055999379605, 55.732699991203845, -43.90400002710521) | BBOX (-107.39390007220209, 172.38329996354878, 55.676099974662066, -43.58330002985895) +B | 61 | BBOX (-116.51340007781982, 153.2021999359131, 60.631899973377585, -41.20620000176132) | BBOX (-116.23080002143979, 153.02809992805123, 60.46669999603182, -41.1500000115484) ; ############################################### diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java index f68f9f2487884..248c151bcf948 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java @@ -39,7 +39,7 @@ protected SpatialAggregateFunction(StreamInput in, FieldExtractPreference fieldE this.fieldExtractPreference = fieldExtractPreference; } - public abstract SpatialAggregateFunction withDocValues(); + public abstract SpatialAggregateFunction withFieldExtractPreference(FieldExtractPreference preference); @Override public boolean licenseCheck(XPackLicenseState state) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java index 54c05cf1bad52..fad308e38cb26 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java @@ -71,8 +71,8 @@ public SpatialCentroid withFilter(Expression filter) { } @Override - public SpatialCentroid withDocValues() { - return new SpatialCentroid(source(), field(), filter(), FieldExtractPreference.DOC_VALUES); + public SpatialCentroid withFieldExtractPreference(FieldExtractPreference preference) { + return new SpatialCentroid(source(), field(), filter(), preference); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java index 34e5c9d68fc86..6326b9522a3cd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java @@ -11,10 +11,12 @@ import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.spatial.SpatialExtentCartesianPointDocValuesAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.spatial.SpatialExtentCartesianPointSourceValuesAggregatorFunctionSupplier; -import org.elasticsearch.compute.aggregation.spatial.SpatialExtentCartesianShapeAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.spatial.SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.spatial.SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.spatial.SpatialExtentGeoPointDocValuesAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.spatial.SpatialExtentGeoPointSourceValuesAggregatorFunctionSupplier; -import org.elasticsearch.compute.aggregation.spatial.SpatialExtentGeoShapeAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.spatial.SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.spatial.SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier; import org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -75,8 +77,8 @@ public SpatialExtent withFilter(Expression filter) { } @Override - public org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialExtent withDocValues() { - return new SpatialExtent(source(), field(), filter(), FieldExtractPreference.DOC_VALUES); + public SpatialExtent withFieldExtractPreference(FieldExtractPreference preference) { + return new SpatialExtent(source(), field(), filter(), preference); } @Override @@ -101,7 +103,8 @@ public SpatialExtent replaceChildren(List newChildren) { @Override public AggregatorFunctionSupplier supplier(List inputChannels) { - return switch (field().dataType()) { + DataType type = field().dataType(); + return switch (type) { case DataType.GEO_POINT -> switch (fieldExtractPreference) { case DOC_VALUES -> new SpatialExtentGeoPointDocValuesAggregatorFunctionSupplier(inputChannels); case NONE, EXTRACT_SPATIAL_BOUNDS -> new SpatialExtentGeoPointSourceValuesAggregatorFunctionSupplier(inputChannels); @@ -110,10 +113,17 @@ public AggregatorFunctionSupplier supplier(List inputChannels) { case DOC_VALUES -> new SpatialExtentCartesianPointDocValuesAggregatorFunctionSupplier(inputChannels); case NONE, EXTRACT_SPATIAL_BOUNDS -> new SpatialExtentCartesianPointSourceValuesAggregatorFunctionSupplier(inputChannels); }; - // Shapes don't differentiate between source and doc values. - case DataType.GEO_SHAPE -> new SpatialExtentGeoShapeAggregatorFunctionSupplier(inputChannels); - case DataType.CARTESIAN_SHAPE -> new SpatialExtentCartesianShapeAggregatorFunctionSupplier(inputChannels); - default -> throw EsqlIllegalArgumentException.illegalDataType(field().dataType()); + case DataType.GEO_SHAPE -> switch (fieldExtractPreference) { + case EXTRACT_SPATIAL_BOUNDS -> new SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier(inputChannels); + case NONE -> new SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier(inputChannels); + default -> throw new EsqlIllegalArgumentException("Illegal field extract preference: " + fieldExtractPreference); + }; + case DataType.CARTESIAN_SHAPE -> switch (fieldExtractPreference) { + case EXTRACT_SPATIAL_BOUNDS -> new SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier(inputChannels); + case NONE -> new SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier(inputChannels); + default -> throw new EsqlIllegalArgumentException("Illegal field extract preference: " + fieldExtractPreference); + }; + default -> throw EsqlIllegalArgumentException.illegalDataType(type); }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java index f66ed5c8e4ec1..d70153258871e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -84,7 +85,9 @@ && allowedForDocValues(fieldAttribute, ctx.searchStats(), agg, foundAttributes)) // We need to both mark the field to load differently, and change the spatial function to know to use it foundAttributes.add(fieldAttribute); changedAggregates = true; - orderedAggregates.add(as.replaceChild(af.withDocValues())); + orderedAggregates.add( + as.replaceChild(af.withFieldExtractPreference(MappedFieldType.FieldExtractPreference.DOC_VALUES)) + ); } else { orderedAggregates.add(aggExpr); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java index f6f087064a02f..c84b83f15231f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.lucene.spatial.GeometryDocValueWriter; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; @@ -49,9 +50,20 @@ public class SpatialShapeBoundsExtraction extends ParameterizedOptimizerRule { @Override protected PhysicalPlan rule(AggregateExec aggregate, LocalPhysicalOptimizerContext ctx) { - var foundAttributes = new HashSet(); + var foundAttributes = findSpatialShapeBoundsAttributes(aggregate, ctx); + if (foundAttributes.isEmpty()) { + return aggregate; + } + return aggregate.transformDown(PhysicalPlan.class, exec -> switch (exec) { + case AggregateExec agg -> transformAggregateExec(agg, foundAttributes); + case FieldExtractExec fieldExtractExec -> transformFieldExtractExec(fieldExtractExec, foundAttributes); + default -> exec; + }); + } - return aggregate.transformDown(UnaryExec.class, exec -> { + private Set findSpatialShapeBoundsAttributes(AggregateExec aggregate, LocalPhysicalOptimizerContext ctx) { + var foundAttributes = new HashSet(); + aggregate.transformDown(UnaryExec.class, exec -> { switch (exec) { case AggregateExec agg -> { List aggregateFunctions = agg.aggregates() @@ -84,18 +96,27 @@ protected PhysicalPlan rule(AggregateExec aggregate, LocalPhysicalOptimizerConte } case EvalExec evalExec -> foundAttributes.removeAll(evalExec.references()); case FilterExec filterExec -> foundAttributes.removeAll(filterExec.condition().references()); - case FieldExtractExec fieldExtractExec -> { - var boundsAttributes = new HashSet<>(foundAttributes); - boundsAttributes.retainAll(fieldExtractExec.attributesToExtract()); - if (boundsAttributes.isEmpty() == false) { - exec = fieldExtractExec.withBoundsAttributes(boundsAttributes); - } - } default -> { // Do nothing } } return exec; }); + return foundAttributes; + } + + private PhysicalPlan transformFieldExtractExec(FieldExtractExec fieldExtractExec, Set foundAttributes) { + var boundsAttributes = new HashSet<>(foundAttributes); + boundsAttributes.retainAll(fieldExtractExec.attributesToExtract()); + return fieldExtractExec.withBoundsAttributes(boundsAttributes); + } + + private PhysicalPlan transformAggregateExec(AggregateExec agg, Set foundAttributes) { + return agg.transformExpressionsDown( + SpatialExtent.class, + spatialExtent -> foundAttributes.contains(spatialExtent.field()) + ? spatialExtent.withFieldExtractPreference(MappedFieldType.FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS) + : spatialExtent + ); } private static boolean isShape(DataType dataType) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index 1918e3036e2b0..e420cd501cccd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -180,10 +180,8 @@ private static Stream, Tuple>> typeAndNames(Class types = List.of("GeoPoint", "CartesianPoint"); extraConfigs = SPATIAL_EXTRA_CONFIGS; } else if (clazz == SpatialExtent.class) { - return Stream.concat( - combine(clazz, List.of("GeoPoint", "CartesianPoint"), SPATIAL_EXTRA_CONFIGS), - combine(clazz, List.of("GeoShape", "CartesianShape"), List.of("")) - ); + types = List.of("GeoPoint", "CartesianPoint", "GeoShape", "CartesianShape"); + extraConfigs = SPATIAL_EXTRA_CONFIGS; } else if (Values.class.isAssignableFrom(clazz)) { // TODO can't we figure this out from the function itself? types = List.of("Int", "Long", "Double", "Boolean", "BytesRef"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index a312d048db0ad..6ecba81aa6eb2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -59,6 +59,7 @@ import static java.util.Arrays.asList; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES; +import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.NONE; import static org.elasticsearch.xpack.esql.core.util.Queries.Clause.FILTER; import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.canPushToSource; @@ -271,7 +272,7 @@ public static ElementType toElementType(DataType dataType, MappedFieldType.Field case DOC_DATA_TYPE -> ElementType.DOC; case TSID_DATA_TYPE -> ElementType.BYTES_REF; case GEO_POINT, CARTESIAN_POINT -> fieldExtractPreference == DOC_VALUES ? ElementType.LONG : ElementType.BYTES_REF; - case GEO_SHAPE, CARTESIAN_SHAPE -> ElementType.BYTES_REF; + case GEO_SHAPE, CARTESIAN_SHAPE -> fieldExtractPreference == EXTRACT_SPATIAL_BOUNDS ? ElementType.INT : ElementType.BYTES_REF; case PARTIAL_AGG -> ElementType.COMPOSITE; case SHORT, BYTE, DATE_PERIOD, TIME_DURATION, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT -> throw EsqlIllegalArgumentException .illegalDataType(dataType); @@ -287,11 +288,4 @@ public static ElementType toElementType(DataType dataType, MappedFieldType.Field new NoopCircuitBreaker("noop-esql-breaker"), BigArrays.NON_RECYCLING_INSTANCE ); - - /** - * Returns DOC_VALUES if the given boolean is set. - */ - public static MappedFieldType.FieldExtractPreference extractPreference(boolean hasPreference) { - return hasPreference ? DOC_VALUES : NONE; - } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 80f2772945e93..0200f54a983b3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -59,6 +59,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; @@ -215,7 +216,10 @@ public class PhysicalPlanOptimizerTests extends ESTestCase { private TestDataSource airportsNotIndexed; // Test when spatial field has doc values but is not indexed private TestDataSource airportsNotIndexedNorDocValues; // Test when spatial field is neither indexed nor has doc-values private TestDataSource airportsWeb; // Cartesian point field tests - private TestDataSource airportsCityBoundaries; + private TestDataSource airportsCityBoundaries; // geo_shape field tests + private TestDataSource airportsCityBoundariesNoPointDocValues; // Disable doc-values on geo_point fields, but not geo_shape fields + private TestDataSource airportsCityBoundariesNoShapeDocValues; // Disable doc-values on geo_shape fields, but not geo_point fields + private TestDataSource airportsCityBoundariesNoDocValues; // Dsiable doc-values on both geo_point and geo_shape fields private TestDataSource cartesianMultipolygons; // cartesian_shape field tests private TestDataSource cartesianMultipolygonsNoDocValues; // cartesian_shape field tests but has no doc values private TestDataSource countriesBbox; // geo_shape field tests @@ -293,6 +297,27 @@ public void init() { functionRegistry, enrichResolution ); + this.airportsCityBoundariesNoPointDocValues = makeTestDataSource( + "airports_city_boundaries", + "mapping-airport_city_boundaries.json", + functionRegistry, + enrichResolution, + new TestConfigurableSearchStats().exclude(Config.DOC_VALUES, "location", "city_location") + ); + this.airportsCityBoundariesNoShapeDocValues = makeTestDataSource( + "airports_city_boundaries", + "mapping-airport_city_boundaries.json", + functionRegistry, + enrichResolution, + new TestConfigurableSearchStats().exclude(Config.DOC_VALUES, "city_boundary") + ); + this.airportsCityBoundariesNoDocValues = makeTestDataSource( + "airports_city_boundaries", + "mapping-airport_city_boundaries.json", + functionRegistry, + enrichResolution, + new TestConfigurableSearchStats().exclude(Config.DOC_VALUES, "city_boundary", "location", "city_location") + ); this.cartesianMultipolygons = makeTestDataSource( "cartesian_multipolygons", "mapping-cartesian_multipolygons.json", @@ -3271,39 +3296,39 @@ public void testSpatialTypesAndStatsExtentAndCentroidUseDocValues() { * ][_doc{f}#36], limit[], sort[] estimatedRowSize[204] * */ - public void testSpatialTypesAndStatsExtentOfGeoShapeDoesNotUseBinaryExtraction() { - // TODO: When we get geo_shape working with bounds extraction from doc-values, change the name of this test + public void testSpatialTypesAndStatsExtentOfGeoShapeUsesBinaryExtraction() { var query = "FROM airports_city_boundaries | STATS extent = ST_EXTENT_AGG(city_boundary)"; - var testData = airportsCityBoundaries; - var plan = physicalPlan(query, testData); + for (boolean useDocValues : new Boolean[] { true, false }) { + var testData = useDocValues ? airportsCityBoundaries : airportsCityBoundariesNoDocValues; + var plan = physicalPlan(query, testData); - var limit = as(plan, LimitExec.class); - var agg = as(limit.child(), AggregateExec.class); - // Before optimization the aggregation does not use extent extraction - assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + // Before optimization the aggregation does not use extent extraction + assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); - var exchange = as(agg.child(), ExchangeExec.class); - var fragment = as(exchange.child(), FragmentExec.class); - var fAgg = as(fragment.fragment(), Aggregate.class); - as(fAgg.child(), EsRelation.class); + var exchange = as(agg.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var fAgg = as(fragment.fragment(), Aggregate.class); + as(fAgg.child(), EsRelation.class); - // Now optimize the plan and assert the aggregation uses extent extraction - var optimized = optimizedPlan(plan, testData.stats); - limit = as(optimized, LimitExec.class); - agg = as(limit.child(), AggregateExec.class); - // Above the exchange (in coordinator) the aggregation is not using doc-values - assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); - exchange = as(agg.child(), ExchangeExec.class); - agg = as(exchange.child(), AggregateExec.class); - // below the exchange (in data node) the aggregation is using a specific - assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); - assertChildIsExtractedAs(agg, FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS, GEO_SHAPE); + // Now optimize the plan and assert the aggregation uses extent extraction + var optimized = optimizedPlan(plan, testData.stats); + limit = as(optimized, LimitExec.class); + agg = as(limit.child(), AggregateExec.class); + // Above the exchange (in coordinator) the aggregation is not using doc-values + assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); + exchange = as(agg.child(), ExchangeExec.class); + agg = as(exchange.child(), AggregateExec.class); + // below the exchange (in data node) the aggregation is using a specific int[] which the aggregation needs to know about. + var fieldExtractPreference = useDocValues ? FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS : FieldExtractPreference.NONE; + assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, fieldExtractPreference); + assertChildIsExtractedAs(agg, fieldExtractPreference, GEO_SHAPE); + } } /** * This test verifies that the aggregation does not use spatial bounds extraction when the shape appears in an eval or filter. - * TODO: Currently this tests nothing, because geo_shape is not supported anyway for bounds extraction, - * but it should be updated when it is supported. */ public void testSpatialTypesAndStatsExtentOfShapesNegativeCases() { for (String query : new String[] { """ @@ -3326,6 +3351,7 @@ public void testSpatialTypesAndStatsExtentOfShapesNegativeCases() { assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); var exchange = as(agg.child(), ExchangeExec.class); agg = as(exchange.child(), AggregateExec.class); + // Because the shape was used in EVAL/WHERE we cannot use doc-values bounds extraction optimization assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); var exec = agg.child() instanceof FieldExtractExec ? agg : as(agg.child(), UnaryExec.class); assertChildIsExtractedAs(exec, FieldExtractPreference.NONE, GEO_SHAPE); @@ -3351,19 +3377,11 @@ public void testSpatialTypesAndStatsExtentOfCartesianShapesWithAndWithoutDocValu var optimized = optimizedPlan(plan, testData.stats); limit = as(optimized, LimitExec.class); agg = as(limit.child(), AggregateExec.class); - // For cartesian_shape extraction, we extract bounds from doc-values directly into a BBOX encoded as BytesRef, - // so the aggregation does not need to know about it. assertAggregation(agg, "extent", SpatialExtent.class, CARTESIAN_SHAPE, FieldExtractPreference.NONE); var exchange = as(agg.child(), ExchangeExec.class); agg = as(exchange.child(), AggregateExec.class); - assertAggregation( - agg, - "extent", - "hasDocValues:" + hasDocValues, - SpatialExtent.class, - CARTESIAN_SHAPE, - FieldExtractPreference.NONE - ); + // We extract bounds from doc-values into a special int[] which the aggregation needs to know about. + assertAggregation(agg, "extent", "hasDocValues:" + hasDocValues, SpatialExtent.class, CARTESIAN_SHAPE, fieldExtractPreference); var exec = agg.child() instanceof FieldExtractExec ? agg : as(agg.child(), UnaryExec.class); // For cartesian_shape, the bounds extraction is done in the FieldExtractExec, so it does need to know about this assertChildIsExtractedAs(exec, fieldExtractPreference, CARTESIAN_SHAPE); @@ -3371,60 +3389,72 @@ public void testSpatialTypesAndStatsExtentOfCartesianShapesWithAndWithoutDocValu } /** - * Before local optimizations: - * - * LimitExec[1000[INTEGER]] - * \_AggregateExec[[],[SPATIALEXTENT(city_boundary{f}#13,true[BOOLEAN]) AS extent, SPATIALCENTROID(city_location{f}#12,true[BOOLEA - * N]) AS centroid],...] - * \_ExchangeExec[[..]] - * \_FragmentExec[filter=null, estimatedRowSize=0, reducer=[], fragment=[...]] - * \_EsRelation[airports_city_boundaries][abbrev{f}#8, airport{f}#9, city{f}#11, city_boundar..] - * - * After local optimizations: + * This tests all four combinations of geo_point and geo_shape with and without doc-values. + * Since each will be extracted differently (points as encoded longs, and shapes as int[5] bounds representing Extents), + * we want to verify that the combinations do not clash and work together. + * The optimized query plan in the case when both points and shapes have doc-values will look like: * * LimitExec[1000[INTEGER]] - * \_AggregateExec[[],[SPATIALSTEXTENT(location{f}#48,true[BOOLEAN]) AS extent],FINAL,[minNegX{r}#52, minPosX{r}#53, maxNegX{r}#54, - * maxPosX{r}#55, maxY{r}#56, minY{r}#57],21] - * \_ExchangeExec[[minNegX{r}#52, minPosX{r}#53, maxNegX{r}#54, maxPosX{r}#55, maxY{r}#56, minY{r}#57],true] - * \_AggregateExec[[],[SPATIALSTEXTENT(location{f}#48,true[BOOLEAN]) AS extent],INITIAL,[ - * minNegX{r}#73, minPosX{r}#74, maxNegX{rb#75, maxPosX{r}#76, maxY{r}#77, minY{r}#78],21] - * \_FieldExtractExec[location{f}#48][location{f}#48] - * \_EsQueryExec[airports], indexMode[standard], query[{"exists":{"field":"location","boost":1.0}}][ - * _doc{f}#79], limit[], sort[] estimatedRowSize[25] + * \_AggregateExec[[],[ + * SPATIALEXTENT(city_boundary{f}#13,true[BOOLEAN]) AS extent, + * SPATIALCENTROID(city_location{f}#12,true[BOOLEAN]) AS centroid + * ],FINAL,[...bounds attributes..., ...centroid attributes...],221] + * \_ExchangeExec[[...bounds attributes..., ...centroid attributes...],true] + * \_AggregateExec[[],[ + * SPATIALEXTENT(city_boundary{f}#13,true[BOOLEAN]) AS extent, + * SPATIALCENTROID(city_location{f}#12,true[BOOLEAN]) AS centroid + * ],INITIAL,[...bounds attributes..., ...centroid attributes...],221] + * \_FieldExtractExec[city_boundary{f}#13, city_location{f}#12][city_location{f}#12],[city_boundary{f}#13] + * \_EsQueryExec[airports_city_boundaries], indexMode[standard], query[ + * {"bool":{"should":[ + * {"exists":{"field":"city_boundary","boost":1.0}}, + * {"exists":{"field":"city_location","boost":1.0}} + * ],"boost":1.0}} + * ][_doc{f}#55], limit[], sort[] estimatedRowSize[225] * */ public void testMixedSpatialBoundsAndPointsExtracted() { var query = """ FROM airports_city_boundaries \ | STATS extent = ST_EXTENT_AGG(city_boundary), centroid = ST_CENTROID_AGG(city_location)"""; - var testData = airportsCityBoundaries; - var plan = physicalPlan(query, testData); + for (boolean pointDocValues : new Boolean[] { true, false }) { + for (boolean shapeDocValues : new Boolean[] { true, false }) { + var testData = pointDocValues + ? (shapeDocValues ? airportsCityBoundaries : airportsCityBoundariesNoShapeDocValues) + : (shapeDocValues ? airportsCityBoundariesNoPointDocValues : airportsCityBoundariesNoDocValues); + var msg = "DocValues[point:" + pointDocValues + ", shape:" + shapeDocValues + "]"; + var plan = physicalPlan(query, testData); - var limit = as(plan, LimitExec.class); - var agg = as(limit.child(), AggregateExec.class); - // Before optimization the aggregation does not use doc-values - assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); - assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE); + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + // Before optimization the aggregation does not use doc-values + assertAggregation(agg, "extent", msg, SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); + assertAggregation(agg, "centroid", msg, SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE); - var exchange = as(agg.child(), ExchangeExec.class); - var fragment = as(exchange.child(), FragmentExec.class); - var fAgg = as(fragment.fragment(), Aggregate.class); - as(fAgg.child(), EsRelation.class); + var exchange = as(agg.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var fAgg = as(fragment.fragment(), Aggregate.class); + as(fAgg.child(), EsRelation.class); - // Now optimize the plan and assert the aggregation uses both doc-values and bounds extraction - var optimized = optimizedPlan(plan, testData.stats); - limit = as(optimized, LimitExec.class); - agg = as(limit.child(), AggregateExec.class); - // Above the exchange (in coordinator) the aggregation is not field-optimized. - assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); - assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE); - exchange = as(agg.child(), ExchangeExec.class); - agg = as(exchange.child(), AggregateExec.class); - // below the exchange (in data node) the aggregation is field optimized. - assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); - var fieldExtractExec = as(agg.child(), FieldExtractExec.class); - assertThat(fieldExtractExec.boundsAttributes().stream().map(a -> a.sourceText()).toList(), equalTo(List.of("city_boundary"))); - assertThat(fieldExtractExec.docValuesAttributes().stream().map(a -> a.sourceText()).toList(), equalTo(List.of("city_location"))); + // Now optimize the plan and assert the aggregation uses both doc-values and bounds extraction + var optimized = optimizedPlan(plan, testData.stats); + limit = as(optimized, LimitExec.class); + agg = as(limit.child(), AggregateExec.class); + // Above the exchange (in coordinator) the aggregation is not field-optimized. + assertAggregation(agg, "extent", msg, SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); + assertAggregation(agg, "centroid", msg, SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE); + exchange = as(agg.child(), ExchangeExec.class); + agg = as(exchange.child(), AggregateExec.class); + var fieldExtractExec = as(agg.child(), FieldExtractExec.class); + // below the exchange (in data node) the aggregation is field optimized. + var shapeExtractPreference = shapeDocValues ? FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS : FieldExtractPreference.NONE; + assertAggregation(agg, "extent", msg, SpatialExtent.class, GEO_SHAPE, shapeExtractPreference); + List boundsAttributes = shapeDocValues ? List.of("city_boundary") : List.of(); + List docValuesAttributes = pointDocValues ? List.of("city_location") : List.of(); + assertThat(fieldExtractExec.boundsAttributes().stream().map(Node::sourceText).toList(), equalTo(boundsAttributes)); + assertThat(fieldExtractExec.docValuesAttributes().stream().map(Node::sourceText).toList(), equalTo(docValuesAttributes)); + } + } } /** @@ -7718,7 +7748,7 @@ private static void assertAggregation( var aggFunc = assertAggregation(plan, aliasName, aggClass); var aggField = as(aggFunc.field(), Attribute.class); var spatialAgg = as(aggFunc, SpatialAggregateFunction.class); - assertThat(spatialAgg.fieldExtractPreference(), equalTo(fieldExtractPreference)); + assertThat(reason, spatialAgg.fieldExtractPreference(), equalTo(fieldExtractPreference)); assertThat(reason, aggField.dataType(), equalTo(fieldType)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index 78512636b57e9..b92fdf816c2fc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -21,6 +21,7 @@ import org.elasticsearch.compute.data.DocBlock; import org.elasticsearch.compute.data.DocVector; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; @@ -32,8 +33,13 @@ import org.elasticsearch.compute.operator.SourceOperator.SourceOperatorFactory; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.utils.GeometryValidator; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.geometry.utils.WellKnownBinary; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.TestBlockFactory; @@ -57,7 +63,9 @@ import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween; import static java.util.stream.Collectors.joining; import static org.apache.lucene.tests.util.LuceneTestCase.createTempDir; +import static org.elasticsearch.compute.aggregation.spatial.SpatialAggregationUtils.encodeLongitude; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES; +import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.NONE; public class TestPhysicalOperationProviders extends AbstractPhysicalOperationProviders { @@ -85,13 +93,7 @@ public PhysicalOperation fieldExtractPhysicalOperation(FieldExtractExec fieldExt PhysicalOperation op = source; for (Attribute attr : fieldExtractExec.attributesToExtract()) { layout.append(attr); - op = op.with( - new TestFieldExtractOperatorFactory( - attr, - PlannerUtils.extractPreference(fieldExtractExec.docValuesAttributes().contains(attr)) - ), - layout.build() - ); + op = op.with(new TestFieldExtractOperatorFactory(attr, fieldExtractExec.fieldExtractPreference(attr)), layout.build()); } return op; } @@ -336,14 +338,13 @@ private Block extractBlockForColumn( DocBlock docBlock = page.getBlock(0); IntVector docIndices = docBlock.asVector().docs(); Block originalData = testData.getBlock(columnIndex); - var blockCopier = shouldMapToDocValues(dataType, extractPreference) - ? TestSpatialPointStatsBlockCopier.create(docIndices, dataType) - : new TestBlockCopier(docIndices); - return blockCopier.copyBlock(originalData); - } - - private boolean shouldMapToDocValues(DataType dataType, MappedFieldType.FieldExtractPreference extractPreference) { - return extractPreference == DOC_VALUES && DataType.isSpatialPoint(dataType); + if (extractPreference == DOC_VALUES && DataType.isSpatialPoint(dataType)) { + return TestSpatialPointStatsBlockCopier.create(docIndices, dataType).copyBlock(originalData); + } else if (extractPreference == EXTRACT_SPATIAL_BOUNDS && DataType.isSpatial(dataType)) { + return TestSpatialShapeExtentBlockCopier.create(docIndices, dataType).copyBlock(originalData); + } else { + return new TestBlockCopier(docIndices).copyBlock(originalData); + } } private static class TestBlockCopier { @@ -371,7 +372,6 @@ protected Block copyBlock(Block originalData) { /** * geo_point and cartesian_point are normally loaded as WKT from source, but for aggregations we can load them as doc-values * which are encoded Long values. This class is used to convert the test loaded WKB into encoded longs for the aggregators. - * TODO: We need a different solution to support geo_shape and cartesian_shape */ private abstract static class TestSpatialPointStatsBlockCopier extends TestBlockCopier { @@ -422,4 +422,94 @@ protected long encode(BytesRef wkb) { }; } } + + /** + * geo_shape and cartesian_shape are normally loaded as WKT from source, but for ST_EXTENT_AGG we can load them from doc-values + * extracting the spatial Extent information. This class is used to convert the test loaded WKB into the int[6] used in the aggregators. + */ + private abstract static class TestSpatialShapeExtentBlockCopier extends TestBlockCopier { + protected final SpatialEnvelopeVisitor.PointVisitor pointVisitor; + private final SpatialEnvelopeVisitor visitor; + + private TestSpatialShapeExtentBlockCopier(IntVector docIndices, SpatialEnvelopeVisitor.PointVisitor pointVisitor) { + super(docIndices); + this.pointVisitor = pointVisitor; + this.visitor = new SpatialEnvelopeVisitor(pointVisitor); + } + + @Override + protected Block copyBlock(Block originalData) { + BytesRef scratch = new BytesRef(100); + BytesRefBlock bytesRefBlock = (BytesRefBlock) originalData; + try (IntBlock.Builder builder = bytesRefBlock.blockFactory().newIntBlockBuilder(docIndices.getPositionCount())) { + for (int c = 0; c < docIndices.getPositionCount(); c++) { + int doc = docIndices.getInt(c); + int count = bytesRefBlock.getValueCount(doc); + int i = bytesRefBlock.getFirstValueIndex(doc); + if (count == 0) { + builder.appendNull(); + } else { + pointVisitor.reset(); + for (int v = 0; v < count; v++) { + BytesRef wkb = bytesRefBlock.getBytesRef(i + v, scratch); + Geometry geometry = WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, wkb.bytes, wkb.offset, wkb.length); + geometry.visit(visitor); + } + encodeExtent(builder); + } + } + return builder.build(); + } + } + + protected abstract void encodeExtent(IntBlock.Builder builder); + + private static TestSpatialShapeExtentBlockCopier create(IntVector docIndices, DataType dataType) { + return switch (dataType) { + case GEO_SHAPE -> new TestGeoCopier(docIndices); + case CARTESIAN_SHAPE -> new TestCartesianCopier(docIndices); + default -> throw new IllegalArgumentException("Unsupported spatial data type: " + dataType); + }; + } + + private static class TestGeoCopier extends TestSpatialShapeExtentBlockCopier { + private TestGeoCopier(IntVector docIndices) { + super(docIndices, new SpatialEnvelopeVisitor.GeoPointVisitor(SpatialEnvelopeVisitor.WrapLongitude.WRAP)); + } + + @Override + protected void encodeExtent(IntBlock.Builder builder) { + // We store the 6 values as a single multi-valued field, in the same order as the fields in the Extent class + // This requires that consumers also know the meaning of the values, which they can learn from the Extent class + SpatialEnvelopeVisitor.GeoPointVisitor visitor = (SpatialEnvelopeVisitor.GeoPointVisitor) pointVisitor; + builder.beginPositionEntry(); + builder.appendInt(CoordinateEncoder.GEO.encodeY(visitor.getMaxY())); + builder.appendInt(CoordinateEncoder.GEO.encodeY(visitor.getMinY())); + builder.appendInt(encodeLongitude(visitor.getMinNegX())); + builder.appendInt(encodeLongitude(visitor.getMaxNegX())); + builder.appendInt(encodeLongitude(visitor.getMinPosX())); + builder.appendInt(encodeLongitude(visitor.getMaxPosX())); + builder.endPositionEntry(); + } + } + + private static class TestCartesianCopier extends TestSpatialShapeExtentBlockCopier { + private TestCartesianCopier(IntVector docIndices) { + super(docIndices, new SpatialEnvelopeVisitor.CartesianPointVisitor()); + } + + @Override + protected void encodeExtent(IntBlock.Builder builder) { + // We store the 4 values as a single multi-valued field, in the same order as the fields in the Rectangle class + // This requires that consumers also know the meaning of the values, which they can learn from the Rectangle class + SpatialEnvelopeVisitor.CartesianPointVisitor visitor = (SpatialEnvelopeVisitor.CartesianPointVisitor) pointVisitor; + builder.beginPositionEntry(); + builder.appendInt(CoordinateEncoder.CARTESIAN.encodeX(visitor.getMinX())); + builder.appendInt(CoordinateEncoder.CARTESIAN.encodeX(visitor.getMaxX())); + builder.appendInt(CoordinateEncoder.CARTESIAN.encodeY(visitor.getMaxY())); + builder.appendInt(CoordinateEncoder.CARTESIAN.encodeY(visitor.getMinY())); + builder.endPositionEntry(); + } + } + } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 67d25556a2aa7..b6f820b69fd0d 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -301,8 +301,7 @@ protected Function, List> getFormatter(String format) { @Override protected boolean isBoundsExtractionSupported() { - // Extracting bounds for geo shapes is not implemented yet. - return false; + return true; } @Override From 5c79f04a3422084909298f1bd995a35b61bf2635 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 9 Jan 2025 18:22:16 +0100 Subject: [PATCH 02/16] Update docs/changelog/119889.yaml --- docs/changelog/119889.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/119889.yaml diff --git a/docs/changelog/119889.yaml b/docs/changelog/119889.yaml new file mode 100644 index 0000000000000..102df3493662f --- /dev/null +++ b/docs/changelog/119889.yaml @@ -0,0 +1,5 @@ +pr: 119889 +summary: Optimize ST_EXTENT_AGG for `geo_shape` and `cartesian_shape` +area: "ES|QL, Geo" +type: enhancement +issues: [] From f7f9403f70305f9af23779020f6e89c9fe856cdc Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 9 Jan 2025 18:24:26 +0100 Subject: [PATCH 03/16] Fix changelog --- docs/changelog/119889.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/119889.yaml b/docs/changelog/119889.yaml index 102df3493662f..e07d8643e379c 100644 --- a/docs/changelog/119889.yaml +++ b/docs/changelog/119889.yaml @@ -1,5 +1,5 @@ pr: 119889 summary: Optimize ST_EXTENT_AGG for `geo_shape` and `cartesian_shape` -area: "ES|QL, Geo" +area: "ES|QL" type: enhancement issues: [] From 898d1ba262984909914d82e27f39243c6d687024 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 10 Jan 2025 11:19:38 +0100 Subject: [PATCH 04/16] Added capability to block mixed-cluster tests which fail --- .../qa/testFixtures/src/main/resources/spatial.csv-spec | 9 +++++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index 868d5c50ec2b0..a52ec1b0dc26f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -572,6 +572,7 @@ BBOX (-0.6067969836294651, 6.621946580708027, 36.69972063973546, 35.620274716056 stExtentGeoShapes required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -584,6 +585,7 @@ BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.476599964313 stExtentGeoPoints required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -596,6 +598,7 @@ BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.694299992173 stExtentGeoShapesAndPoints required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -608,6 +611,7 @@ BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.476599964313 stExtentGeoShapesGrouped required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -625,6 +629,7 @@ L | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.9 stExtentGeoPointsGrouped required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -642,6 +647,7 @@ L | 1 | BBOX (-73.92490002326667, -73.92490002326667, 40.6 stExtentGeoShapesAndPointsGrouped required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -659,6 +665,7 @@ L | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.9 stExtentManyGeoShapesGrouped required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | EVAL prefix = SUBSTRING(abbrev, 1, 1) @@ -676,6 +683,7 @@ B | 61 | BBOX (-116.51340007781982, 153.2021999359131, 60.6 stExtentManyGeoPointsGrouped required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | EVAL prefix = SUBSTRING(abbrev, 1, 1) @@ -693,6 +701,7 @@ B | 61 | BBOX (-116.23080002143979, 153.02809992805123, 60. stExtentManyGeoShapesAndPointsGrouped required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | EVAL prefix = SUBSTRING(abbrev, 1, 1) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 387e48702e708..4aea088478deb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -189,9 +189,12 @@ public enum Cap { */ ST_DISTANCE, - /** Support for function {@code ST_EXTENT}. */ + /** Support for function {@code ST_EXTENT_AGG}. */ ST_EXTENT_AGG, + /** Optimization of ST_EXTENT_AGG with doc-values as IntBlock. */ + ST_EXTENT_AGG_DOCVALUES, + /** * Fix determination of CRS types in spatial functions when folding. */ From a32e45ce9def44e80a54b76754c754e83cd5df0d Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 10 Jan 2025 14:45:52 +0100 Subject: [PATCH 05/16] Revert "Added capability to block mixed-cluster tests which fail" This reverts commit 898d1ba262984909914d82e27f39243c6d687024. --- .../qa/testFixtures/src/main/resources/spatial.csv-spec | 9 --------- .../xpack/esql/action/EsqlCapabilities.java | 5 +---- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index a52ec1b0dc26f..868d5c50ec2b0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -572,7 +572,6 @@ BBOX (-0.6067969836294651, 6.621946580708027, 36.69972063973546, 35.620274716056 stExtentGeoShapes required_capability: st_extent_agg -required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -585,7 +584,6 @@ BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.476599964313 stExtentGeoPoints required_capability: st_extent_agg -required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -598,7 +596,6 @@ BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.694299992173 stExtentGeoShapesAndPoints required_capability: st_extent_agg -required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -611,7 +608,6 @@ BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.476599964313 stExtentGeoShapesGrouped required_capability: st_extent_agg -required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -629,7 +625,6 @@ L | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.9 stExtentGeoPointsGrouped required_capability: st_extent_agg -required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -647,7 +642,6 @@ L | 1 | BBOX (-73.92490002326667, -73.92490002326667, 40.6 stExtentGeoShapesAndPointsGrouped required_capability: st_extent_agg -required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -665,7 +659,6 @@ L | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.9 stExtentManyGeoShapesGrouped required_capability: st_extent_agg -required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | EVAL prefix = SUBSTRING(abbrev, 1, 1) @@ -683,7 +676,6 @@ B | 61 | BBOX (-116.51340007781982, 153.2021999359131, 60.6 stExtentManyGeoPointsGrouped required_capability: st_extent_agg -required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | EVAL prefix = SUBSTRING(abbrev, 1, 1) @@ -701,7 +693,6 @@ B | 61 | BBOX (-116.23080002143979, 153.02809992805123, 60. stExtentManyGeoShapesAndPointsGrouped required_capability: st_extent_agg -required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | EVAL prefix = SUBSTRING(abbrev, 1, 1) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 00890d6fd6456..574c0f176bcf8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -189,12 +189,9 @@ public enum Cap { */ ST_DISTANCE, - /** Support for function {@code ST_EXTENT_AGG}. */ + /** Support for function {@code ST_EXTENT}. */ ST_EXTENT_AGG, - /** Optimization of ST_EXTENT_AGG with doc-values as IntBlock. */ - ST_EXTENT_AGG_DOCVALUES, - /** * Fix determination of CRS types in spatial functions when folding. */ From 26d370866e0700098c07c0a19afd45d74fd3c917 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 10 Jan 2025 15:34:09 +0100 Subject: [PATCH 06/16] Change ST_EXTENT_AGG tests on airport_city_boundary to avoid multi-cluster issues Currently the fact that this index is also used for ENRICH means the test infrastructure duplicates the index across the cluster, which leads to incorrect results in COUNT stats (double the rows). We avoid this by using airports index only if we want to get counts, and ENRICH against airport boundaries to get the geo_shapes. --- .../src/main/resources/spatial.csv-spec | 126 +++++++++++------- 1 file changed, 75 insertions(+), 51 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index 868d5c50ec2b0..b22a68e9c240b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -547,27 +547,45 @@ BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) // end::st_extent_agg-airports-result[] ; -stExtentMultipleGeoPointsNoDocValues +stExtentMultipleGeoPointsCount required_capability: st_extent_agg -FROM airports_no_doc_values | WHERE country == "India" | STATS extent = ST_EXTENT_AGG(location) + +FROM airports +| WHERE country == "India" +| STATS extent = ST_EXTENT_AGG(location), count = COUNT() ; -extent:geo_shape -BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) +extent:geo_shape | count:long +BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) | 50 +; + +stExtentMultipleGeoPointsCountNoDocValues +required_capability: st_extent_agg + +FROM airports_no_doc_values +| WHERE country == "India" +| STATS extent = ST_EXTENT_AGG(location), count = COUNT() +; + +extent:geo_shape | count:long +BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) | 50 ; stExtentMultipleGeoPointGrouping required_capability: st_extent_agg + FROM airports -| STATS extent = ST_EXTENT_AGG(location) BY country -| SORT country -| LIMIT 3 +| STATS extent = ST_EXTENT_AGG(location), count = COUNT() BY country +| SORT count DESC, country ASC +| LIMIT 5 ; -extent:geo_shape | country:keyword -BBOX (69.2100736219436, 69.2100736219436, 34.56339786294848, 34.56339786294848) | Afghanistan -BBOX (19.715032372623682, 19.715032372623682, 41.4208514476195, 41.4208514476195) | Albania -BBOX (-0.6067969836294651, 6.621946580708027, 36.69972063973546, 35.62027471605688) | Algeria +extent:geo_shape | count:long | country:keyword +BBOX (-159.34908430092037, -71.01640669628978, 64.81809809803963, 19.71479767933488) | 129 | United States +BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) | 50 | India +BBOX (-117.19751106575131, -86.87441730871797, 32.833958650007844, 14.791128113865852) | 45 | Mexico +BBOX (76.01301474496722, 130.45620465651155, 46.84301500674337, 18.309095981530845) | 41 | China +BBOX (-135.07621010765433, -52.743333745747805, 63.751152316108346, 43.163360520266) | 37 | Canada ; stExtentGeoShapes @@ -575,11 +593,11 @@ required_capability: st_extent_agg FROM airport_city_boundaries | WHERE region == "City of New York" -| STATS extent = ST_EXTENT_AGG(city_boundary), count = COUNT() +| STATS extent = ST_EXTENT_AGG(city_boundary) ; -extent:geo_shape | count:long -BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | 3 +extent:geo_shape +BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) ; stExtentGeoPoints @@ -587,11 +605,11 @@ required_capability: st_extent_agg FROM airport_city_boundaries | WHERE region == "City of New York" -| STATS extent = ST_EXTENT_AGG(city_location), count = COUNT() +| STATS extent = ST_EXTENT_AGG(city_location) ; -extent:geo_shape | count:long -BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) | 3 +extent:geo_shape +BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) ; stExtentGeoShapesAndPoints @@ -599,11 +617,11 @@ required_capability: st_extent_agg FROM airport_city_boundaries | WHERE region == "City of New York" -| STATS extent_shapes = ST_EXTENT_AGG(city_boundary), extent_points = ST_EXTENT_AGG(city_location), count = COUNT() +| STATS extent_shapes = ST_EXTENT_AGG(city_boundary), extent_points = ST_EXTENT_AGG(city_location) ; -extent_shapes:geo_shape | extent_points:geo_shape | count:long -BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) | 3 +extent_shapes:geo_shape | extent_points:geo_shape +BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) ; stExtentGeoShapesGrouped @@ -612,15 +630,15 @@ required_capability: st_extent_agg FROM airport_city_boundaries | WHERE region == "City of New York" | EVAL prefix = SUBSTRING(abbrev, 1, 1) -| STATS extent = ST_EXTENT_AGG(city_boundary), count = COUNT() BY prefix -| KEEP prefix, count, extent +| STATS extent = ST_EXTENT_AGG(city_boundary) BY prefix +| KEEP prefix, extent | SORT prefix ASC ; -prefix:keyword | count:long | extent:geo_shape -E | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) -J | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) -L | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) +prefix:keyword | extent:geo_shape +E | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) +J | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) +L | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) ; stExtentGeoPointsGrouped @@ -629,15 +647,15 @@ required_capability: st_extent_agg FROM airport_city_boundaries | WHERE region == "City of New York" | EVAL prefix = SUBSTRING(abbrev, 1, 1) -| STATS extent = ST_EXTENT_AGG(city_location), count = COUNT() BY prefix -| KEEP prefix, count, extent +| STATS extent = ST_EXTENT_AGG(city_location) BY prefix +| KEEP prefix, extent | SORT prefix ASC ; -prefix:keyword | count:long | extent:geo_shape -E | 1 | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) -J | 1 | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) -L | 1 | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +prefix:keyword | extent:geo_shape +E | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +J | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +L | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) ; stExtentGeoShapesAndPointsGrouped @@ -646,21 +664,23 @@ required_capability: st_extent_agg FROM airport_city_boundaries | WHERE region == "City of New York" | EVAL prefix = SUBSTRING(abbrev, 1, 1) -| STATS extent_shapes = ST_EXTENT_AGG(city_boundary), extent_points = ST_EXTENT_AGG(city_location), count = COUNT() BY prefix -| KEEP prefix, count, extent_shapes, extent_points +| STATS extent_shapes = ST_EXTENT_AGG(city_boundary), extent_points = ST_EXTENT_AGG(city_location) BY prefix +| KEEP prefix, extent_shapes, extent_points | SORT prefix ASC ; -prefix:keyword | count:long | extent_shapes:geo_shape | extent_points:geo_shape -E | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) -J | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) -L | 1 | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +prefix:keyword | extent_shapes:geo_shape | extent_points:geo_shape +E | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +J | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +L | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) ; stExtentManyGeoShapesGrouped required_capability: st_extent_agg +required_capability: enrich_load -FROM airport_city_boundaries +FROM airports +| ENRICH city_boundaries ON city_location WITH airport, region, city_boundary | EVAL prefix = SUBSTRING(abbrev, 1, 1) | STATS extent = ST_EXTENT_AGG(city_boundary), count = COUNT() BY prefix | KEEP prefix, count, extent @@ -669,15 +689,17 @@ FROM airport_city_boundaries ; prefix:keyword | count:long | extent:geo_shape -S | 70 | BBOX (-136.45440001040697, 178.8686999771744, 61.38089996762574, -33.92440003808588) -C | 64 | BBOX (-107.51820000819862, 172.6055999379605, 55.732699991203845, -43.90400002710521) -B | 61 | BBOX (-116.51340007781982, 153.2021999359131, 60.631899973377585, -41.20620000176132) +S | 77 | BBOX (-136.45440001040697, 178.8686999771744, 61.38089996762574, -33.92440003808588) +C | 75 | BBOX (-107.51820000819862, 172.6055999379605, 55.732699991203845, -43.90400002710521) +B | 69 | BBOX (-116.51340007781982, 153.2021999359131, 60.631899973377585, -41.20620000176132) ; stExtentManyGeoPointsGrouped required_capability: st_extent_agg +required_capability: enrich_load -FROM airport_city_boundaries +FROM airports +| ENRICH city_boundaries ON city_location WITH airport, region, city_boundary | EVAL prefix = SUBSTRING(abbrev, 1, 1) | STATS extent = ST_EXTENT_AGG(city_location), count = COUNT() BY prefix | KEEP prefix, count, extent @@ -686,15 +708,17 @@ FROM airport_city_boundaries ; prefix:keyword | count:long | extent:geo_shape -S | 70 | BBOX (-135.3152000438422, 178.54539999738336, 61.24999999534339, -33.8678000215441) -C | 64 | BBOX (-107.39390007220209, 172.38329996354878, 55.676099974662066, -43.58330002985895) -B | 61 | BBOX (-116.23080002143979, 153.02809992805123, 60.46669999603182, -41.1500000115484) +S | 77 | BBOX (-135.3152000438422, 178.54539999738336, 69.21669997740537, -33.8678000215441) +C | 75 | BBOX (-107.39390007220209, 172.38329996354878, 55.676099974662066, -43.58330002985895) +B | 69 | BBOX (-116.23080002143979, 153.02809992805123, 60.46669999603182, -41.1500000115484) ; stExtentManyGeoShapesAndPointsGrouped required_capability: st_extent_agg +required_capability: enrich_load -FROM airport_city_boundaries +FROM airports +| ENRICH city_boundaries ON city_location WITH airport, region, city_boundary | EVAL prefix = SUBSTRING(abbrev, 1, 1) | STATS extent_shapes = ST_EXTENT_AGG(city_boundary), extent_points = ST_EXTENT_AGG(city_location), count = COUNT() BY prefix | KEEP prefix, count, extent_shapes, extent_points @@ -702,10 +726,10 @@ FROM airport_city_boundaries | LIMIT 3 ; -prefix:keyword | count:long | extent_shapes:geo_shape | extent_points:geo_shape -S | 70 | BBOX (-136.45440001040697, 178.8686999771744, 61.38089996762574, -33.92440003808588) | BBOX (-135.3152000438422, 178.54539999738336, 61.24999999534339, -33.8678000215441) -C | 64 | BBOX (-107.51820000819862, 172.6055999379605, 55.732699991203845, -43.90400002710521) | BBOX (-107.39390007220209, 172.38329996354878, 55.676099974662066, -43.58330002985895) -B | 61 | BBOX (-116.51340007781982, 153.2021999359131, 60.631899973377585, -41.20620000176132) | BBOX (-116.23080002143979, 153.02809992805123, 60.46669999603182, -41.1500000115484) +prefix:keyword | count:long | extent_shapes:geo_shape | extent_points:geo_shape +S | 77 | BBOX (-136.45440001040697, 178.8686999771744, 61.38089996762574, -33.92440003808588) | BBOX (-135.3152000438422, 178.54539999738336, 69.21669997740537, -33.8678000215441) +C | 75 | BBOX (-107.51820000819862, 172.6055999379605, 55.732699991203845, -43.90400002710521) | BBOX (-107.39390007220209, 172.38329996354878, 55.676099974662066, -43.58330002985895) +B | 69 | BBOX (-116.51340007781982, 153.2021999359131, 60.631899973377585, -41.20620000176132) | BBOX (-116.23080002143979, 153.02809992805123, 60.46669999603182, -41.1500000115484) ; ############################################### From ad3f84693d42855326425f92fd039767c0bd32bb Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 10 Jan 2025 15:43:13 +0100 Subject: [PATCH 07/16] Add back non-ENRICH versions to be sure we have multi-shape aggs These versions cannot use COUNT() due to the issue with indexes that are also used in enrich policies being duplicated across multi-cluster IT environments. --- .../src/main/resources/spatial.csv-spec | 58 ++++++++++++++++++- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index b22a68e9c240b..a7948830ff436 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -679,6 +679,60 @@ stExtentManyGeoShapesGrouped required_capability: st_extent_agg required_capability: enrich_load +FROM airport_city_boundaries +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent = ST_EXTENT_AGG(city_boundary) BY prefix +| KEEP prefix, extent +| SORT prefix +| LIMIT 3 +; + +prefix:keyword | extent:geo_shape +A | BBOX (-171.91890003159642, 175.90319998562336, 64.61419996339828, -37.36450002528727) +B | BBOX (-116.51340007781982, 153.2021999359131, 60.631899973377585, -41.20620000176132) +C | BBOX (-107.51820000819862, 172.6055999379605, 55.732699991203845, -43.90400002710521) +; + +stExtentManyGeoPointsGrouped +required_capability: st_extent_agg +required_capability: enrich_load + +FROM airport_city_boundaries +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent = ST_EXTENT_AGG(city_location) BY prefix +| KEEP prefix, extent +| SORT prefix +| LIMIT 3 +; + +prefix:keyword | extent:geo_shape +A | BBOX (-171.75000007264316, 174.73999994806945, 64.54999999143183, -36.84060002211481) +B | BBOX (-116.23080002143979, 153.02809992805123, 60.46669999603182, -41.1500000115484) +C | BBOX (-107.39390007220209, 172.38329996354878, 55.676099974662066, -43.58330002985895) +; + +stExtentManyGeoShapesAndPointsGrouped +required_capability: st_extent_agg +required_capability: enrich_load + +FROM airport_city_boundaries +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent_shapes = ST_EXTENT_AGG(city_boundary), extent_points = ST_EXTENT_AGG(city_location) BY prefix +| KEEP prefix, extent_shapes, extent_points +| SORT prefix +| LIMIT 3 +; + +prefix:keyword | extent_shapes:geo_shape | extent_points:geo_shape +A | BBOX (-171.91890003159642, 175.90319998562336, 64.61419996339828, -37.36450002528727) | BBOX (-171.75000007264316, 174.73999994806945, 64.54999999143183, -36.84060002211481) +B | BBOX (-116.51340007781982, 153.2021999359131, 60.631899973377585, -41.20620000176132) | BBOX (-116.23080002143979, 153.02809992805123, 60.46669999603182, -41.1500000115484) +C | BBOX (-107.51820000819862, 172.6055999379605, 55.732699991203845, -43.90400002710521) | BBOX (-107.39390007220209, 172.38329996354878, 55.676099974662066, -43.58330002985895) +; + +stExtentManyGeoShapesGroupedEnrich +required_capability: st_extent_agg +required_capability: enrich_load + FROM airports | ENRICH city_boundaries ON city_location WITH airport, region, city_boundary | EVAL prefix = SUBSTRING(abbrev, 1, 1) @@ -694,7 +748,7 @@ C | 75 | BBOX (-107.51820000819862, 172.6055999379605, 55.7 B | 69 | BBOX (-116.51340007781982, 153.2021999359131, 60.631899973377585, -41.20620000176132) ; -stExtentManyGeoPointsGrouped +stExtentManyGeoPointsGroupedEnrich required_capability: st_extent_agg required_capability: enrich_load @@ -713,7 +767,7 @@ C | 75 | BBOX (-107.39390007220209, 172.38329996354878, 55. B | 69 | BBOX (-116.23080002143979, 153.02809992805123, 60.46669999603182, -41.1500000115484) ; -stExtentManyGeoShapesAndPointsGrouped +stExtentManyGeoShapesAndPointsGroupedEnrich required_capability: st_extent_agg required_capability: enrich_load From 49e912a199c1a5ff3c9a70f190e99e44f37bf4ba Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 10 Jan 2025 18:08:47 +0100 Subject: [PATCH 08/16] Bring back capability because the mixed-cluster tests failed on the change in meaning Of some int values in the block, particularly those that mean unset or infinity. --- .../src/main/resources/spatial.csv-spec | 19 +++++++++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 ++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index a7948830ff436..96c8a711dd2c8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -524,6 +524,8 @@ POINT (42.97109629958868 14.7552534006536) | 1 stExtentSingleGeoPoint required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + ROW point = TO_GEOPOINT("POINT(42.97109629958868 14.7552534006536)") | STATS extent = ST_EXTENT_AGG(point) ; @@ -534,6 +536,8 @@ BBOX(42.97109629958868, 42.97109629958868, 14.7552534006536, 14.7552534006536) stExtentMultipleGeoPoints required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + // tag::st_extent_agg-airports[] FROM airports | WHERE country == "India" @@ -549,6 +553,7 @@ BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) stExtentMultipleGeoPointsCount required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airports | WHERE country == "India" @@ -561,6 +566,7 @@ BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) | stExtentMultipleGeoPointsCountNoDocValues required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airports_no_doc_values | WHERE country == "India" @@ -573,6 +579,7 @@ BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) | stExtentMultipleGeoPointGrouping required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airports | STATS extent = ST_EXTENT_AGG(location), count = COUNT() BY country @@ -590,6 +597,7 @@ BBOX (-135.07621010765433, -52.743333745747805, 63.751152316108346, 43.163360520 stExtentGeoShapes required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -602,6 +610,7 @@ BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.476599964313 stExtentGeoPoints required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -614,6 +623,7 @@ BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.694299992173 stExtentGeoShapesAndPoints required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -626,6 +636,7 @@ BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.476599964313 stExtentGeoShapesGrouped required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -643,6 +654,7 @@ L | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352 stExtentGeoPointsGrouped required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -660,6 +672,7 @@ L | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391 stExtentGeoShapesAndPointsGrouped required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues FROM airport_city_boundaries | WHERE region == "City of New York" @@ -677,6 +690,7 @@ L | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352 stExtentManyGeoShapesGrouped required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues required_capability: enrich_load FROM airport_city_boundaries @@ -695,6 +709,7 @@ C | BBOX (-107.51820000819862, 172.6055999379605, 55.73269999120384 stExtentManyGeoPointsGrouped required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues required_capability: enrich_load FROM airport_city_boundaries @@ -713,6 +728,7 @@ C | BBOX (-107.39390007220209, 172.38329996354878, 55.6760999746620 stExtentManyGeoShapesAndPointsGrouped required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues required_capability: enrich_load FROM airport_city_boundaries @@ -731,6 +747,7 @@ C | BBOX (-107.51820000819862, 172.6055999379605, 55.73269999120384 stExtentManyGeoShapesGroupedEnrich required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues required_capability: enrich_load FROM airports @@ -750,6 +767,7 @@ B | 69 | BBOX (-116.51340007781982, 153.2021999359131, 60.6 stExtentManyGeoPointsGroupedEnrich required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues required_capability: enrich_load FROM airports @@ -769,6 +787,7 @@ B | 69 | BBOX (-116.23080002143979, 153.02809992805123, 60. stExtentManyGeoShapesAndPointsGroupedEnrich required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues required_capability: enrich_load FROM airports diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 999ada6feba03..377ad51aee96f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -189,9 +189,12 @@ public enum Cap { */ ST_DISTANCE, - /** Support for function {@code ST_EXTENT}. */ + /** Support for function {@code ST_EXTENT_AGG}. */ ST_EXTENT_AGG, + /** Optimization of ST_EXTENT_AGG with doc-values as IntBlock. */ + ST_EXTENT_AGG_DOCVALUES, + /** * Fix determination of CRS types in spatial functions when folding. */ From 2716c031d7604eac32232092b8aac023c50999a6 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 10 Jan 2025 19:07:20 +0100 Subject: [PATCH 09/16] Reorder parameters in geo_point/geo_shape aggregators We had many different parameter orderings, but want to stardize on just two: * For cartesian we use Rectangle class order (minX, maxX, maxY, minY) * For geo we use Extent class order (top, bottom, negLeft, negRight, posLeft, posRight) Since the PR already changed the meaning of the invalid/infinite values of the intermediate state integers, it was already not compatible with the previous cluster versions. We disabled mixed-cluster testing to prevent errors as a result of that. This leaves us the opportunity to make further changes that are mixed-cluster incompatible, hence the decision to perform this consistency update now. --- .../utils/SpatialEnvelopeVisitor.java | 124 ++++++++-------- ...ntGeoPointDocValuesAggregatorFunction.java | 62 ++++---- ...ntDocValuesGroupingAggregatorFunction.java | 52 +++---- ...eoPointSourceValuesAggregatorFunction.java | 62 ++++---- ...ourceValuesGroupingAggregatorFunction.java | 52 +++---- ...ntGeoShapeDocValuesAggregatorFunction.java | 62 ++++---- ...peDocValuesGroupingAggregatorFunction.java | 52 +++---- ...eoShapeSourceValuesAggregatorFunction.java | 62 ++++---- ...ourceValuesGroupingAggregatorFunction.java | 52 +++---- ...tentCartesianPointDocValuesAggregator.java | 3 + ...tCartesianPointSourceValuesAggregator.java | 5 +- ...tentCartesianShapeDocValuesAggregator.java | 7 +- ...tCartesianShapeSourceValuesAggregator.java | 7 +- ...tialExtentGeoPointDocValuesAggregator.java | 16 +- ...lExtentGeoPointSourceValuesAggregator.java | 18 ++- ...tialExtentGeoShapeDocValuesAggregator.java | 20 ++- ...lExtentGeoShapeSourceValuesAggregator.java | 20 ++- ...entGroupingStateWrappedLongitudeState.java | 140 +++++++++--------- ...tialExtentLongitudeWrappingAggregator.java | 28 ++-- ...atialExtentStateWrappedLongitudeState.java | 70 ++++----- .../TestPhysicalOperationProviders.java | 12 +- 21 files changed, 475 insertions(+), 451 deletions(-) diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java index 6e647bfd7f9eb..636a8aafba1b5 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java @@ -126,18 +126,14 @@ public interface PointVisitor { */ public static class CartesianPointVisitor implements PointVisitor { private double minX = Double.POSITIVE_INFINITY; - private double minY = Double.POSITIVE_INFINITY; private double maxX = Double.NEGATIVE_INFINITY; private double maxY = Double.NEGATIVE_INFINITY; + private double minY = Double.POSITIVE_INFINITY; public double getMinX() { return minX; } - public double getMinY() { - return minY; - } - public double getMaxX() { return maxX; } @@ -146,12 +142,16 @@ public double getMaxY() { return maxY; } + public double getMinY() { + return minY; + } + @Override public void visitPoint(double x, double y) { minX = Math.min(minX, x); - minY = Math.min(minY, y); maxX = Math.max(maxX, x); maxY = Math.max(maxY, y); + minY = Math.min(minY, y); } @Override @@ -162,9 +162,9 @@ public void visitRectangle(double minX, double maxX, double maxY, double minY) { ); } this.minX = Math.min(this.minX, minX); - this.minY = Math.min(this.minY, minY); this.maxX = Math.max(this.maxX, maxX); this.maxY = Math.max(this.maxY, maxY); + this.minY = Math.min(this.minY, minY); } @Override @@ -179,10 +179,10 @@ public Rectangle getResult() { @Override public void reset() { - minY = Double.POSITIVE_INFINITY; - maxY = Double.NEGATIVE_INFINITY; minX = Double.POSITIVE_INFINITY; maxX = Double.NEGATIVE_INFINITY; + maxY = Double.NEGATIVE_INFINITY; + minY = Double.POSITIVE_INFINITY; } } @@ -196,12 +196,12 @@ public void reset() { * */ public static class GeoPointVisitor implements PointVisitor { - protected double minY = Double.POSITIVE_INFINITY; - protected double maxY = Double.NEGATIVE_INFINITY; - protected double minNegX = Double.POSITIVE_INFINITY; - protected double maxNegX = Double.NEGATIVE_INFINITY; - protected double minPosX = Double.POSITIVE_INFINITY; - protected double maxPosX = Double.NEGATIVE_INFINITY; + protected double top = Double.NEGATIVE_INFINITY; + protected double bottom = Double.POSITIVE_INFINITY; + protected double negLeft = Double.POSITIVE_INFINITY; + protected double negRight = Double.NEGATIVE_INFINITY; + protected double posLeft = Double.POSITIVE_INFINITY; + protected double posRight = Double.NEGATIVE_INFINITY; private final WrapLongitude wrapLongitude; @@ -209,105 +209,105 @@ public GeoPointVisitor(WrapLongitude wrapLongitude) { this.wrapLongitude = wrapLongitude; } - public double getMinNegX() { - return minNegX; + public double getTop() { + return top; } - public double getMinPosX() { - return minPosX; + public double getBottom() { + return bottom; } - public double getMaxNegX() { - return maxNegX; + public double getNegLeft() { + return negLeft; } - public double getMaxPosX() { - return maxPosX; + public double getNegRight() { + return negRight; } - public double getMaxY() { - return maxY; + public double getPosLeft() { + return posLeft; } - public double getMinY() { - return minY; + public double getPosRight() { + return posRight; } @Override public void visitPoint(double x, double y) { - minY = Math.min(minY, y); - maxY = Math.max(maxY, y); + bottom = Math.min(bottom, y); + top = Math.max(top, y); visitLongitude(x); } @Override public void visitRectangle(double minX, double maxX, double maxY, double minY) { - this.minY = Math.min(this.minY, minY); - this.maxY = Math.max(this.maxY, maxY); + this.bottom = Math.min(this.bottom, minY); + this.top = Math.max(this.top, maxY); visitLongitude(minX); visitLongitude(maxX); } private void visitLongitude(double x) { if (x >= 0) { - minPosX = Math.min(minPosX, x); - maxPosX = Math.max(maxPosX, x); + posLeft = Math.min(posLeft, x); + posRight = Math.max(posRight, x); } else { - minNegX = Math.min(minNegX, x); - maxNegX = Math.max(maxNegX, x); + negLeft = Math.min(negLeft, x); + negRight = Math.max(negRight, x); } } @Override public boolean isValid() { - return minY != Double.POSITIVE_INFINITY; + return bottom != Double.POSITIVE_INFINITY; } @Override public Rectangle getResult() { - return getResult(minNegX, minPosX, maxNegX, maxPosX, maxY, minY, wrapLongitude); + return getResult(top, bottom, negLeft, negRight, posLeft, posRight, wrapLongitude); } @Override public void reset() { - minY = Double.POSITIVE_INFINITY; - maxY = Double.NEGATIVE_INFINITY; - minNegX = Double.POSITIVE_INFINITY; - maxNegX = Double.NEGATIVE_INFINITY; - minPosX = Double.POSITIVE_INFINITY; - maxPosX = Double.NEGATIVE_INFINITY; + bottom = Double.POSITIVE_INFINITY; + top = Double.NEGATIVE_INFINITY; + negLeft = Double.POSITIVE_INFINITY; + negRight = Double.NEGATIVE_INFINITY; + posLeft = Double.POSITIVE_INFINITY; + posRight = Double.NEGATIVE_INFINITY; } public static Rectangle getResult( - double minNegX, - double minPosX, - double maxNegX, - double maxPosX, - double maxY, - double minY, + double top, + double bottom, + double negLeft, + double negRight, + double posLeft, + double posRight, WrapLongitude wrapLongitude ) { - assert Double.isFinite(maxY); + assert Double.isFinite(top); // Due to this data coming through Extent (and aggs that use the same approach), which saves values as integers, - // we must use maxPosX==-Inf for all-neg check, and minNegX==+Inf for all-pos check. - if (Double.isInfinite(maxPosX)) { - return new Rectangle(minNegX, maxNegX, maxY, minY); - } else if (Double.isInfinite(minNegX)) { - return new Rectangle(minPosX, maxPosX, maxY, minY); + // we must use posRight==-Inf for all-neg check, and negLeft==+Inf for all-pos check. + if (Double.isInfinite(posRight)) { + return new Rectangle(negLeft, negRight, top, bottom); + } else if (Double.isInfinite(negLeft)) { + return new Rectangle(posLeft, posRight, top, bottom); } else { return switch (wrapLongitude) { - case NO_WRAP -> new Rectangle(minNegX, maxPosX, maxY, minY); - case WRAP -> maybeWrap(minNegX, minPosX, maxNegX, maxPosX, maxY, minY); + case NO_WRAP -> new Rectangle(negLeft, posRight, top, bottom); + case WRAP -> maybeWrap(top, bottom, negLeft, negRight, posLeft, posRight); }; } } - private static Rectangle maybeWrap(double minNegX, double minPosX, double maxNegX, double maxPosX, double maxY, double minY) { - double unwrappedWidth = maxPosX - minNegX; - double wrappedWidth = 360 + maxNegX - minPosX; + private static Rectangle maybeWrap(double top, double bottom, double negLeft, double negRight, double posLeft, double posRight) { + double unwrappedWidth = posRight - negLeft; + double wrappedWidth = 360 + negRight - posLeft; return unwrappedWidth <= wrappedWidth - ? new Rectangle(minNegX, maxPosX, maxY, minY) - : new Rectangle(minPosX, maxNegX, maxY, minY); + ? new Rectangle(negLeft, posRight, top, bottom) + : new Rectangle(posLeft, negRight, top, bottom); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregatorFunction.java index c883e82d45989..4e76d3dbe0298 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregatorFunction.java @@ -27,12 +27,12 @@ */ public final class SpatialExtentGeoPointDocValuesAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final DriverContext driverContext; @@ -136,43 +136,43 @@ private void addRawBlock(LongBlock block, BooleanVector mask) { public void addIntermediateInput(Page page) { assert channels.size() == intermediateBlockCount(); assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - assert minNegX.getPositionCount() == 1; - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + assert top.getPositionCount() == 1; + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - assert minPosX.getPositionCount() == 1; - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + assert bottom.getPositionCount() == 1; + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - assert maxNegX.getPositionCount() == 1; - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + assert negLeft.getPositionCount() == 1; + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - assert maxPosX.getPositionCount() == 1; - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + assert negRight.getPositionCount() == 1; + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - assert maxY.getPositionCount() == 1; - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + assert posLeft.getPositionCount() == 1; + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minY.getPositionCount() == 1; - SpatialExtentGeoPointDocValuesAggregator.combineIntermediate(state, minNegX.getInt(0), minPosX.getInt(0), maxNegX.getInt(0), maxPosX.getInt(0), maxY.getInt(0), minY.getInt(0)); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert posRight.getPositionCount() == 1; + SpatialExtentGeoPointDocValuesAggregator.combineIntermediate(state, top.getInt(0), bottom.getInt(0), negLeft.getInt(0), negRight.getInt(0), posLeft.getInt(0), posRight.getInt(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesGroupingAggregatorFunction.java index eee5bc5df41a4..9a97a37b22ca1 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesGroupingAggregatorFunction.java @@ -27,12 +27,12 @@ */ public final class SpatialExtentGeoPointDocValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final SpatialExtentGroupingStateWrappedLongitudeState state; @@ -168,40 +168,40 @@ public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { state.enableGroupIdTracking(new SeenGroupIds.Empty()); assert channels.size() == intermediateBlockCount(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minNegX.getPositionCount() == minPosX.getPositionCount() && minNegX.getPositionCount() == maxNegX.getPositionCount() && minNegX.getPositionCount() == maxPosX.getPositionCount() && minNegX.getPositionCount() == maxY.getPositionCount() && minNegX.getPositionCount() == minY.getPositionCount(); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert top.getPositionCount() == bottom.getPositionCount() && top.getPositionCount() == negLeft.getPositionCount() && top.getPositionCount() == negRight.getPositionCount() && top.getPositionCount() == posLeft.getPositionCount() && top.getPositionCount() == posRight.getPositionCount(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentGeoPointDocValuesAggregator.combineIntermediate(state, groupId, minNegX.getInt(groupPosition + positionOffset), minPosX.getInt(groupPosition + positionOffset), maxNegX.getInt(groupPosition + positionOffset), maxPosX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); + SpatialExtentGeoPointDocValuesAggregator.combineIntermediate(state, groupId, top.getInt(groupPosition + positionOffset), bottom.getInt(groupPosition + positionOffset), negLeft.getInt(groupPosition + positionOffset), negRight.getInt(groupPosition + positionOffset), posLeft.getInt(groupPosition + positionOffset), posRight.getInt(groupPosition + positionOffset)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregatorFunction.java index cf65fbdde594c..05bcc79db4f34 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregatorFunction.java @@ -28,12 +28,12 @@ */ public final class SpatialExtentGeoPointSourceValuesAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final DriverContext driverContext; @@ -141,43 +141,43 @@ private void addRawBlock(BytesRefBlock block, BooleanVector mask) { public void addIntermediateInput(Page page) { assert channels.size() == intermediateBlockCount(); assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - assert minNegX.getPositionCount() == 1; - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + assert top.getPositionCount() == 1; + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - assert minPosX.getPositionCount() == 1; - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + assert bottom.getPositionCount() == 1; + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - assert maxNegX.getPositionCount() == 1; - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + assert negLeft.getPositionCount() == 1; + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - assert maxPosX.getPositionCount() == 1; - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + assert negRight.getPositionCount() == 1; + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - assert maxY.getPositionCount() == 1; - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + assert posLeft.getPositionCount() == 1; + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minY.getPositionCount() == 1; - SpatialExtentGeoPointSourceValuesAggregator.combineIntermediate(state, minNegX.getInt(0), minPosX.getInt(0), maxNegX.getInt(0), maxPosX.getInt(0), maxY.getInt(0), minY.getInt(0)); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert posRight.getPositionCount() == 1; + SpatialExtentGeoPointSourceValuesAggregator.combineIntermediate(state, top.getInt(0), bottom.getInt(0), negLeft.getInt(0), negRight.getInt(0), posLeft.getInt(0), posRight.getInt(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesGroupingAggregatorFunction.java index bf8ab2554c7b7..1231e24382887 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesGroupingAggregatorFunction.java @@ -28,12 +28,12 @@ */ public final class SpatialExtentGeoPointSourceValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final SpatialExtentGroupingStateWrappedLongitudeState state; @@ -173,40 +173,40 @@ public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { state.enableGroupIdTracking(new SeenGroupIds.Empty()); assert channels.size() == intermediateBlockCount(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minNegX.getPositionCount() == minPosX.getPositionCount() && minNegX.getPositionCount() == maxNegX.getPositionCount() && minNegX.getPositionCount() == maxPosX.getPositionCount() && minNegX.getPositionCount() == maxY.getPositionCount() && minNegX.getPositionCount() == minY.getPositionCount(); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert top.getPositionCount() == bottom.getPositionCount() && top.getPositionCount() == negLeft.getPositionCount() && top.getPositionCount() == negRight.getPositionCount() && top.getPositionCount() == posLeft.getPositionCount() && top.getPositionCount() == posRight.getPositionCount(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentGeoPointSourceValuesAggregator.combineIntermediate(state, groupId, minNegX.getInt(groupPosition + positionOffset), minPosX.getInt(groupPosition + positionOffset), maxNegX.getInt(groupPosition + positionOffset), maxPosX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); + SpatialExtentGeoPointSourceValuesAggregator.combineIntermediate(state, groupId, top.getInt(groupPosition + positionOffset), bottom.getInt(groupPosition + positionOffset), negLeft.getInt(groupPosition + positionOffset), negRight.getInt(groupPosition + positionOffset), posLeft.getInt(groupPosition + positionOffset), posRight.getInt(groupPosition + positionOffset)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunction.java index 11d301549bf18..fefef6edf6dc7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunction.java @@ -25,12 +25,12 @@ */ public final class SpatialExtentGeoShapeDocValuesAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final DriverContext driverContext; @@ -131,43 +131,43 @@ private void addRawBlock(IntBlock block, BooleanVector mask) { public void addIntermediateInput(Page page) { assert channels.size() == intermediateBlockCount(); assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - assert minNegX.getPositionCount() == 1; - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + assert top.getPositionCount() == 1; + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - assert minPosX.getPositionCount() == 1; - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + assert bottom.getPositionCount() == 1; + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - assert maxNegX.getPositionCount() == 1; - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + assert negLeft.getPositionCount() == 1; + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - assert maxPosX.getPositionCount() == 1; - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + assert negRight.getPositionCount() == 1; + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - assert maxY.getPositionCount() == 1; - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + assert posLeft.getPositionCount() == 1; + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minY.getPositionCount() == 1; - SpatialExtentGeoShapeDocValuesAggregator.combineIntermediate(state, minNegX.getInt(0), minPosX.getInt(0), maxNegX.getInt(0), maxPosX.getInt(0), maxY.getInt(0), minY.getInt(0)); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert posRight.getPositionCount() == 1; + SpatialExtentGeoShapeDocValuesAggregator.combineIntermediate(state, top.getInt(0), bottom.getInt(0), negLeft.getInt(0), negRight.getInt(0), posLeft.getInt(0), posRight.getInt(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction.java index 17300e5ee1a41..7d286eba12ffc 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction.java @@ -25,12 +25,12 @@ */ public final class SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final SpatialExtentGroupingStateWrappedLongitudeState state; @@ -157,40 +157,40 @@ public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { state.enableGroupIdTracking(new SeenGroupIds.Empty()); assert channels.size() == intermediateBlockCount(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minNegX.getPositionCount() == minPosX.getPositionCount() && minNegX.getPositionCount() == maxNegX.getPositionCount() && minNegX.getPositionCount() == maxPosX.getPositionCount() && minNegX.getPositionCount() == maxY.getPositionCount() && minNegX.getPositionCount() == minY.getPositionCount(); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert top.getPositionCount() == bottom.getPositionCount() && top.getPositionCount() == negLeft.getPositionCount() && top.getPositionCount() == negRight.getPositionCount() && top.getPositionCount() == posLeft.getPositionCount() && top.getPositionCount() == posRight.getPositionCount(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentGeoShapeDocValuesAggregator.combineIntermediate(state, groupId, minNegX.getInt(groupPosition + positionOffset), minPosX.getInt(groupPosition + positionOffset), maxNegX.getInt(groupPosition + positionOffset), maxPosX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); + SpatialExtentGeoShapeDocValuesAggregator.combineIntermediate(state, groupId, top.getInt(groupPosition + positionOffset), bottom.getInt(groupPosition + positionOffset), negLeft.getInt(groupPosition + positionOffset), negRight.getInt(groupPosition + positionOffset), posLeft.getInt(groupPosition + positionOffset), posRight.getInt(groupPosition + positionOffset)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunction.java index e23fbcd1b4032..a16f8911d7816 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunction.java @@ -28,12 +28,12 @@ */ public final class SpatialExtentGeoShapeSourceValuesAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final DriverContext driverContext; @@ -141,43 +141,43 @@ private void addRawBlock(BytesRefBlock block, BooleanVector mask) { public void addIntermediateInput(Page page) { assert channels.size() == intermediateBlockCount(); assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - assert minNegX.getPositionCount() == 1; - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + assert top.getPositionCount() == 1; + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - assert minPosX.getPositionCount() == 1; - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + assert bottom.getPositionCount() == 1; + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - assert maxNegX.getPositionCount() == 1; - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + assert negLeft.getPositionCount() == 1; + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - assert maxPosX.getPositionCount() == 1; - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + assert negRight.getPositionCount() == 1; + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - assert maxY.getPositionCount() == 1; - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + assert posLeft.getPositionCount() == 1; + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minY.getPositionCount() == 1; - SpatialExtentGeoShapeSourceValuesAggregator.combineIntermediate(state, minNegX.getInt(0), minPosX.getInt(0), maxNegX.getInt(0), maxPosX.getInt(0), maxY.getInt(0), minY.getInt(0)); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert posRight.getPositionCount() == 1; + SpatialExtentGeoShapeSourceValuesAggregator.combineIntermediate(state, top.getInt(0), bottom.getInt(0), negLeft.getInt(0), negRight.getInt(0), posLeft.getInt(0), posRight.getInt(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction.java index b2b865b9f8a33..8c768496e5905 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction.java @@ -28,12 +28,12 @@ */ public final class SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final SpatialExtentGroupingStateWrappedLongitudeState state; @@ -173,40 +173,40 @@ public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { state.enableGroupIdTracking(new SeenGroupIds.Empty()); assert channels.size() == intermediateBlockCount(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minNegX.getPositionCount() == minPosX.getPositionCount() && minNegX.getPositionCount() == maxNegX.getPositionCount() && minNegX.getPositionCount() == maxPosX.getPositionCount() && minNegX.getPositionCount() == maxY.getPositionCount() && minNegX.getPositionCount() == minY.getPositionCount(); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert top.getPositionCount() == bottom.getPositionCount() && top.getPositionCount() == negLeft.getPositionCount() && top.getPositionCount() == negRight.getPositionCount() && top.getPositionCount() == posLeft.getPositionCount() && top.getPositionCount() == posRight.getPositionCount(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentGeoShapeSourceValuesAggregator.combineIntermediate(state, groupId, minNegX.getInt(groupPosition + positionOffset), minPosX.getInt(groupPosition + positionOffset), maxNegX.getInt(groupPosition + positionOffset), maxPosX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); + SpatialExtentGeoShapeSourceValuesAggregator.combineIntermediate(state, groupId, top.getInt(groupPosition + positionOffset), bottom.getInt(groupPosition + positionOffset), negLeft.getInt(groupPosition + positionOffset), negRight.getInt(groupPosition + positionOffset), posLeft.getInt(groupPosition + positionOffset), posRight.getInt(groupPosition + positionOffset)); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointDocValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointDocValuesAggregator.java index f64949b77707c..3a07754588566 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointDocValuesAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointDocValuesAggregator.java @@ -14,6 +14,9 @@ /** * Computes the extent of a set of cartesian points. It is assumed the points are encoded as longs. * This requires that the planner has planned that points are loaded from the index as doc-values. + * The intermediate state is the extent of the shapes, encoded as four integers: minX, maxX, maxY, minY. + * The order of the integers is the same as defined in the constructor of the Rectangle class. + * Note that this is very different from the six values used for the intermediate state of geo_shape geometries. */ @Aggregator( { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointSourceValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointSourceValuesAggregator.java index 3488af4525dcb..f7a74915f852e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointSourceValuesAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointSourceValuesAggregator.java @@ -15,8 +15,9 @@ /** * Computes the extent of a set of cartesian points. It is assumed that the cartesian points are encoded as WKB BytesRef. * This requires that the planner has NOT planned that points are loaded from the index as doc-values, but from source instead. - * This is also used for final aggregations and aggregations in the coordinator node, - * even if the local node partial aggregation is done with {@link SpatialExtentCartesianPointDocValuesAggregator}. + * The intermediate state is the extent of the shapes, encoded as four integers: minX, maxX, maxY, minY. + * The order of the integers is the same as defined in the constructor of the Rectangle class. + * Note that this is very different from the six values used for the intermediate state of geo_shape geometries. */ @Aggregator( { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregator.java index 2c2c3013a2510..1305139ab2c29 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregator.java @@ -12,8 +12,11 @@ import org.elasticsearch.compute.ann.IntermediateState; /** - * Computes the extent of a set of cartesian shapes. It is assumed that the cartesian shapes are encoded as WKB BytesRef. - * We do not currently support reading shape values or extents from doc values. + * Computes the extent of a set of cartesian shapes read from doc-values, which means they are encoded as an array of integers. + * This requires that the planner has planned that the shape extent is loaded from the index as doc-values. + * The intermediate state is the extent of the shapes, encoded as four integers: minX, maxX, maxY, minY. + * The order of the integers is the same as defined in the constructor of the Rectangle class. + * Note that this is very different from the six values used for the intermediate state of geo_shape geometries. */ @Aggregator( { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregator.java index e90a8fe2bb00a..adcf072fbddd2 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregator.java @@ -13,8 +13,11 @@ import org.elasticsearch.compute.ann.IntermediateState; /** - * Computes the extent of a set of cartesian shapes. It is assumed that the cartesian shapes are encoded as WKB BytesRef. - * We do not currently support reading shape values or extents from doc values. + * Computes the extent of a set of cartesian shapes read from source, which means they are encoded as WKB BytesRef. + * This requires that the planner has NOT planned that shapes are loaded from the index as doc-values, but from source instead. + * The intermediate state is the extent of the shapes, encoded as four integers: minX, maxX, maxY, minY. + * The order of the integers is the same as defined in the constructor of the Rectangle class. + * Note that this is very different from the six values used for the intermediate state of geo_shape geometries. */ @Aggregator( { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregator.java index b9b8bf65e116b..93008d4ee4ff0 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregator.java @@ -14,15 +14,19 @@ /** * Computes the extent of a set of geo points. It is assumed the points are encoded as longs. * This requires that the planner has planned that points are loaded from the index as doc-values. + * The intermediate state is the extent of the shapes, encoded as six integers: top, bottom, negLeft, negRight, posLeft, posRight. + * The order of the integers is the same as defined in the constructor of the Extent class, + * as that is the order in which the values are stored in shape doc-values. + * Note that this is very different from the four values used for the intermediate state of cartesian_shape geometries. */ @Aggregator( { - @IntermediateState(name = "minNegX", type = "INT"), - @IntermediateState(name = "minPosX", type = "INT"), - @IntermediateState(name = "maxNegX", type = "INT"), - @IntermediateState(name = "maxPosX", type = "INT"), - @IntermediateState(name = "maxY", type = "INT"), - @IntermediateState(name = "minY", type = "INT") } + @IntermediateState(name = "top", type = "INT"), + @IntermediateState(name = "bottom", type = "INT"), + @IntermediateState(name = "negLeft", type = "INT"), + @IntermediateState(name = "negRight", type = "INT"), + @IntermediateState(name = "posLeft", type = "INT"), + @IntermediateState(name = "posRight", type = "INT") } ) @GroupingAggregator class SpatialExtentGeoPointDocValuesAggregator extends SpatialExtentLongitudeWrappingAggregator { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregator.java index 36a4e359f23fc..d454b40b1a44f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregator.java @@ -15,17 +15,19 @@ /** * Computes the extent of a set of geo points. It is assumed that the geo points are encoded as WKB BytesRef. * This requires that the planner has NOT planned that points are loaded from the index as doc-values, but from source instead. - * This is also used for final aggregations and aggregations in the coordinator node, - * even if the local node partial aggregation is done with {@link SpatialExtentGeoPointDocValuesAggregator}. + * The intermediate state is the extent of the shapes, encoded as six integers: top, bottom, negLeft, negRight, posLeft, posRight. + * The order of the integers is the same as defined in the constructor of the Extent class, + * as that is the order in which the values are stored in shape doc-values. + * Note that this is very different from the four values used for the intermediate state of cartesian_shape geometries. */ @Aggregator( { - @IntermediateState(name = "minNegX", type = "INT"), - @IntermediateState(name = "minPosX", type = "INT"), - @IntermediateState(name = "maxNegX", type = "INT"), - @IntermediateState(name = "maxPosX", type = "INT"), - @IntermediateState(name = "maxY", type = "INT"), - @IntermediateState(name = "minY", type = "INT") } + @IntermediateState(name = "top", type = "INT"), + @IntermediateState(name = "bottom", type = "INT"), + @IntermediateState(name = "negLeft", type = "INT"), + @IntermediateState(name = "negRight", type = "INT"), + @IntermediateState(name = "posLeft", type = "INT"), + @IntermediateState(name = "posRight", type = "INT") } ) @GroupingAggregator class SpatialExtentGeoPointSourceValuesAggregator extends SpatialExtentLongitudeWrappingAggregator { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregator.java index 14d2d88325209..26f8ae156aacc 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregator.java @@ -12,17 +12,21 @@ import org.elasticsearch.compute.ann.IntermediateState; /** - * Computes the extent of a set of geo shapes. It is assumed that the geo shapes are encoded as WKB BytesRef. - * We do not currently support reading shape values or extents from doc values. + * Computes the extent of a set of geo shapes read from doc-values, which means they are encoded as an array of integers. + * This requires that the planner has planned that the shape extent is loaded from the index as doc-values. + * The intermediate state is the extent of the shapes, encoded as six integers: top, bottom, negLeft, negRight, posLeft, posRight. + * The order of the integers is the same as defined in the constructor of the Extent class, + * as that is the order in which the values are stored in shape doc-values. + * Note that this is very different from the four values used for the intermediate state of cartesian_shape geometries. */ @Aggregator( { - @IntermediateState(name = "minNegX", type = "INT"), - @IntermediateState(name = "minPosX", type = "INT"), - @IntermediateState(name = "maxNegX", type = "INT"), - @IntermediateState(name = "maxPosX", type = "INT"), - @IntermediateState(name = "maxY", type = "INT"), - @IntermediateState(name = "minY", type = "INT") } + @IntermediateState(name = "top", type = "INT"), + @IntermediateState(name = "bottom", type = "INT"), + @IntermediateState(name = "negLeft", type = "INT"), + @IntermediateState(name = "negRight", type = "INT"), + @IntermediateState(name = "posLeft", type = "INT"), + @IntermediateState(name = "posRight", type = "INT") } ) @GroupingAggregator class SpatialExtentGeoShapeDocValuesAggregator extends SpatialExtentLongitudeWrappingAggregator { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregator.java index 2bda4f5ddd9c1..cda0aedfb3ae4 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregator.java @@ -13,17 +13,21 @@ import org.elasticsearch.compute.ann.IntermediateState; /** - * Computes the extent of a set of geo shapes. It is assumed that the geo shapes are encoded as WKB BytesRef. - * We do not currently support reading shape values or extents from doc values. + * Computes the extent of a set of geo shapes read from source, which means they are encoded as WKB BytesRef. + * This requires that the planner has NOT planned that shapes are loaded from the index as doc-values, but from source instead. + * The intermediate state is the extent of the shapes, encoded as six integers: top, bottom, negLeft, negRight, posLeft, posRight. + * The order of the integers is the same as defined in the constructor of the Extent class, + * as that is the order in which the values are stored in shape doc-values. + * Note that this is very different from the four values used for the intermediate state of cartesian_shape geometries. */ @Aggregator( { - @IntermediateState(name = "minNegX", type = "INT"), - @IntermediateState(name = "minPosX", type = "INT"), - @IntermediateState(name = "maxNegX", type = "INT"), - @IntermediateState(name = "maxPosX", type = "INT"), - @IntermediateState(name = "maxY", type = "INT"), - @IntermediateState(name = "minY", type = "INT") } + @IntermediateState(name = "top", type = "INT"), + @IntermediateState(name = "bottom", type = "INT"), + @IntermediateState(name = "negLeft", type = "INT"), + @IntermediateState(name = "negRight", type = "INT"), + @IntermediateState(name = "posLeft", type = "INT"), + @IntermediateState(name = "posRight", type = "INT") } ) @GroupingAggregator class SpatialExtentGeoShapeSourceValuesAggregator extends SpatialExtentLongitudeWrappingAggregator { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java index cdbf82d56efc8..36bb38e4d0d6e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java @@ -28,12 +28,12 @@ final class SpatialExtentGroupingStateWrappedLongitudeState extends AbstractArrayState implements GroupingAggregatorState { // Only geo points support longitude wrapping. private static final PointType POINT_TYPE = PointType.GEO; - private IntArray maxYs; - private IntArray minYs; - private IntArray minNegXs; - private IntArray maxNegXs; - private IntArray minPosXs; - private IntArray maxPosXs; + private IntArray tops; + private IntArray bottoms; + private IntArray negLefts; + private IntArray negRights; + private IntArray posLefts; + private IntArray posRights; private final SpatialEnvelopeVisitor.GeoPointVisitor geoPointVisitor; @@ -43,12 +43,12 @@ final class SpatialExtentGroupingStateWrappedLongitudeState extends AbstractArra SpatialExtentGroupingStateWrappedLongitudeState(BigArrays bigArrays) { super(bigArrays); - this.minNegXs = bigArrays.newIntArray(0, false); - this.minPosXs = bigArrays.newIntArray(0, false); - this.maxNegXs = bigArrays.newIntArray(0, false); - this.maxPosXs = bigArrays.newIntArray(0, false); - this.maxYs = bigArrays.newIntArray(0, false); - this.minYs = bigArrays.newIntArray(0, false); + this.tops = bigArrays.newIntArray(0, false); + this.bottoms = bigArrays.newIntArray(0, false); + this.negLefts = bigArrays.newIntArray(0, false); + this.negRights = bigArrays.newIntArray(0, false); + this.posLefts = bigArrays.newIntArray(0, false); + this.posRights = bigArrays.newIntArray(0, false); enableGroupIdTracking(new SeenGroupIds.Empty()); this.geoPointVisitor = new SpatialEnvelopeVisitor.GeoPointVisitor(SpatialEnvelopeVisitor.WrapLongitude.WRAP); } @@ -57,29 +57,29 @@ final class SpatialExtentGroupingStateWrappedLongitudeState extends AbstractArra public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { assert blocks.length >= offset; try ( - var minNegXsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); - var minPosXsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); - var maxNegXsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); - var maxPosXsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); - var maxYsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); - var minYsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); + var topsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); + var bottomsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); + var negLeftsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); + var negRightsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); + var posLeftsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); + var posRightsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); ) { for (int i = 0; i < selected.getPositionCount(); i++) { int group = selected.getInt(i); assert hasValue(group); - minNegXsBuilder.appendInt(minNegXs.get(group)); - minPosXsBuilder.appendInt(minPosXs.get(group)); - maxNegXsBuilder.appendInt(maxNegXs.get(group)); - maxPosXsBuilder.appendInt(maxPosXs.get(group)); - maxYsBuilder.appendInt(maxYs.get(group)); - minYsBuilder.appendInt(minYs.get(group)); + topsBuilder.appendInt(tops.get(group)); + bottomsBuilder.appendInt(bottoms.get(group)); + negLeftsBuilder.appendInt(negLefts.get(group)); + negRightsBuilder.appendInt(negRights.get(group)); + posLeftsBuilder.appendInt(posLefts.get(group)); + posRightsBuilder.appendInt(posRights.get(group)); } - blocks[offset + 0] = minNegXsBuilder.build(); - blocks[offset + 1] = minPosXsBuilder.build(); - blocks[offset + 2] = maxNegXsBuilder.build(); - blocks[offset + 3] = maxPosXsBuilder.build(); - blocks[offset + 4] = maxYsBuilder.build(); - blocks[offset + 5] = minYsBuilder.build(); + blocks[offset + 0] = topsBuilder.build(); + blocks[offset + 1] = bottomsBuilder.build(); + blocks[offset + 2] = negLeftsBuilder.build(); + blocks[offset + 3] = negRightsBuilder.build(); + blocks[offset + 4] = posLeftsBuilder.build(); + blocks[offset + 5] = posRightsBuilder.build(); } } @@ -89,12 +89,12 @@ public void add(int groupId, Geometry geo) { if (geo.visit(new SpatialEnvelopeVisitor(geoPointVisitor))) { add( groupId, - SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMinNegX()), - SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMinPosX()), - SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMaxNegX()), - SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMaxPosX()), - POINT_TYPE.encoder().encodeY(geoPointVisitor.getMaxY()), - POINT_TYPE.encoder().encodeY(geoPointVisitor.getMinY()) + POINT_TYPE.encoder().encodeY(geoPointVisitor.getTop()), + POINT_TYPE.encoder().encodeY(geoPointVisitor.getBottom()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getNegLeft()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getNegRight()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getPosLeft()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getPosRight()) ); } } @@ -104,12 +104,12 @@ public void add(int groupId, SpatialExtentGroupingStateWrappedLongitudeState inS if (inState.hasValue(inPosition)) { add( groupId, - inState.minNegXs.get(inPosition), - inState.minPosXs.get(inPosition), - inState.maxNegXs.get(inPosition), - inState.maxPosXs.get(inPosition), - inState.maxYs.get(inPosition), - inState.minYs.get(inPosition) + inState.tops.get(inPosition), + inState.bottoms.get(inPosition), + inState.negLefts.get(inPosition), + inState.negRights.get(inPosition), + inState.posLefts.get(inPosition), + inState.posRights.get(inPosition) ); } } @@ -121,7 +121,7 @@ public void add(int groupId, SpatialExtentGroupingStateWrappedLongitudeState inS public void add(int groupId, long encoded) { int x = POINT_TYPE.extractX(encoded); int y = POINT_TYPE.extractY(encoded); - add(groupId, x, x, x, x, y, y); + add(groupId, y, y, x, x, x, x); } /** @@ -137,41 +137,41 @@ public void add(int groupId, int[] values) { int negRight = values[3]; int posLeft = values[4]; int posRight = values[5]; - add(groupId, negLeft, posLeft, negRight, posRight, top, bottom); + add(groupId, top, bottom, negLeft, negRight, posLeft, posRight); } else { throw new IllegalArgumentException("Expected 6 values, got " + values.length); } } - public void add(int groupId, int minNegX, int minPosX, int maxNegX, int maxPosX, int maxY, int minY) { + public void add(int groupId, int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { ensureCapacity(groupId); if (hasValue(groupId)) { - minNegXs.set(groupId, Math.min(minNegXs.get(groupId), minNegX)); - minPosXs.set(groupId, SpatialAggregationUtils.minPos(minPosXs.get(groupId), minPosX)); - maxNegXs.set(groupId, SpatialAggregationUtils.maxNeg(maxNegXs.get(groupId), maxNegX)); - maxPosXs.set(groupId, Math.max(maxPosXs.get(groupId), maxPosX)); - maxYs.set(groupId, Math.max(maxYs.get(groupId), maxY)); - minYs.set(groupId, Math.min(minYs.get(groupId), minY)); + tops.set(groupId, Math.max(tops.get(groupId), top)); + bottoms.set(groupId, Math.min(bottoms.get(groupId), bottom)); + negLefts.set(groupId, Math.min(negLefts.get(groupId), negLeft)); + negRights.set(groupId, SpatialAggregationUtils.maxNeg(negRights.get(groupId), negRight)); + posLefts.set(groupId, SpatialAggregationUtils.minPos(posLefts.get(groupId), posLeft)); + posRights.set(groupId, Math.max(posRights.get(groupId), posRight)); } else { - minNegXs.set(groupId, minNegX); - minPosXs.set(groupId, minPosX); - maxNegXs.set(groupId, maxNegX); - maxPosXs.set(groupId, maxPosX); - maxYs.set(groupId, maxY); - minYs.set(groupId, minY); + tops.set(groupId, top); + bottoms.set(groupId, bottom); + negLefts.set(groupId, negLeft); + negRights.set(groupId, negRight); + posLefts.set(groupId, posLeft); + posRights.set(groupId, posRight); } trackGroupId(groupId); } private void ensureCapacity(int groupId) { long requiredSize = groupId + 1; - if (minNegXs.size() < requiredSize) { - minNegXs = bigArrays.grow(minNegXs, requiredSize); - minPosXs = bigArrays.grow(minPosXs, requiredSize); - maxNegXs = bigArrays.grow(maxNegXs, requiredSize); - maxPosXs = bigArrays.grow(maxPosXs, requiredSize); - minYs = bigArrays.grow(minYs, requiredSize); - maxYs = bigArrays.grow(maxYs, requiredSize); + if (negLefts.size() < requiredSize) { + tops = bigArrays.grow(tops, requiredSize); + bottoms = bigArrays.grow(bottoms, requiredSize); + negLefts = bigArrays.grow(negLefts, requiredSize); + negRights = bigArrays.grow(negRights, requiredSize); + posLefts = bigArrays.grow(posLefts, requiredSize); + posRights = bigArrays.grow(posRights, requiredSize); } } @@ -184,12 +184,12 @@ public Block toBlock(IntVector selected, DriverContext driverContext) { new BytesRef( WellKnownBinary.toWKB( asRectangle( - minNegXs.get(si), - minPosXs.get(si), - maxNegXs.get(si), - maxPosXs.get(si), - maxYs.get(si), - minYs.get(si) + tops.get(si), + bottoms.get(si), + negLefts.get(si), + negRights.get(si), + posLefts.get(si), + posRights.get(si) ), ByteOrder.LITTLE_ENDIAN ) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentLongitudeWrappingAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentLongitudeWrappingAggregator.java index 80ba2d5e45658..2d89ba78d1025 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentLongitudeWrappingAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentLongitudeWrappingAggregator.java @@ -16,27 +16,27 @@ abstract class SpatialExtentLongitudeWrappingAggregator { public static void combineIntermediate( SpatialExtentStateWrappedLongitudeState current, - int minNegX, - int minPosX, - int maxNegX, - int maxPosX, - int maxY, - int minY + int top, + int bottom, + int negLeft, + int negRight, + int posLeft, + int posRight ) { - current.add(minNegX, minPosX, maxNegX, maxPosX, maxY, minY); + current.add(top, bottom, negLeft, negRight, posLeft, posRight); } public static void combineIntermediate( SpatialExtentGroupingStateWrappedLongitudeState current, int groupId, - int minNegX, - int minPosX, - int maxNegX, - int maxPosX, - int maxY, - int minY + int top, + int bottom, + int negLeft, + int negRight, + int posLeft, + int posRight ) { - current.add(groupId, minNegX, minPosX, maxNegX, maxPosX, maxY, minY); + current.add(groupId, top, bottom, negLeft, negRight, posLeft, posRight); } public static Block evaluateFinal(SpatialExtentStateWrappedLongitudeState state, DriverContext driverContext) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java index 043b632f64ee5..8017ad8b75628 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java @@ -26,12 +26,12 @@ final class SpatialExtentStateWrappedLongitudeState implements AggregatorState { // Only geo points support longitude wrapping. private static final PointType POINT_TYPE = PointType.GEO; private boolean seen = false; - private int maxY = Integer.MIN_VALUE; - private int minY = Integer.MAX_VALUE; - private int minNegX = Integer.MAX_VALUE; - private int maxNegX = Integer.MIN_VALUE; - private int minPosX = Integer.MAX_VALUE; - private int maxPosX = Integer.MIN_VALUE; + private int top = Integer.MIN_VALUE; + private int bottom = Integer.MAX_VALUE; + private int negLeft = Integer.MAX_VALUE; + private int negRight = Integer.MIN_VALUE; + private int posLeft = Integer.MAX_VALUE; + private int posRight = Integer.MIN_VALUE; private final SpatialEnvelopeVisitor.GeoPointVisitor geoPointVisitor = new SpatialEnvelopeVisitor.GeoPointVisitor( SpatialEnvelopeVisitor.WrapLongitude.WRAP @@ -44,24 +44,24 @@ public void close() {} public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { assert blocks.length >= offset + 6; var blockFactory = driverContext.blockFactory(); - blocks[offset + 0] = blockFactory.newConstantIntBlockWith(minNegX, 1); - blocks[offset + 1] = blockFactory.newConstantIntBlockWith(minPosX, 1); - blocks[offset + 2] = blockFactory.newConstantIntBlockWith(maxNegX, 1); - blocks[offset + 3] = blockFactory.newConstantIntBlockWith(maxPosX, 1); - blocks[offset + 4] = blockFactory.newConstantIntBlockWith(maxY, 1); - blocks[offset + 5] = blockFactory.newConstantIntBlockWith(minY, 1); + blocks[offset + 0] = blockFactory.newConstantIntBlockWith(top, 1); + blocks[offset + 1] = blockFactory.newConstantIntBlockWith(bottom, 1); + blocks[offset + 2] = blockFactory.newConstantIntBlockWith(negLeft, 1); + blocks[offset + 3] = blockFactory.newConstantIntBlockWith(negRight, 1); + blocks[offset + 4] = blockFactory.newConstantIntBlockWith(posLeft, 1); + blocks[offset + 5] = blockFactory.newConstantIntBlockWith(posRight, 1); } public void add(Geometry geo) { geoPointVisitor.reset(); if (geo.visit(new SpatialEnvelopeVisitor(geoPointVisitor))) { add( - SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMinNegX()), - SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMinPosX()), - SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMaxNegX()), - SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getMaxPosX()), - POINT_TYPE.encoder().encodeY(geoPointVisitor.getMaxY()), - POINT_TYPE.encoder().encodeY(geoPointVisitor.getMinY()) + POINT_TYPE.encoder().encodeY(geoPointVisitor.getTop()), + POINT_TYPE.encoder().encodeY(geoPointVisitor.getBottom()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getNegLeft()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getNegRight()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getPosLeft()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getPosRight()) ); } } @@ -79,20 +79,20 @@ public void add(int[] values) { int negRight = values[3]; int posLeft = values[4]; int posRight = values[5]; - add(negLeft, posLeft, negRight, posRight, top, bottom); + add(top, bottom, negLeft, negRight, posLeft, posRight); } else { throw new IllegalArgumentException("Expected 6 values, got " + values.length); } } - public void add(int minNegX, int minPosX, int maxNegX, int maxPosX, int maxY, int minY) { + public void add(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { seen = true; - this.minNegX = Math.min(this.minNegX, minNegX); - this.minPosX = SpatialAggregationUtils.minPos(this.minPosX, minPosX); - this.maxNegX = SpatialAggregationUtils.maxNeg(this.maxNegX, maxNegX); - this.maxPosX = Math.max(this.maxPosX, maxPosX); - this.maxY = Math.max(this.maxY, maxY); - this.minY = Math.min(this.minY, minY); + this.top = Math.max(this.top, top); + this.bottom = Math.min(this.bottom, bottom); + this.negLeft = Math.min(this.negLeft, negLeft); + this.negRight = SpatialAggregationUtils.maxNeg(this.negRight, negRight); + this.posLeft = SpatialAggregationUtils.minPos(this.posLeft, posLeft); + this.posRight = Math.max(this.posRight, posRight); } /** @@ -102,7 +102,7 @@ public void add(int minNegX, int minPosX, int maxNegX, int maxPosX, int maxY, in public void add(long encoded) { int x = POINT_TYPE.extractX(encoded); int y = POINT_TYPE.extractY(encoded); - add(x, x, x, x, y, y); + add(y, y, x, x, x, x); } public Block toBlock(DriverContext driverContext) { @@ -111,17 +111,17 @@ public Block toBlock(DriverContext driverContext) { } private byte[] toWKB() { - return WellKnownBinary.toWKB(asRectangle(minNegX, minPosX, maxNegX, maxPosX, maxY, minY), ByteOrder.LITTLE_ENDIAN); + return WellKnownBinary.toWKB(asRectangle(top, bottom, negLeft, negRight, posLeft, posRight), ByteOrder.LITTLE_ENDIAN); } - static Rectangle asRectangle(int minNegX, int minPosX, int maxNegX, int maxPosX, int maxY, int minY) { + static Rectangle asRectangle(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { return SpatialEnvelopeVisitor.GeoPointVisitor.getResult( - minNegX <= 0 ? decodeLongitude(minNegX) : Double.POSITIVE_INFINITY, - minPosX >= 0 ? decodeLongitude(minPosX) : Double.POSITIVE_INFINITY, - maxNegX <= 0 ? decodeLongitude(maxNegX) : Double.NEGATIVE_INFINITY, - maxPosX >= 0 ? decodeLongitude(maxPosX) : Double.NEGATIVE_INFINITY, - GeoEncodingUtils.decodeLatitude(maxY), - GeoEncodingUtils.decodeLatitude(minY), + GeoEncodingUtils.decodeLatitude(top), + GeoEncodingUtils.decodeLatitude(bottom), + negLeft <= 0 ? decodeLongitude(negLeft) : Double.POSITIVE_INFINITY, + negRight <= 0 ? decodeLongitude(negRight) : Double.NEGATIVE_INFINITY, + posLeft >= 0 ? decodeLongitude(posLeft) : Double.POSITIVE_INFINITY, + posRight >= 0 ? decodeLongitude(posRight) : Double.NEGATIVE_INFINITY, SpatialEnvelopeVisitor.WrapLongitude.WRAP ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index 21332cc621fb3..b9eab75965164 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -555,12 +555,12 @@ protected void encodeExtent(IntBlock.Builder builder) { // This requires that consumers also know the meaning of the values, which they can learn from the Extent class SpatialEnvelopeVisitor.GeoPointVisitor visitor = (SpatialEnvelopeVisitor.GeoPointVisitor) pointVisitor; builder.beginPositionEntry(); - builder.appendInt(CoordinateEncoder.GEO.encodeY(visitor.getMaxY())); - builder.appendInt(CoordinateEncoder.GEO.encodeY(visitor.getMinY())); - builder.appendInt(encodeLongitude(visitor.getMinNegX())); - builder.appendInt(encodeLongitude(visitor.getMaxNegX())); - builder.appendInt(encodeLongitude(visitor.getMinPosX())); - builder.appendInt(encodeLongitude(visitor.getMaxPosX())); + builder.appendInt(CoordinateEncoder.GEO.encodeY(visitor.getTop())); + builder.appendInt(CoordinateEncoder.GEO.encodeY(visitor.getBottom())); + builder.appendInt(encodeLongitude(visitor.getNegLeft())); + builder.appendInt(encodeLongitude(visitor.getNegRight())); + builder.appendInt(encodeLongitude(visitor.getPosLeft())); + builder.appendInt(encodeLongitude(visitor.getPosRight())); builder.endPositionEntry(); } } From 9c45f722d228d807019aefe8459ab4034ab96147 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Tue, 14 Jan 2025 10:40:44 +0100 Subject: [PATCH 10/16] Code review updates --- .../utils/SpatialEnvelopeVisitor.java | 7 +-- .../compute/gen/AggregatorImplementer.java | 49 +++++++-------- .../gen/GroupingAggregatorImplementer.java | 60 +++++++++---------- ...entGroupingStateWrappedLongitudeState.java | 19 +++--- ...atialExtentStateWrappedLongitudeState.java | 19 +++--- .../function/aggregate/SpatialExtent.java | 4 +- .../local/SpatialShapeBoundsExtraction.java | 6 +- 7 files changed, 75 insertions(+), 89 deletions(-) diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java index 636a8aafba1b5..3060052296329 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java @@ -117,6 +117,7 @@ public interface PointVisitor { Rectangle getResult(); + /** To allow for memory optimizations through object reuse, the visitor can be reset to its initial state. */ void reset(); } @@ -288,11 +289,9 @@ public static Rectangle getResult( WrapLongitude wrapLongitude ) { assert Double.isFinite(top); - // Due to this data coming through Extent (and aggs that use the same approach), which saves values as integers, - // we must use posRight==-Inf for all-neg check, and negLeft==+Inf for all-pos check. - if (Double.isInfinite(posRight)) { + if (posRight == Double.NEGATIVE_INFINITY) { return new Rectangle(negLeft, negRight, top, bottom); - } else if (Double.isInfinite(negLeft)) { + } else if (negLeft == Double.POSITIVE_INFINITY) { return new Rectangle(posLeft, posRight, top, bottom); } else { return switch (wrapLongitude) { diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java index ea64d1660dbe7..4589ab13a4e39 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java @@ -467,17 +467,17 @@ private MethodSpec addRawBlock(boolean masked) { private void combineRawInput(MethodSpec.Builder builder, String blockVariable) { TypeName returnType = TypeName.get(combine.getReturnType()); - startWarningsBlock(builder); - if (valuesIsBytesRef) { - combineRawInputForBytesRef(builder, blockVariable); - } else if (returnType.isPrimitive()) { - combineRawInputForPrimitive(returnType, builder, blockVariable); - } else if (returnType == TypeName.VOID) { - combineRawInputForVoid(builder, blockVariable); - } else { - throw new IllegalArgumentException("combine must return void or a primitive"); - } - endWarningsBlock(builder); + warningsBlock(builder, () -> { + if (valuesIsBytesRef) { + combineRawInputForBytesRef(builder, blockVariable); + } else if (returnType.isPrimitive()) { + combineRawInputForPrimitive(returnType, builder, blockVariable); + } else if (returnType == TypeName.VOID) { + combineRawInputForVoid(builder, blockVariable); + } else { + throw new IllegalArgumentException("combine must return void or a primitive"); + } + }); } private void combineRawInputForPrimitive(TypeName returnType, MethodSpec.Builder builder, String blockVariable) { @@ -492,9 +492,7 @@ private void combineRawInputForPrimitive(TypeName returnType, MethodSpec.Builder } private void combineRawInputForArray(MethodSpec.Builder builder, String arrayVariable) { - startWarningsBlock(builder); - builder.addStatement("$T.combine(state, $L)", declarationType, arrayVariable); - endWarningsBlock(builder); + warningsBlock(builder, () -> builder.addStatement("$T.combine(state, $L)", declarationType, arrayVariable)); } private void combineRawInputForVoid(MethodSpec.Builder builder, String blockVariable) { @@ -511,13 +509,11 @@ private void combineRawInputForBytesRef(MethodSpec.Builder builder, String block builder.addStatement("$T.combine(state, $L.getBytesRef(i, scratch))", declarationType, blockVariable); } - private void startWarningsBlock(MethodSpec.Builder builder) { + private void warningsBlock(MethodSpec.Builder builder, Runnable block) { if (warnExceptions.isEmpty() == false) { builder.beginControlFlow("try"); } - } - - private void endWarningsBlock(MethodSpec.Builder builder) { + block.run(); if (warnExceptions.isEmpty() == false) { String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); @@ -560,12 +556,12 @@ private MethodSpec addIntermediateInput() { builder.nextControlFlow("else if (seen.getBoolean(0))"); } - startWarningsBlock(builder); - var state = intermediateState.get(0); - var s = "state.$L($T.combine(state.$L(), " + state.name() + "." + vectorAccessorName(state.elementType()) + "(0)))"; - builder.addStatement(s, primitiveStateMethod(), declarationType, primitiveStateMethod()); - builder.addStatement("state.seen(true)"); - endWarningsBlock(builder); + warningsBlock(builder, () -> { + var state = intermediateState.get(0); + var s = "state.$L($T.combine(state.$L(), " + state.name() + "." + vectorAccessorName(state.elementType()) + "(0)))"; + builder.addStatement(s, primitiveStateMethod(), declarationType, primitiveStateMethod()); + builder.addStatement("state.seen(true)"); + }); builder.endControlFlow(); } else { throw new IllegalArgumentException("Don't know how to combine intermediate input. Define combineIntermediate"); @@ -731,9 +727,6 @@ private TypeKind valueTypeKind() { private String valueTypeString() { String valueTypeString = TypeName.get(valueTypeMirror()).toString(); - if (valuesIsArray) { - valueTypeString = valueTypeString.substring(0, valueTypeString.length() - 2); - } - return valueTypeString; + return valuesIsArray ? valueTypeString.substring(0, valueTypeString.length() - 2) : valueTypeString; } } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java index eaef32c8ce395..bae8800d3d62f 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java @@ -425,21 +425,21 @@ private void combineRawInput(MethodSpec.Builder builder, String blockVariable, S TypeName valueType = valueTypeName(); TypeName returnType = TypeName.get(combine.getReturnType()); - startWarningsBlock(builder); - if (valuesIsBytesRef) { - combineRawInputForBytesRef(builder, blockVariable, offsetVariable); - } else if (includeTimestampVector) { - combineRawInputWithTimestamp(builder, offsetVariable); - } else if (valueType.isPrimitive() == false) { - throw new IllegalArgumentException("second parameter to combine must be a primitive, array or BytesRef: " + valueType); - } else if (returnType.isPrimitive()) { - combineRawInputForPrimitive(builder, blockVariable, offsetVariable); - } else if (returnType == TypeName.VOID) { - combineRawInputForVoid(builder, blockVariable, offsetVariable); - } else { - throw new IllegalArgumentException("combine must return void or a primitive"); - } - endWarningsBlock(builder); + warningsBlock(builder, () -> { + if (valuesIsBytesRef) { + combineRawInputForBytesRef(builder, blockVariable, offsetVariable); + } else if (includeTimestampVector) { + combineRawInputWithTimestamp(builder, offsetVariable); + } else if (valueType.isPrimitive() == false) { + throw new IllegalArgumentException("second parameter to combine must be a primitive, array or BytesRef: " + valueType); + } else if (returnType.isPrimitive()) { + combineRawInputForPrimitive(builder, blockVariable, offsetVariable); + } else if (returnType == TypeName.VOID) { + combineRawInputForVoid(builder, blockVariable, offsetVariable); + } else { + throw new IllegalArgumentException("combine must return void or a primitive"); + } + }); } private void combineRawInputForPrimitive(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { @@ -453,9 +453,7 @@ private void combineRawInputForPrimitive(MethodSpec.Builder builder, String bloc } private void combineRawInputForArray(MethodSpec.Builder builder, String arrayVariable) { - startWarningsBlock(builder); - builder.addStatement("$T.combine(state, groupId, $L)", declarationType, arrayVariable); - endWarningsBlock(builder); + warningsBlock(builder, () -> builder.addStatement("$T.combine(state, groupId, $L)", declarationType, arrayVariable)); } private void combineRawInputForVoid(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { @@ -488,13 +486,11 @@ private void combineRawInputForBytesRef(MethodSpec.Builder builder, String block builder.addStatement("$T.combine(state, groupId, $L.getBytesRef($L, scratch))", declarationType, blockVariable, offsetVariable); } - private void startWarningsBlock(MethodSpec.Builder builder) { + private void warningsBlock(MethodSpec.Builder builder, Runnable block) { if (warnExceptions.isEmpty() == false) { builder.beginControlFlow("try"); } - } - - private void endWarningsBlock(MethodSpec.Builder builder) { + block.run(); if (warnExceptions.isEmpty() == false) { String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); @@ -559,16 +555,16 @@ private MethodSpec addIntermediateInput() { builder.nextControlFlow("else if (seen.getBoolean(groupPosition + positionOffset))"); } - startWarningsBlock(builder); - var name = intermediateState.get(0).name(); - var vectorAccessor = vectorAccessorName(intermediateState.get(0).elementType()); - builder.addStatement( - "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.$L(groupPosition + positionOffset)))", - declarationType, - name, - vectorAccessor - ); - endWarningsBlock(builder); + warningsBlock(builder, () -> { + var name = intermediateState.get(0).name(); + var vectorAccessor = vectorAccessorName(intermediateState.get(0).elementType()); + builder.addStatement( + "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.$L(groupPosition + positionOffset)))", + declarationType, + name, + vectorAccessor + ); + }); builder.endControlFlow(); } else { builder.addStatement("$T.combineIntermediate(state, groupId, " + intermediateStateRowAccess() + ")", declarationType); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java index 36bb38e4d0d6e..d8056f235626c 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java @@ -129,18 +129,17 @@ public void add(int groupId, long encoded) { * This optimization is enabled when the field has doc-values and is only used in the ST_EXTENT aggregation. */ public void add(int groupId, int[] values) { - if (values.length == 6) { - // Values are stored according to the order defined in the Extent class - int top = values[0]; - int bottom = values[1]; - int negLeft = values[2]; - int negRight = values[3]; - int posLeft = values[4]; - int posRight = values[5]; - add(groupId, top, bottom, negLeft, negRight, posLeft, posRight); - } else { + if (values.length != 6) { throw new IllegalArgumentException("Expected 6 values, got " + values.length); } + // Values are stored according to the order defined in the Extent class + int top = values[0]; + int bottom = values[1]; + int negLeft = values[2]; + int negRight = values[3]; + int posLeft = values[4]; + int posRight = values[5]; + add(groupId, top, bottom, negLeft, negRight, posLeft, posRight); } public void add(int groupId, int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java index 8017ad8b75628..86b41b5b8359c 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java @@ -71,18 +71,17 @@ public void add(Geometry geo) { * This optimization is enabled when the field has doc-values and is only used in the ST_EXTENT aggregation. */ public void add(int[] values) { - if (values.length == 6) { - // Values are stored according to the order defined in the Extent class - int top = values[0]; - int bottom = values[1]; - int negLeft = values[2]; - int negRight = values[3]; - int posLeft = values[4]; - int posRight = values[5]; - add(top, bottom, negLeft, negRight, posLeft, posRight); - } else { + if (values.length != 6) { throw new IllegalArgumentException("Expected 6 values, got " + values.length); } + // Values are stored according to the order defined in the Extent class + int top = values[0]; + int bottom = values[1]; + int negLeft = values[2]; + int negRight = values[3]; + int posLeft = values[4]; + int posRight = values[5]; + add(top, bottom, negLeft, negRight, posLeft, posRight); } public void add(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java index 6326b9522a3cd..5d56fe1e1169a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java @@ -116,12 +116,12 @@ public AggregatorFunctionSupplier supplier(List inputChannels) { case DataType.GEO_SHAPE -> switch (fieldExtractPreference) { case EXTRACT_SPATIAL_BOUNDS -> new SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier(inputChannels); case NONE -> new SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier(inputChannels); - default -> throw new EsqlIllegalArgumentException("Illegal field extract preference: " + fieldExtractPreference); + case DOC_VALUES -> throw new EsqlIllegalArgumentException("Illegal field extract preference: " + fieldExtractPreference); }; case DataType.CARTESIAN_SHAPE -> switch (fieldExtractPreference) { case EXTRACT_SPATIAL_BOUNDS -> new SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier(inputChannels); case NONE -> new SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier(inputChannels); - default -> throw new EsqlIllegalArgumentException("Illegal field extract preference: " + fieldExtractPreference); + case DOC_VALUES -> throw new EsqlIllegalArgumentException("Illegal field extract preference: " + fieldExtractPreference); }; default -> throw EsqlIllegalArgumentException.illegalDataType(type); }; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java index c84b83f15231f..366482738790c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java @@ -61,7 +61,7 @@ protected PhysicalPlan rule(AggregateExec aggregate, LocalPhysicalOptimizerConte }); } - private Set findSpatialShapeBoundsAttributes(AggregateExec aggregate, LocalPhysicalOptimizerContext ctx) { + private static Set findSpatialShapeBoundsAttributes(AggregateExec aggregate, LocalPhysicalOptimizerContext ctx) { var foundAttributes = new HashSet(); aggregate.transformDown(UnaryExec.class, exec -> { switch (exec) { @@ -104,13 +104,13 @@ private Set findSpatialShapeBoundsAttributes(AggregateExec aggregate, return foundAttributes; } - private PhysicalPlan transformFieldExtractExec(FieldExtractExec fieldExtractExec, Set foundAttributes) { + private static PhysicalPlan transformFieldExtractExec(FieldExtractExec fieldExtractExec, Set foundAttributes) { var boundsAttributes = new HashSet<>(foundAttributes); boundsAttributes.retainAll(fieldExtractExec.attributesToExtract()); return fieldExtractExec.withBoundsAttributes(boundsAttributes); } - private PhysicalPlan transformAggregateExec(AggregateExec agg, Set foundAttributes) { + private static PhysicalPlan transformAggregateExec(AggregateExec agg, Set foundAttributes) { return agg.transformExpressionsDown( SpatialExtent.class, spatialExtent -> foundAttributes.contains(spatialExtent.field()) From a05c43569bf5db751304de4069712f42d479268e Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 15 Jan 2025 10:34:57 +0100 Subject: [PATCH 11/16] One more code review comment --- .../rules/physical/local/SpatialShapeBoundsExtraction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java index 366482738790c..eb0d82a59079f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java @@ -50,7 +50,7 @@ public class SpatialShapeBoundsExtraction extends ParameterizedOptimizerRule { @Override protected PhysicalPlan rule(AggregateExec aggregate, LocalPhysicalOptimizerContext ctx) { - var foundAttributes = findSpatialShapeBoundsAttributes(aggregate, ctx); + Set foundAttributes = findSpatialShapeBoundsAttributes(aggregate, ctx); if (foundAttributes.isEmpty()) { return aggregate; } From 8adf8db69e06351ef2968889005f533da1be92e4 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 15 Jan 2025 10:48:34 +0100 Subject: [PATCH 12/16] Two more code review comments --- .../mapper/AbstractShapeGeometryFieldMapper.java | 1 - .../esql/planner/TestPhysicalOperationProviders.java | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java index 57de2e5b497fe..0c4f1072f23c1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java @@ -127,7 +127,6 @@ private void read(BinaryDocValues binaryDocValues, int doc, GeometryDocValueRead reader.reset(binaryDocValues.binaryValue()); var extent = reader.getExtent(); // We store the 6 values as a single multi-valued field, in the same order as the fields in the Extent class - // This requires that consumers also know the meaning of the values, which they can learn from the Extent class builder.beginPositionEntry(); builder.appendInt(extent.top); builder.appendInt(extent.bottom); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index 6049fe1d426e3..c5933f134f9a9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -462,15 +462,15 @@ protected Block copyBlock(Block originalData) { for (int c = 0; c < docIndices.getPositionCount(); c++) { int doc = docIndices.getInt(c); int count = bytesRefBlock.getValueCount(doc); - int i = bytesRefBlock.getFirstValueIndex(doc); if (count == 0) { builder.appendNull(); } else { if (count > 1) { builder.beginPositionEntry(); } - for (int v = 0; v < count; v++) { - builder.appendLong(encode(bytesRefBlock.getBytesRef(i + v, scratch))); + int firstValueIndex = bytesRefBlock.getFirstValueIndex(doc); + for (int i = firstValueIndex; i < firstValueIndex + count; i++) { + builder.appendLong(encode(bytesRefBlock.getBytesRef(i, scratch))); } if (count > 1) { builder.endPositionEntry(); @@ -518,13 +518,13 @@ protected Block copyBlock(Block originalData) { for (int c = 0; c < docIndices.getPositionCount(); c++) { int doc = docIndices.getInt(c); int count = bytesRefBlock.getValueCount(doc); - int i = bytesRefBlock.getFirstValueIndex(doc); if (count == 0) { builder.appendNull(); } else { pointVisitor.reset(); - for (int v = 0; v < count; v++) { - BytesRef wkb = bytesRefBlock.getBytesRef(i + v, scratch); + int firstValueIndex = bytesRefBlock.getFirstValueIndex(doc); + for (int i = firstValueIndex; i < firstValueIndex + count; i++) { + BytesRef wkb = bytesRefBlock.getBytesRef(i, scratch); Geometry geometry = WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, wkb.bytes, wkb.offset, wkb.length); geometry.visit(visitor); } From 7b63b8a38f3643f45d9b6f11270d86dee88183f6 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 15 Jan 2025 11:45:59 +0100 Subject: [PATCH 13/16] Support optimization for cartesian shapes Only saves 4 ints to the IntBlock from extents. This moves the conversion of 6->4 from the aggregator to the blockloader, which is a slight memory efficiency improvement. --- .../mapper/LegacyGeoShapeFieldMapper.java | 13 ++---- .../mapper/AbstractGeometryFieldMapper.java | 6 --- .../AbstractShapeGeometryFieldMapper.java | 45 +++++++------------ ...AbstractShapeGeometryFieldMapperTests.java | 2 +- .../GeoShapeWithDocValuesFieldMapper.java | 15 ++++--- .../index/mapper/ShapeFieldMapper.java | 25 ++++++++--- 6 files changed, 52 insertions(+), 54 deletions(-) diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java index a7a3dccdadd03..d2bda8c4cc81c 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapper; +import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.FieldMapper; @@ -46,7 +47,6 @@ import org.elasticsearch.legacygeo.builders.ShapeBuilder; import org.elasticsearch.legacygeo.parsers.ShapeParser; import org.elasticsearch.legacygeo.query.LegacyGeoShapeQueryProcessor; -import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.locationtech.spatial4j.shape.Point; @@ -534,14 +534,9 @@ public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) { } @Override - protected boolean isBoundsExtractionSupported() { - // Legacy geo-shapes do not support doc-values, so extracting bounds is not possible. - return false; - } - - @Override - protected CoordinateEncoder coordinateEncoder() { - return CoordinateEncoder.GEO; + public BlockLoader blockLoader(BlockLoaderContext blContext) { + // Legacy geo-shapes do not support doc-values, we can only lead from source in ES|QL + return blockLoaderFromSource(blContext); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index c38b5beeb55a0..6e00cc765bd8b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -180,12 +180,6 @@ protected Object parseSourceValue(Object value) { }; } - @Override - public BlockLoader blockLoader(BlockLoaderContext blContext) { - // Currently we can only load from source in ESQL - return blockLoaderFromSource(blContext); - } - protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) { ValueFetcher fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB); // TODO consider optimization using BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java index 0c4f1072f23c1..22b198b10a7ad 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java @@ -12,7 +12,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.Orientation; -import org.elasticsearch.lucene.spatial.CoordinateEncoder; +import org.elasticsearch.lucene.spatial.Extent; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import java.io.IOException; @@ -71,29 +71,27 @@ public Orientation orientation() { @Override protected Object nullValueAsSource(T nullValue) { - // we don't support null value fors shapes + // we don't support null value for shapes return nullValue; } - @Override - public BlockLoader blockLoader(BlockLoaderContext blContext) { - return blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS && isBoundsExtractionSupported() - ? new BoundsBlockLoader(name(), coordinateEncoder()) - : blockLoaderFromSource(blContext); - } - - protected abstract boolean isBoundsExtractionSupported(); - - protected abstract CoordinateEncoder coordinateEncoder(); - - // Visible for testing - static class BoundsBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { + protected static class BoundsBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { private final String fieldName; - private final CoordinateEncoder encoder; - BoundsBlockLoader(String fieldName, CoordinateEncoder encoder) { + protected BoundsBlockLoader(String fieldName) { this.fieldName = fieldName; - this.encoder = encoder; + } + + protected void writeExtent(BlockLoader.IntBuilder builder, Extent extent) { + // We store the 6 values as a single multi-valued field, in the same order as the fields in the Extent class + builder.beginPositionEntry(); + builder.appendInt(extent.top); + builder.appendInt(extent.bottom); + builder.appendInt(extent.negLeft); + builder.appendInt(extent.negRight); + builder.appendInt(extent.posLeft); + builder.appendInt(extent.posRight); + builder.endPositionEntry(); } @Override @@ -125,16 +123,7 @@ private void read(BinaryDocValues binaryDocValues, int doc, GeometryDocValueRead return; } reader.reset(binaryDocValues.binaryValue()); - var extent = reader.getExtent(); - // We store the 6 values as a single multi-valued field, in the same order as the fields in the Extent class - builder.beginPositionEntry(); - builder.appendInt(extent.top); - builder.appendInt(extent.bottom); - builder.appendInt(extent.negLeft); - builder.appendInt(extent.negRight); - builder.appendInt(extent.posLeft); - builder.appendInt(extent.posRight); - builder.endPositionEntry(); + writeExtent(builder, reader.getExtent()); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java index 130c10130c4f3..6470f85bcf539 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java @@ -63,7 +63,7 @@ private static void testBoundsBlockLoaderAux( Function> visitor ) throws IOException { var geometries = IntStream.range(0, 50).mapToObj(i -> generator.get()).toList(); - var loader = new AbstractShapeGeometryFieldMapper.AbstractShapeGeometryFieldType.BoundsBlockLoader("field", encoder); + var loader = new AbstractShapeGeometryFieldMapper.AbstractShapeGeometryFieldType.BoundsBlockLoader("field"); try (Directory directory = newDirectory()) { try (var iw = new RandomIndexWriter(random(), directory)) { for (Geometry geometry : geometries) { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index b6f820b69fd0d..f7c5f1b8072f3 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapper; +import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.GeoShapeIndexer; @@ -300,13 +301,17 @@ protected Function, List> getFormatter(String format) { } @Override - protected boolean isBoundsExtractionSupported() { - return true; + public BlockLoader blockLoader(BlockLoaderContext blContext) { + return blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS + ? new GeoBoundsBlockLoader(name()) + : blockLoaderFromSource(blContext); } - @Override - protected CoordinateEncoder coordinateEncoder() { - return CoordinateEncoder.GEO; + static class GeoBoundsBlockLoader extends AbstractShapeGeometryFieldMapper.AbstractShapeGeometryFieldType.BoundsBlockLoader { + + GeoBoundsBlockLoader(String fieldName) { + super(fieldName); + } } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java index f1140093f2368..198e0ba3011bf 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java @@ -22,6 +22,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapper; +import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -31,6 +32,7 @@ import org.elasticsearch.lucene.spatial.BinaryShapeDocValuesField; import org.elasticsearch.lucene.spatial.CartesianShapeIndexer; import org.elasticsearch.lucene.spatial.CoordinateEncoder; +import org.elasticsearch.lucene.spatial.Extent; import org.elasticsearch.lucene.spatial.XYQueriesUtils; import org.elasticsearch.script.field.AbstractScriptFieldFactory; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; @@ -186,13 +188,26 @@ protected Function, List> getFormatter(String format) { } @Override - protected boolean isBoundsExtractionSupported() { - return true; + public BlockLoader blockLoader(BlockLoaderContext blContext) { + return blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS + ? new CartesianBoundsBlockLoader(name()) + : blockLoaderFromSource(blContext); } - @Override - protected CoordinateEncoder coordinateEncoder() { - return CoordinateEncoder.CARTESIAN; + static class CartesianBoundsBlockLoader extends BoundsBlockLoader { + protected CartesianBoundsBlockLoader(String fieldName) { + super(fieldName); + } + + protected void writeExtent(BlockLoader.IntBuilder builder, Extent extent) { + // For cartesian_shape we store 4 values as a multi-valued field, in the same order as the fields in the Rectangle class + builder.beginPositionEntry(); + builder.appendInt(Math.min(extent.negLeft, extent.posLeft)); + builder.appendInt(Math.max(extent.negRight, extent.posRight)); + builder.appendInt(extent.top); + builder.appendInt(extent.bottom); + builder.endPositionEntry(); + } } } From 9fb92b1738db31faeef5efd6a69dee117abbf81d Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 15 Jan 2025 18:38:53 +0100 Subject: [PATCH 14/16] Updated SpatialGeometryFieldMapperTests and identified bug We left the bug as a TODO for the next PR, so at least the primary optimization can get merged for tech-preview in 8.18. --- .../utils/SpatialEnvelopeVisitor.java | 1 + ...AbstractShapeGeometryFieldMapperTests.java | 114 ---------- .../mapper/ShapeGeometryFieldMapperTests.java | 201 ++++++++++++++++++ 3 files changed, 202 insertions(+), 114 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/ShapeGeometryFieldMapperTests.java diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java index 3060052296329..f00db4f1e6601 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java @@ -243,6 +243,7 @@ public void visitPoint(double x, double y) { @Override public void visitRectangle(double minX, double maxX, double maxY, double minY) { + // TODO: Fix bug with rectangle crossing the dateline (see Extent.addRectangle for correct behaviour) this.bottom = Math.min(this.bottom, minY); this.top = Math.max(this.top, maxY); visitLongitude(minX); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java deleted file mode 100644 index 6470f85bcf539..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.mapper; - -import org.apache.lucene.document.Document; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.store.Directory; -import org.apache.lucene.tests.index.RandomIndexWriter; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.geo.Orientation; -import org.elasticsearch.core.Strings; -import org.elasticsearch.geo.GeometryTestUtils; -import org.elasticsearch.geo.ShapeTestUtils; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; -import org.elasticsearch.lucene.spatial.BinaryShapeDocValuesField; -import org.elasticsearch.lucene.spatial.CartesianShapeIndexer; -import org.elasticsearch.lucene.spatial.CoordinateEncoder; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.hamcrest.RectangleMatcher; -import org.elasticsearch.test.hamcrest.WellKnownBinaryBytesRefMatcher; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Optional; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.IntStream; - -public class AbstractShapeGeometryFieldMapperTests extends ESTestCase { - public void testCartesianBoundsBlockLoader() throws IOException { - testBoundsBlockLoaderAux( - CoordinateEncoder.CARTESIAN, - () -> ShapeTestUtils.randomGeometryWithoutCircle(0, false), - CartesianShapeIndexer::new, - SpatialEnvelopeVisitor::visitCartesian - ); - } - - // TODO when we turn this optimization on for geo, this test should pass. - public void ignoreTestGeoBoundsBlockLoader() throws IOException { - testBoundsBlockLoaderAux( - CoordinateEncoder.GEO, - () -> GeometryTestUtils.randomGeometryWithoutCircle(0, false), - field -> new GeoShapeIndexer(Orientation.RIGHT, field), - g -> SpatialEnvelopeVisitor.visitGeo(g, SpatialEnvelopeVisitor.WrapLongitude.WRAP) - ); - } - - private static void testBoundsBlockLoaderAux( - CoordinateEncoder encoder, - Supplier generator, - Function indexerFactory, - Function> visitor - ) throws IOException { - var geometries = IntStream.range(0, 50).mapToObj(i -> generator.get()).toList(); - var loader = new AbstractShapeGeometryFieldMapper.AbstractShapeGeometryFieldType.BoundsBlockLoader("field"); - try (Directory directory = newDirectory()) { - try (var iw = new RandomIndexWriter(random(), directory)) { - for (Geometry geometry : geometries) { - var shape = new BinaryShapeDocValuesField("field", encoder); - shape.add(indexerFactory.apply("field").indexShape(geometry), geometry); - var doc = new Document(); - doc.add(shape); - iw.addDocument(doc); - } - } - - var expected = new ArrayList(); - var byteRefResults = new ArrayList(); - int currentIndex = 0; - try (DirectoryReader reader = DirectoryReader.open(directory)) { - for (var leaf : reader.leaves()) { - LeafReader leafReader = leaf.reader(); - int numDocs = leafReader.numDocs(); - // We specifically check just the even indices, to verify the loader can skip documents correctly. - int[] array = evenArray(numDocs); - for (int i = 0; i < array.length; i += 1) { - expected.add(visitor.apply(geometries.get(array[i] + currentIndex)).get()); - } - try (var block = (TestBlock) loader.reader(leaf).read(TestBlock.factory(leafReader.numDocs()), TestBlock.docs(array))) { - for (int i = 0; i < block.size(); i++) { - byteRefResults.add((BytesRef) block.get(i)); - } - } - currentIndex += numDocs; - } - } - - for (int i = 0; i < expected.size(); i++) { - Rectangle rectangle = expected.get(i); - var geoString = rectangle.toString(); - assertThat( - Strings.format("geometry '%s' wasn't extracted correctly", geoString), - byteRefResults.get(i), - WellKnownBinaryBytesRefMatcher.encodes(RectangleMatcher.closeToFloat(rectangle, 1e-3, encoder)) - ); - } - } - } - - private static int[] evenArray(int maxIndex) { - return IntStream.range(0, maxIndex / 2).map(x -> x * 2).toArray(); - } -} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ShapeGeometryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ShapeGeometryFieldMapperTests.java new file mode 100644 index 0000000000000..0322286277b25 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/ShapeGeometryFieldMapperTests.java @@ -0,0 +1,201 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.Document; +import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.common.geo.GeometryNormalizer; +import org.elasticsearch.core.Strings; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geo.ShapeTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.lucene.spatial.BinaryShapeDocValuesField; +import org.elasticsearch.lucene.spatial.CartesianShapeIndexer; +import org.elasticsearch.lucene.spatial.CoordinateEncoder; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.hamcrest.RectangleMatcher; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.IntStream; + +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude; +import static org.elasticsearch.common.geo.Orientation.RIGHT; + +public class ShapeGeometryFieldMapperTests extends ESTestCase { + public void testCartesianBoundsBlockLoader() throws IOException { + testBoundsBlockLoader( + CoordinateEncoder.CARTESIAN, + () -> ShapeTestUtils.randomGeometryWithoutCircle(0, false), + CartesianShapeIndexer::new, + SpatialEnvelopeVisitor::visitCartesian, + ShapeGeometryFieldMapperTests::makeCartesianRectangle + ); + } + + // TODO: Re-enable this test after fixing the bug in the ShapeEnvelopeVisitor regarding Rectangle crossing the dateline + // Currently it is flaky if the geometries include a Rectangle like one defined in the test below + public void ignoreTestGeoBoundsBlockLoader() throws IOException { + testBoundsBlockLoader( + CoordinateEncoder.GEO, + () -> normalize(GeometryTestUtils.randomGeometryWithoutCircle(0, false)), + field -> new GeoShapeIndexer(RIGHT, field), + g -> SpatialEnvelopeVisitor.visitGeo(g, SpatialEnvelopeVisitor.WrapLongitude.WRAP), + ShapeGeometryFieldMapperTests::makeGeoRectangle + ); + } + + // TODO: Re-enable this test after fixing the bug in the SpatialEnvelopeVisitor regarding Rectangle crossing the dateline + // See the difference between GeoShapeIndexer.visitRectangle() and SpatialEnvelopeVisitor.GeoPointVisitor.visitRectangle() + public void ignoreTestRectangleCrossingDateline() throws IOException { + var geometries = new ArrayList(); + geometries.add(new Rectangle(180, 51.62247094594227, -18.5, -24.902304006345503)); + testBoundsBlockLoaderAux( + CoordinateEncoder.GEO, + geometries, + field -> new GeoShapeIndexer(RIGHT, field), + g -> SpatialEnvelopeVisitor.visitGeo(g, SpatialEnvelopeVisitor.WrapLongitude.WRAP), + ShapeGeometryFieldMapperTests::makeGeoRectangle + ); + } + + private Geometry normalize(Geometry geometry) { + return GeometryNormalizer.needsNormalize(RIGHT, geometry) ? GeometryNormalizer.apply(RIGHT, geometry) : geometry; + } + + private static void testBoundsBlockLoader( + CoordinateEncoder encoder, + Supplier generator, + Function indexerFactory, + Function> visitor, + BiFunction rectangleMaker + ) throws IOException { + var geometries = IntStream.range(0, 50).mapToObj(i -> generator.get()).toList(); + testBoundsBlockLoaderAux(encoder, geometries, indexerFactory, visitor, rectangleMaker); + } + + private static void testBoundsBlockLoaderAux( + CoordinateEncoder encoder, + java.util.List geometries, + Function indexerFactory, + Function> visitor, + BiFunction rectangleMaker + ) throws IOException { + var loader = new AbstractShapeGeometryFieldMapper.AbstractShapeGeometryFieldType.BoundsBlockLoader("field"); + try (Directory directory = newDirectory()) { + try (var iw = new RandomIndexWriter(random(), directory)) { + for (Geometry geometry : geometries) { + var shape = new BinaryShapeDocValuesField("field", encoder); + shape.add(indexerFactory.apply("field").indexShape(geometry), geometry); + var doc = new Document(); + doc.add(shape); + iw.addDocument(doc); + } + } + + var expected = new ArrayList(); + ArrayList intArrayResults = new ArrayList<>(); + int currentIndex = 0; + try (DirectoryReader reader = DirectoryReader.open(directory)) { + for (var leaf : reader.leaves()) { + LeafReader leafReader = leaf.reader(); + int numDocs = leafReader.numDocs(); + // We specifically check just the even indices, to verify the loader can skip documents correctly. + int[] array = evenArray(numDocs); + for (int j : array) { + expected.add(visitor.apply(geometries.get(j + currentIndex)).get()); + } + try (var block = (TestBlock) loader.reader(leaf).read(TestBlock.factory(leafReader.numDocs()), TestBlock.docs(array))) { + for (int i = 0; i < block.size(); i++) { + intArrayResults.add(block.get(i)); + } + } + currentIndex += numDocs; + } + } + + for (int i = 0; i < expected.size(); i++) { + Rectangle rectangle = expected.get(i); + var geoString = rectangle.toString(); + Rectangle result = rectangleMaker.apply(encoder, intArrayResults.get(i)); + assertThat( + Strings.format("geometry[%d] '%s' wasn't extracted correctly", i, geoString), + result, + RectangleMatcher.closeToFloat(rectangle, 1e-3, encoder) + ); + } + } + } + + private static Rectangle makeCartesianRectangle(CoordinateEncoder encoder, Object integers) { + if (integers instanceof ArrayList list) { + int[] ints = list.stream().mapToInt(x -> (int) x).toArray(); + if (list.size() == 6) { + // Data in order defined by Extent class + double top = encoder.decodeY(ints[0]); + double bottom = encoder.decodeY(ints[1]); + double negLeft = encoder.decodeX(ints[2]); + double negRight = encoder.decodeX(ints[3]); + double posLeft = encoder.decodeX(ints[4]); + double posRight = encoder.decodeX(ints[5]); + return new Rectangle(Math.min(negLeft, posLeft), Math.max(negRight, posRight), top, bottom); + } else if (list.size() == 4) { + // Data in order defined by Rectangle class + return new Rectangle( + encoder.decodeX(ints[0]), + encoder.decodeX(ints[1]), + encoder.decodeY(ints[2]), + encoder.decodeY(ints[3]) + ); + } else { + throw new IllegalArgumentException("Expected 4 or 6 integers"); + } + } + throw new IllegalArgumentException("Expected an array of integers"); + } + + private static Rectangle makeGeoRectangle(CoordinateEncoder encoder, Object integers) { + if (integers instanceof ArrayList list) { + int[] ints = list.stream().mapToInt(x -> (int) x).toArray(); + if (list.size() != 6) { + throw new IllegalArgumentException("Expected 6 integers"); + } + // Data in order defined by Extent class + return asGeoRectangle(ints[0], ints[1], ints[2], ints[3], ints[4], ints[5]); + } + throw new IllegalArgumentException("Expected an array of integers"); + } + + private static Rectangle asGeoRectangle(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { + return SpatialEnvelopeVisitor.GeoPointVisitor.getResult( + GeoEncodingUtils.decodeLatitude(top), + GeoEncodingUtils.decodeLatitude(bottom), + negLeft <= 0 ? decodeLongitude(negLeft) : Double.POSITIVE_INFINITY, + negRight <= 0 ? decodeLongitude(negRight) : Double.NEGATIVE_INFINITY, + posLeft >= 0 ? decodeLongitude(posLeft) : Double.POSITIVE_INFINITY, + posRight >= 0 ? decodeLongitude(posRight) : Double.NEGATIVE_INFINITY, + SpatialEnvelopeVisitor.WrapLongitude.WRAP + ); + } + + private static int[] evenArray(int maxIndex) { + return IntStream.range(0, maxIndex / 2).map(x -> x * 2).toArray(); + } +} From 33322241d0f4e89eb8172f8e6a0cde833aebc3ee Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 16 Jan 2025 15:16:54 +0100 Subject: [PATCH 15/16] Support missing groupIds in intermediate state creation --- .../spatial/SpatialExtentGroupingState.java | 17 ++++++++++---- ...entGroupingStateWrappedLongitudeState.java | 23 +++++++++++++------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java index c9ebc23773e0f..9fb548dceaad9 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java @@ -54,11 +54,18 @@ public void toIntermediate(Block[] blocks, int offset, IntVector selected, Drive ) { for (int i = 0; i < selected.getPositionCount(); i++) { int group = selected.getInt(i); - assert hasValue(group); - minXsBuilder.appendInt(minXs.get(group)); - maxXsBuilder.appendInt(maxXs.get(group)); - maxYsBuilder.appendInt(maxYs.get(group)); - minYsBuilder.appendInt(minYs.get(group)); + if (hasValue(group)) { + minXsBuilder.appendInt(minXs.get(group)); + maxXsBuilder.appendInt(maxXs.get(group)); + maxYsBuilder.appendInt(maxYs.get(group)); + minYsBuilder.appendInt(minYs.get(group)); + } else { + // TODO: Should we add Nulls here instead? + minXsBuilder.appendInt(Integer.MAX_VALUE); + maxXsBuilder.appendInt(Integer.MIN_VALUE); + maxYsBuilder.appendInt(Integer.MIN_VALUE); + minYsBuilder.appendInt(Integer.MAX_VALUE); + } } blocks[offset + 0] = minXsBuilder.build(); blocks[offset + 1] = maxXsBuilder.build(); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java index d8056f235626c..9f8fca5236d14 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java @@ -66,13 +66,22 @@ public void toIntermediate(Block[] blocks, int offset, IntVector selected, Drive ) { for (int i = 0; i < selected.getPositionCount(); i++) { int group = selected.getInt(i); - assert hasValue(group); - topsBuilder.appendInt(tops.get(group)); - bottomsBuilder.appendInt(bottoms.get(group)); - negLeftsBuilder.appendInt(negLefts.get(group)); - negRightsBuilder.appendInt(negRights.get(group)); - posLeftsBuilder.appendInt(posLefts.get(group)); - posRightsBuilder.appendInt(posRights.get(group)); + if (hasValue(group)) { + topsBuilder.appendInt(tops.get(group)); + bottomsBuilder.appendInt(bottoms.get(group)); + negLeftsBuilder.appendInt(negLefts.get(group)); + negRightsBuilder.appendInt(negRights.get(group)); + posLeftsBuilder.appendInt(posLefts.get(group)); + posRightsBuilder.appendInt(posRights.get(group)); + } else { + // TODO: Should we add Nulls here instead? + topsBuilder.appendInt(Integer.MIN_VALUE); + bottomsBuilder.appendInt(Integer.MAX_VALUE); + negLeftsBuilder.appendInt(Integer.MAX_VALUE); + negRightsBuilder.appendInt(Integer.MIN_VALUE); + posLeftsBuilder.appendInt(Integer.MAX_VALUE); + posRightsBuilder.appendInt(Integer.MIN_VALUE); + } } blocks[offset + 0] = topsBuilder.build(); blocks[offset + 1] = bottomsBuilder.build(); From f0b7ab1de347b28b64e3a13a88e336f6501c9912 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 16 Jan 2025 18:13:38 +0100 Subject: [PATCH 16/16] Added more cartesian tests with larger result sets This is to increase the chance that the same failure seen for geo_shape in serverless would also occur with cartesian_shape in serverless. --- .../src/main/resources/spatial.csv-spec | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index 96c8a711dd2c8..8718112979ce6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -691,7 +691,6 @@ L | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352 stExtentManyGeoShapesGrouped required_capability: st_extent_agg required_capability: st_extent_agg_docvalues -required_capability: enrich_load FROM airport_city_boundaries | EVAL prefix = SUBSTRING(abbrev, 1, 1) @@ -710,7 +709,6 @@ C | BBOX (-107.51820000819862, 172.6055999379605, 55.73269999120384 stExtentManyGeoPointsGrouped required_capability: st_extent_agg required_capability: st_extent_agg_docvalues -required_capability: enrich_load FROM airport_city_boundaries | EVAL prefix = SUBSTRING(abbrev, 1, 1) @@ -729,7 +727,6 @@ C | BBOX (-107.39390007220209, 172.38329996354878, 55.6760999746620 stExtentManyGeoShapesAndPointsGrouped required_capability: st_extent_agg required_capability: st_extent_agg_docvalues -required_capability: enrich_load FROM airport_city_boundaries | EVAL prefix = SUBSTRING(abbrev, 1, 1) @@ -2006,6 +2003,18 @@ extent:cartesian_shape BBOX (4783520.5, 1.6168486E7, 8704352.0, -584415.9375) ; +stExtentMultipleCartesianPointsCount +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM airports_web +| STATS extent = ST_EXTENT_AGG(location), count = COUNT() +; + +extent:cartesian_shape | count:long +BBOX (-1.949601E7, 1.9947946E7, 1.4502138E7, -7128878.5) | 849 +; + stExtentMultipleCartesianPointGrouping required_capability: st_extent_agg FROM airports_web | STATS extent = ST_EXTENT_AGG(location) BY scalerank | SORT scalerank DESC | LIMIT 3 @@ -2067,6 +2076,42 @@ count:long | key:keyword | extent:cartesian_shape 4 | Fou | BBOX (0.0, 3.0, 3.0, 0.0) ; +stExtentManyCartesianShapesGrouped +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM countries_bbox_web +| EVAL prefix = SUBSTRING(id, 1, 1) +| STATS extent = ST_EXTENT_AGG(shape) BY prefix +| KEEP prefix, extent +| SORT prefix +| LIMIT 3 +; + +prefix:keyword | extent:cartesian_shape +A | BBOX (-2.0037508E7, 2.0037508E7, 6278042.5, -4.748140544E9) +B | BBOX (-9931524.0, 1.2841846E7, 7591831.0, -3994093.25) +C | BBOX (-1.8462154E7, 1.5002357E7, 1.7926778E7, -7538976.5) +; + +stExtentManyCartesianShapesGroupedCount +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM countries_bbox_web +| EVAL prefix = SUBSTRING(id, 1, 1) +| STATS extent = ST_EXTENT_AGG(shape), count = COUNT() BY prefix +| KEEP prefix, count, extent +| SORT prefix +| LIMIT 3 +; + +prefix:keyword | count:long | extent:cartesian_shape +A | 17 | BBOX (-2.0037508E7, 2.0037508E7, 6278042.5, -4.748140544E9) +B | 18 | BBOX (-9931524.0, 1.2841846E7, 7591831.0, -3994093.25) +C | 19 | BBOX (-1.8462154E7, 1.5002357E7, 1.7926778E7, -7538976.5) +; + ############################################### # Tests for ST_INTERSECTS on CARTESIAN_POINT type