From 08aa4417fedf58a58de7c2f3a917da51f01a2499 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Thu, 11 Jun 2026 16:45:48 +1000 Subject: [PATCH 1/5] use script score for parameter sorting --- .../elastic/CQLToElasticFilterFactory.java | 16 ++++++++++ .../server/core/service/ElasticSearch.java | 32 +++++++++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java index 8c64809d..b2ebb9c2 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java @@ -64,6 +64,12 @@ public class CQLToElasticFilterFactory & CQLFieldsInterface> i @Getter protected Map querySetting; + @Getter + protected boolean parameterPrioritySort = false; + + @Getter + protected Set parameterPrioritySortTerms = new HashSet<>(); + public CQLToElasticFilterFactory(CQLCrsType cqlCoorSystem, Class tClass) { this(cqlCoorSystem, tClass, new HashMap<>()); } @@ -267,6 +273,16 @@ public PropertyIsEqualTo equal(Expression expression, Expression expression1, bo return setting; } + // If the filter compares parameter_vocabs, collect the searched term so the caller can + // add a value-aware sort key that ranks matching human-curated records above AI ones. + if (expression instanceof AttributeExpressionImpl attribute && expression1 instanceof LiteralExpressionImpl literal) { + String fieldName = attribute.toString().toLowerCase(); + if (fieldName.equals("parameter_vocabs")) { + this.parameterPrioritySort = true; + this.parameterPrioritySortTerms.add(literal.toString()); + } + } + return new PropertyEqualToImpl<>(expression, expression1, b, matchAction, collectionFieldType); } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index b3d0e3ee..f5b53d9c 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -8,8 +8,9 @@ import au.org.aodn.ogcapi.server.core.parser.elastic.CQLToElasticFilterFactory; import au.org.aodn.ogcapi.server.core.parser.elastic.QueryHandler; import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.*; import co.elastic.clients.elasticsearch._types.query_dsl.*; +import co.elastic.clients.json.JsonData; import co.elastic.clients.elasticsearch.core.SearchMvtRequest; import co.elastic.clients.elasticsearch.core.SearchRequest; import co.elastic.clients.elasticsearch.core.SearchResponse; @@ -255,6 +256,22 @@ public ElasticSearchBase.SearchResult searchAllCollections( return searchCollectionsByIds(null, Boolean.FALSE, sortBy); } + protected SortOptions parameterVocabsPrioritySort(Collection terms) { + String parameterVocabsField = StacBasicField.ParameterVocabs.searchField; + return SortOptions.of(so -> so + .script(s -> s + .type(ScriptSortType.Number) + .script(sc -> sc + .lang("painless") + .params("terms", JsonData.of(new ArrayList<>(terms))) + .source("if (!doc.containsKey('" + parameterVocabsField + ".keyword') || " + + "doc['" + parameterVocabsField + ".keyword'].empty) { return 0; } " + + "for (def value : doc['" + parameterVocabsField + ".keyword']) { " + + "if (params.terms.contains(value)) { return 1; } } " + + "return 0;")) + .order(SortOrder.Desc))); + } + @Override public ElasticSearchBase.SearchResult searchByParameters(List keywords, String cql, List properties, String sortBy, CQLCrsType coor) throws CQLException { @@ -376,13 +393,24 @@ public ElasticSearchBase.SearchResult searchByParameters(Li .toList(); } + List sortOptions = createSortOptions(sortBy, CQLFields.class); + // When the filter searches parameter_vocabs, prepend a value-aware priority sort key + // so matching human-curated records rank above AI-generated fallback records. This is + // the first sort key; existing -score,-rank ordering is preserved within each tier. + if (factory.isParameterPrioritySort()) { + if (sortOptions == null) { + sortOptions = new ArrayList<>(); + } + sortOptions.add(0, parameterVocabsPrioritySort(factory.getParameterPrioritySortTerms())); + } + return searchCollectionBy( null, should, filters, properties, searchAfter, - createSortOptions(sortBy, CQLFields.class), + sortOptions, score, maxSize ); From 8a999a57ee1d632c2ce3bdc7122f152a6ae5d3b2 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Fri, 12 Jun 2026 11:48:18 +1000 Subject: [PATCH 2/5] save point --- .../elastic/CQLToElasticFilterFactory.java | 12 ++- .../server/core/parser/elastic/OrImpl.java | 81 +++++++++---------- .../server/core/service/ElasticSearch.java | 27 +++++-- 3 files changed, 68 insertions(+), 52 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java index b2ebb9c2..1fd7ac4e 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java @@ -70,6 +70,12 @@ public class CQLToElasticFilterFactory & CQLFieldsInterface> i @Getter protected Set parameterPrioritySortTerms = new HashSet<>(); + @Getter + protected boolean platformPrioritySort = false; + + @Getter + protected Set platformPrioritySortTerms = new HashSet<>(); + public CQLToElasticFilterFactory(CQLCrsType cqlCoorSystem, Class tClass) { this(cqlCoorSystem, tClass, new HashMap<>()); } @@ -273,7 +279,7 @@ public PropertyIsEqualTo equal(Expression expression, Expression expression1, bo return setting; } - // If the filter compares parameter_vocabs, collect the searched term so the caller can + // If the filter compares curated vocab fields, collect the searched term so the caller can // add a value-aware sort key that ranks matching human-curated records above AI ones. if (expression instanceof AttributeExpressionImpl attribute && expression1 instanceof LiteralExpressionImpl literal) { String fieldName = attribute.toString().toLowerCase(); @@ -281,6 +287,10 @@ public PropertyIsEqualTo equal(Expression expression, Expression expression1, bo this.parameterPrioritySort = true; this.parameterPrioritySortTerms.add(literal.toString()); } + if (fieldName.equals("platform_vocabs")) { + this.platformPrioritySort = true; + this.platformPrioritySortTerms.add(literal.toString()); + } } return new PropertyEqualToImpl<>(expression, expression1, b, matchAction, collectionFieldType); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java index 1fed3d2a..ec908721 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java @@ -14,59 +14,54 @@ public class OrImpl extends QueryHandler implements Or { protected List children = new ArrayList<>(); - public OrImpl(Filter filter1, Filter filter2) { - - if(filter1 instanceof ElasticSetting && filter2 instanceof QueryHandler elasticFilter2) { - this.addErrors(elasticFilter2.getErrors()); - throw new IllegalArgumentException("Or combine with query setting do not make sense"); + private static List collectQueries(Filter filter) { + if (filter instanceof OrImpl orFilter) { + return orFilter.getChildren().stream() + .flatMap(child -> collectQueries(child).stream()) + .toList(); } - else if(filter2 instanceof ElasticSetting && filter1 instanceof QueryHandler elasticFilter1){ - this.addErrors(elasticFilter1.getErrors()); - throw new IllegalArgumentException("Or combine with query setting do not make sense"); + + if (filter instanceof QueryHandler handler && handler.getQuery() != null) { + return List.of(handler.getQuery()); } - else if(filter1 instanceof QueryHandler elasticFilter1 && filter2 instanceof QueryHandler elasticFilter2) { - // If the CQL contains ElasticSetting then the query will be null, this check is used to make sure - // we ignore those null query - if(elasticFilter1.query != null && elasticFilter2.query != null) { - this.query = BoolQuery.of(f -> f - .should(elasticFilter1.query, elasticFilter2.query) - )._toQuery(); - } - else if(elasticFilter1.query != null) { - this.query = elasticFilter1.query; - } - else { - this.query = elasticFilter2.query; - } - - children.add(filter1); - children.add(filter2); - - // Remember to copy the error from child - this.addErrors(elasticFilter1.getErrors()); - this.addErrors(elasticFilter2.getErrors()); + + return List.of(); + } + + private void buildQuery(List filters) { + List queries = filters.stream() + .flatMap(filter -> collectQueries(filter).stream()) + .toList(); + + if (queries.size() == 1) { + this.query = queries.get(0); + } else if (!queries.isEmpty()) { + this.query = BoolQuery.of(b -> b.should(queries))._toQuery(); } } - public OrImpl(List filters) { - // Extract query object in the filters, it must be an ElasitcFilter - List elasticFilters = filters.stream() - .filter(f -> f instanceof QueryHandler) - .map(m -> (QueryHandler)m) - .collect(Collectors.toList()); + public OrImpl(Filter filter1, Filter filter2) { + children.add(filter1); + children.add(filter2); - List queries = elasticFilters.stream() - .map(m -> m.query) - .collect(Collectors.toList()); + buildQuery(children); - this.query = BoolQuery.of(f -> f - .should(queries)) - ._toQuery(); + if (filter1 instanceof QueryHandler handler) { + addErrors(handler.getErrors()); + } + if (filter2 instanceof QueryHandler handler) { + addErrors(handler.getErrors()); + } + } + public OrImpl(List filters) { children.addAll(filters); + buildQuery(children); - // Copy child error if any - elasticFilters.stream().forEach(elasticFilter -> {this.addErrors(elasticFilter.getErrors());}); + filters.stream() + .filter(QueryHandler.class::isInstance) + .map(QueryHandler.class::cast) + .forEach(handler -> addErrors(handler.getErrors())); } @Override diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index f5b53d9c..026ad0c4 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -257,18 +257,23 @@ public ElasticSearchBase.SearchResult searchAllCollections( } protected SortOptions parameterVocabsPrioritySort(Collection terms) { - String parameterVocabsField = StacBasicField.ParameterVocabs.searchField; + return vocabPrioritySort(StacBasicField.ParameterVocabs.searchField); + } + + protected SortOptions platformVocabsPrioritySort(Collection terms) { + return vocabPrioritySort(StacBasicField.PlatformVocabs.searchField); + } + + protected SortOptions vocabPrioritySort(String vocabField) { return SortOptions.of(so -> so .script(s -> s .type(ScriptSortType.Number) .script(sc -> sc .lang("painless") - .params("terms", JsonData.of(new ArrayList<>(terms))) - .source("if (!doc.containsKey('" + parameterVocabsField + ".keyword') || " + - "doc['" + parameterVocabsField + ".keyword'].empty) { return 0; } " + - "for (def value : doc['" + parameterVocabsField + ".keyword']) { " + - "if (params.terms.contains(value)) { return 1; } } " + - "return 0;")) + .source( + "return doc.containsKey('" + vocabField + ".keyword') && " + + "!doc['" + vocabField + ".keyword'].empty ? 1 : 0;" + )) .order(SortOrder.Desc))); } @@ -394,7 +399,7 @@ public ElasticSearchBase.SearchResult searchByParameters(Li } List sortOptions = createSortOptions(sortBy, CQLFields.class); - // When the filter searches parameter_vocabs, prepend a value-aware priority sort key + // When the filter searches curated vocab fields, prepend value-aware priority sort keys // so matching human-curated records rank above AI-generated fallback records. This is // the first sort key; existing -score,-rank ordering is preserved within each tier. if (factory.isParameterPrioritySort()) { @@ -403,6 +408,12 @@ public ElasticSearchBase.SearchResult searchByParameters(Li } sortOptions.add(0, parameterVocabsPrioritySort(factory.getParameterPrioritySortTerms())); } + if (factory.isPlatformPrioritySort()) { + if (sortOptions == null) { + sortOptions = new ArrayList<>(); + } + sortOptions.add(0, platformVocabsPrioritySort(factory.getPlatformPrioritySortTerms())); + } return searchCollectionBy( null, From a8163ad078ba88b95705aeb77a8535dddad13ae1 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Fri, 12 Jun 2026 12:45:43 +1000 Subject: [PATCH 3/5] add comment and test --- .../elastic/CQLToElasticFilterFactory.java | 33 ++++++- .../server/core/parser/elastic/OrImpl.java | 15 ++- .../CQLToElasticFilterFactoryTest.java | 99 +++++++++++++++++++ 3 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java index 1fd7ac4e..940a78d6 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java @@ -64,15 +64,29 @@ public class CQLToElasticFilterFactory & CQLFieldsInterface> i @Getter protected Map querySetting; + /** + * Indicates that a parameter vocabulary filter was found and curated parameter values (parameter_vocabs) should + * be prioritised in the Elasticsearch result ordering. + */ @Getter protected boolean parameterPrioritySort = false; + /** + * Parameter vocabulary values collected from equality filters while parsing the CQL query. + */ @Getter protected Set parameterPrioritySortTerms = new HashSet<>(); + /** + * Indicates that a platform vocabulary filter was found and curated platform values (platform_vocabs) should be + * prioritised in the Elasticsearch result ordering. + */ @Getter protected boolean platformPrioritySort = false; + /** + * Platform vocabulary values collected from equality filters while parsing the CQL query. + */ @Getter protected Set platformPrioritySortTerms = new HashSet<>(); @@ -267,6 +281,22 @@ public PropertyIsEqualTo equal(Expression expression, Expression expression1, bo return equal(expression, expression1, b, null); } + /** + * Creates an Elasticsearch equality filter and records metadata used to build the search + * request. + * + *

Query-setting expressions such as {@code page_size=11} are stored in + * {@link #querySetting} and returned without creating a normal Elasticsearch field query. + * Equality filters on {@code parameter_vocabs} or {@code platform_vocabs} enable the + * corresponding priority sort and collect the literal filter value. All other expressions are + * converted directly to a {@link PropertyEqualToImpl}. + * + * @param expression the field expression on the left side of the equality comparison + * @param expression1 the value expression on the right side of the equality comparison + * @param b whether string comparison should be case-sensitive + * @param matchAction how a multi-valued property should be matched + * @return an Elasticsearch query setting or equality filter + */ @Override public PropertyIsEqualTo equal(Expression expression, Expression expression1, boolean b, MultiValuedFilter.MatchAction matchAction) { logger.debug("PropertyIsEqualTo {} {}, {} {}", expression, expression1, b, matchAction); @@ -279,8 +309,7 @@ public PropertyIsEqualTo equal(Expression expression, Expression expression1, bo return setting; } - // If the filter compares curated vocab fields, collect the searched term so the caller can - // add a value-aware sort key that ranks matching human-curated records above AI ones. + // Record curated vocabulary filters so the search service can prioritise curated records. if (expression instanceof AttributeExpressionImpl attribute && expression1 instanceof LiteralExpressionImpl literal) { String fieldName = attribute.toString().toLowerCase(); if (fieldName.equals("parameter_vocabs")) { diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java index ec908721..6be4e44a 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java @@ -8,12 +8,15 @@ import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; public class OrImpl extends QueryHandler implements Or { protected List children = new ArrayList<>(); + /** + * Recursively extracts leaf Elasticsearch queries from nested OR filters and returns them as a flat list. + * The caller uses this list to construct a single bool/should query. + */ private static List collectQueries(Filter filter) { if (filter instanceof OrImpl orFilter) { return orFilter.getChildren().stream() @@ -28,6 +31,14 @@ private static List collectQueries(Filter filter) { return List.of(); } + + /** + * Builds the Elasticsearch representation of an OR expression. + * + * A single query is returned directly. Multiple queries are combined into + * one flat bool/should query to avoid deeply nested bool queries for large + * vocabulary selections. + */ private void buildQuery(List filters) { List queries = filters.stream() .flatMap(filter -> collectQueries(filter).stream()) @@ -54,6 +65,8 @@ public OrImpl(Filter filter1, Filter filter2) { } } + // Flatten the bool should query so that to avoid when many parameters are selected, + // the nested query cause Elasticsearch error public OrImpl(List filters) { children.addAll(filters); buildQuery(children); diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java new file mode 100644 index 00000000..e909aa8f --- /dev/null +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java @@ -0,0 +1,99 @@ +package au.org.aodn.ogcapi.server.core.parser.elastic; + +import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType; +import au.org.aodn.ogcapi.server.core.model.enumeration.CQLElasticSetting; +import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; +import org.geotools.filter.text.commons.CompilerUtil; +import org.geotools.filter.text.commons.Language; +import org.geotools.filter.text.cql2.CQLException; +import org.junit.jupiter.api.Test; +import org.opengis.filter.Filter; + +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class CQLToElasticFilterFactoryTest { + + @Test + public void parameterVocabFilterEnablesPrioritySortAndCollectsTerms() throws CQLException { + String cql = "parameter_vocabs='acoustics' OR ai_parameter_vocabs='acoustics' OR " + + "parameter_vocabs='aerosols' OR ai_parameter_vocabs='aerosols' OR " + + "parameter_vocabs='air pressure' OR ai_parameter_vocabs='air pressure'"; + CQLToElasticFilterFactory factory = newFactory(); + Filter filter = CompilerUtil.parseFilter(Language.ECQL, cql, factory); + + assertTrue(factory.isParameterPrioritySort()); + assertEquals( + Set.of("acoustics", "aerosols", "air pressure"), + factory.getParameterPrioritySortTerms()); + assertFalse(factory.isPlatformPrioritySort()); + assertTrue(factory.getPlatformPrioritySortTerms().isEmpty()); + + OrImpl parameterFilter = assertInstanceOf(OrImpl.class, filter); + assertTrue(parameterFilter.getQuery().isBool()); + assertEquals(6, parameterFilter.getQuery().bool().should().size()); + assertTrue( + parameterFilter.getQuery().bool().should().stream().noneMatch(query -> query.isBool()), + "Parameter vocabulary clauses should be flattened into one should list"); + } + + @Test + public void platformVocabFilterEnablesPrioritySortAndCollectsTerms() throws CQLException { + CQLToElasticFilterFactory factory = parse( + "platform_vocabs='glider' OR platform_vocabs='mooring'"); + + assertTrue(factory.isPlatformPrioritySort()); + assertEquals(Set.of("glider", "mooring"), factory.getPlatformPrioritySortTerms()); + assertFalse(factory.isParameterPrioritySort()); + assertTrue(factory.getParameterPrioritySortTerms().isEmpty()); + } + + @Test + public void duplicateVocabTermsAreCollectedOnce() throws CQLException { + CQLToElasticFilterFactory factory = parse( + "(parameter_vocabs='temperature' OR ai_parameter_vocabs='temperature') OR " + + "(parameter_vocabs='temperature' OR ai_parameter_vocabs='temperature')"); + + assertEquals(Set.of("temperature"), factory.getParameterPrioritySortTerms()); + } + + @Test + public void aiAndUnrelatedFiltersDoNotEnablePrioritySort() throws CQLException { + CQLToElasticFilterFactory factory = parse( + "ai_parameter_vocabs='temperature' OR " + + "ai_platform_vocabs='glider' OR status='ongoing'"); + + assertFalse(factory.isParameterPrioritySort()); + assertTrue(factory.getParameterPrioritySortTerms().isEmpty()); + assertFalse(factory.isPlatformPrioritySort()); + assertTrue(factory.getPlatformPrioritySortTerms().isEmpty()); + } + + @Test + public void prioritySortMetadataIsCollectedAlongsideQuerySettings() throws CQLException { + CQLToElasticFilterFactory factory = parse( + "page_size=11 AND " + + "(parameter_vocabs='heat budget' OR ai_parameter_vocabs='heat budget') " + + "AND platform_vocabs='glider'"); + + assertEquals("11", factory.getQuerySetting().get(CQLElasticSetting.page_size)); + assertTrue(factory.isParameterPrioritySort()); + assertEquals(Set.of("heat budget"), factory.getParameterPrioritySortTerms()); + assertTrue(factory.isPlatformPrioritySort()); + assertEquals(Set.of("glider"), factory.getPlatformPrioritySortTerms()); + } + + private CQLToElasticFilterFactory parse(String cql) throws CQLException { + CQLToElasticFilterFactory factory = newFactory(); + CompilerUtil.parseFilter(Language.ECQL, cql, factory); + return factory; + } + + private CQLToElasticFilterFactory newFactory() { + return new CQLToElasticFilterFactory<>(CQLCrsType.EPSG4326, CQLFields.class); + } +} From 918eefd18d49e86d206f4216e9c990eddb58be3d Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Fri, 12 Jun 2026 13:10:00 +1000 Subject: [PATCH 4/5] fix test --- .../elastic/CQLToElasticFilterFactory.java | 15 +----- .../server/core/parser/elastic/OrImpl.java | 15 +++++- .../ogcapi/server/common/RestApiTest.java | 6 +-- .../CQLToElasticFilterFactoryTest.java | 46 +++++++++---------- 4 files changed, 39 insertions(+), 43 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java index 940a78d6..e7ced7b3 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java @@ -282,20 +282,7 @@ public PropertyIsEqualTo equal(Expression expression, Expression expression1, bo } /** - * Creates an Elasticsearch equality filter and records metadata used to build the search - * request. - * - *

Query-setting expressions such as {@code page_size=11} are stored in - * {@link #querySetting} and returned without creating a normal Elasticsearch field query. - * Equality filters on {@code parameter_vocabs} or {@code platform_vocabs} enable the - * corresponding priority sort and collect the literal filter value. All other expressions are - * converted directly to a {@link PropertyEqualToImpl}. - * - * @param expression the field expression on the left side of the equality comparison - * @param expression1 the value expression on the right side of the equality comparison - * @param b whether string comparison should be case-sensitive - * @param matchAction how a multi-valued property should be matched - * @return an Elasticsearch query setting or equality filter + * Creates an Elasticsearch equality filter and records metadata used to build the search request. */ @Override public PropertyIsEqualTo equal(Expression expression, Expression expression1, boolean b, MultiValuedFilter.MatchAction matchAction) { diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java index 6be4e44a..140786d0 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java @@ -13,6 +13,15 @@ public class OrImpl extends QueryHandler implements Or { protected List children = new ArrayList<>(); + private static boolean containsElasticSetting(Filter filter) { + if (filter instanceof ElasticSetting) { + return true; + } + + return filter instanceof OrImpl orFilter + && orFilter.getChildren().stream().anyMatch(OrImpl::containsElasticSetting); + } + /** * Recursively extracts leaf Elasticsearch queries from nested OR filters and returns them as a flat list. * The caller uses this list to construct a single bool/should query. @@ -40,6 +49,10 @@ private static List collectQueries(Filter filter) { * vocabulary selections. */ private void buildQuery(List filters) { + if (filters.stream().anyMatch(OrImpl::containsElasticSetting)) { + throw new IllegalArgumentException("Or combine with query setting do not make sense"); + } + List queries = filters.stream() .flatMap(filter -> collectQueries(filter).stream()) .toList(); @@ -65,8 +78,6 @@ public OrImpl(Filter filter1, Filter filter2) { } } - // Flatten the bool should query so that to avoid when many parameters are selected, - // the nested query cause Elasticsearch error public OrImpl(List filters) { children.addAll(filters); buildQuery(children); diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java index a4630e0d..46f9fb6d 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java @@ -555,11 +555,11 @@ public void verifyCQLPropertyScore() throws IOException { "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" ); // Make sure AND operation works - ResponseEntity collections = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=score>=2 AND parameter_vocabs='wave'", Collections.class); + ResponseEntity collections = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=score>=2 AND (parameter_vocabs='wave' OR ai_parameter_vocabs='wave')", Collections.class); assertEquals(1, Objects.requireNonNull(collections.getBody()).getCollections().size(), "hit 1, only one record"); // Make sure OR not work as it didn't make sense to use or with setting - ResponseEntity error = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=score>=2 OR parameter_vocabs='wave'", ErrorResponse.class); + ResponseEntity error = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=score>=2 OR (parameter_vocabs='wave' OR ai_parameter_vocabs='wave')", ErrorResponse.class); assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, error.getStatusCode()); assertEquals("Or combine with query setting do not make sense", Objects.requireNonNull(error.getBody()).getMessage(), "correct error"); @@ -621,7 +621,7 @@ public void verifySortBy() throws IOException { // Edge case on sort by with 1 item, but typo in argument sortBy, it should be sortby. Hence use API default sort -score // https://docs.ogc.org/DRAFTS/20-004.html#sorting-parameter-sortby - ResponseEntity collections = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=score>=2 AND parameter_vocabs='wave'&sortBy=-score,+title", ExtendedCollections.class); + ResponseEntity collections = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=score>=2 AND (parameter_vocabs='wave' OR ai_parameter_vocabs='wave')&sortBy=-score,+title", ExtendedCollections.class); assertEquals(1, Objects.requireNonNull(collections.getBody()).getCollections().size(), "hit 1, only one record"); // Now return result should sort by score then title, since no query here, the score will auto adjust to 1 as all search without query default score is 1 diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java index e909aa8f..7c23f223 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java @@ -14,6 +14,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; public class CQLToElasticFilterFactoryTest { @@ -44,33 +45,12 @@ public void parameterVocabFilterEnablesPrioritySortAndCollectsTerms() throws CQL @Test public void platformVocabFilterEnablesPrioritySortAndCollectsTerms() throws CQLException { CQLToElasticFilterFactory factory = parse( - "platform_vocabs='glider' OR platform_vocabs='mooring'"); + "platform_vocabs='glider' OR ai_platform_vocabs='glider'"); assertTrue(factory.isPlatformPrioritySort()); - assertEquals(Set.of("glider", "mooring"), factory.getPlatformPrioritySortTerms()); - assertFalse(factory.isParameterPrioritySort()); - assertTrue(factory.getParameterPrioritySortTerms().isEmpty()); - } - - @Test - public void duplicateVocabTermsAreCollectedOnce() throws CQLException { - CQLToElasticFilterFactory factory = parse( - "(parameter_vocabs='temperature' OR ai_parameter_vocabs='temperature') OR " - + "(parameter_vocabs='temperature' OR ai_parameter_vocabs='temperature')"); - - assertEquals(Set.of("temperature"), factory.getParameterPrioritySortTerms()); - } - - @Test - public void aiAndUnrelatedFiltersDoNotEnablePrioritySort() throws CQLException { - CQLToElasticFilterFactory factory = parse( - "ai_parameter_vocabs='temperature' OR " - + "ai_platform_vocabs='glider' OR status='ongoing'"); - + assertEquals(Set.of("glider"), factory.getPlatformPrioritySortTerms()); assertFalse(factory.isParameterPrioritySort()); assertTrue(factory.getParameterPrioritySortTerms().isEmpty()); - assertFalse(factory.isPlatformPrioritySort()); - assertTrue(factory.getPlatformPrioritySortTerms().isEmpty()); } @Test @@ -78,7 +58,7 @@ public void prioritySortMetadataIsCollectedAlongsideQuerySettings() throws CQLEx CQLToElasticFilterFactory factory = parse( "page_size=11 AND " + "(parameter_vocabs='heat budget' OR ai_parameter_vocabs='heat budget') " - + "AND platform_vocabs='glider'"); + + "AND (platform_vocabs='glider' OR ai_platform_vocabs='glider')"); assertEquals("11", factory.getQuerySetting().get(CQLElasticSetting.page_size)); assertTrue(factory.isParameterPrioritySort()); @@ -87,6 +67,24 @@ public void prioritySortMetadataIsCollectedAlongsideQuerySettings() throws CQLEx assertEquals(Set.of("glider"), factory.getPlatformPrioritySortTerms()); } + @Test + public void querySettingsCannotBeCombinedWithOr() { + IllegalArgumentException settingFirst = assertThrows( + IllegalArgumentException.class, + () -> parse("score>=2 OR parameter_vocabs='wave'")); + assertEquals( + "Or combine with query setting do not make sense", + settingFirst.getMessage()); + + IllegalArgumentException settingLast = assertThrows( + IllegalArgumentException.class, + () -> parse( + "parameter_vocabs='wave' OR ai_parameter_vocabs='wave' OR score>=2")); + assertEquals( + "Or combine with query setting do not make sense", + settingLast.getMessage()); + } + private CQLToElasticFilterFactory parse(String cql) throws CQLException { CQLToElasticFilterFactory factory = newFactory(); CompilerUtil.parseFilter(Language.ECQL, cql, factory); From 2ee6569f6218f3191699d1391e2596e33f8575e6 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Fri, 12 Jun 2026 14:16:13 +1000 Subject: [PATCH 5/5] fix test --- .../elastic/CQLToElasticFilterFactory.java | 16 +------- .../server/core/service/ElasticSearch.java | 10 ++--- .../CQLToElasticFilterFactoryTest.java | 38 +++++++++---------- 3 files changed, 25 insertions(+), 39 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java index e7ced7b3..98c93635 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java @@ -71,12 +71,6 @@ public class CQLToElasticFilterFactory & CQLFieldsInterface> i @Getter protected boolean parameterPrioritySort = false; - /** - * Parameter vocabulary values collected from equality filters while parsing the CQL query. - */ - @Getter - protected Set parameterPrioritySortTerms = new HashSet<>(); - /** * Indicates that a platform vocabulary filter was found and curated platform values (platform_vocabs) should be * prioritised in the Elasticsearch result ordering. @@ -84,12 +78,6 @@ public class CQLToElasticFilterFactory & CQLFieldsInterface> i @Getter protected boolean platformPrioritySort = false; - /** - * Platform vocabulary values collected from equality filters while parsing the CQL query. - */ - @Getter - protected Set platformPrioritySortTerms = new HashSet<>(); - public CQLToElasticFilterFactory(CQLCrsType cqlCoorSystem, Class tClass) { this(cqlCoorSystem, tClass, new HashMap<>()); } @@ -297,15 +285,13 @@ public PropertyIsEqualTo equal(Expression expression, Expression expression1, bo } // Record curated vocabulary filters so the search service can prioritise curated records. - if (expression instanceof AttributeExpressionImpl attribute && expression1 instanceof LiteralExpressionImpl literal) { + if (expression instanceof AttributeExpressionImpl attribute && expression1 instanceof LiteralExpressionImpl) { String fieldName = attribute.toString().toLowerCase(); if (fieldName.equals("parameter_vocabs")) { this.parameterPrioritySort = true; - this.parameterPrioritySortTerms.add(literal.toString()); } if (fieldName.equals("platform_vocabs")) { this.platformPrioritySort = true; - this.platformPrioritySortTerms.add(literal.toString()); } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 026ad0c4..b4b468a1 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -256,11 +256,11 @@ public ElasticSearchBase.SearchResult searchAllCollections( return searchCollectionsByIds(null, Boolean.FALSE, sortBy); } - protected SortOptions parameterVocabsPrioritySort(Collection terms) { + protected SortOptions parameterVocabsPrioritySort() { return vocabPrioritySort(StacBasicField.ParameterVocabs.searchField); } - protected SortOptions platformVocabsPrioritySort(Collection terms) { + protected SortOptions platformVocabsPrioritySort() { return vocabPrioritySort(StacBasicField.PlatformVocabs.searchField); } @@ -399,20 +399,20 @@ public ElasticSearchBase.SearchResult searchByParameters(Li } List sortOptions = createSortOptions(sortBy, CQLFields.class); - // When the filter searches curated vocab fields, prepend value-aware priority sort keys + // When the filter searches curated vocab fields, prepend presence-based priority sort keys // so matching human-curated records rank above AI-generated fallback records. This is // the first sort key; existing -score,-rank ordering is preserved within each tier. if (factory.isParameterPrioritySort()) { if (sortOptions == null) { sortOptions = new ArrayList<>(); } - sortOptions.add(0, parameterVocabsPrioritySort(factory.getParameterPrioritySortTerms())); + sortOptions.add(0, parameterVocabsPrioritySort()); } if (factory.isPlatformPrioritySort()) { if (sortOptions == null) { sortOptions = new ArrayList<>(); } - sortOptions.add(0, platformVocabsPrioritySort(factory.getPlatformPrioritySortTerms())); + sortOptions.add(0, platformVocabsPrioritySort()); } return searchCollectionBy( diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java index 7c23f223..48eead43 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java @@ -9,8 +9,6 @@ import org.junit.jupiter.api.Test; import org.opengis.filter.Filter; -import java.util.Set; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -20,19 +18,15 @@ public class CQLToElasticFilterFactoryTest { @Test - public void parameterVocabFilterEnablesPrioritySortAndCollectsTerms() throws CQLException { - String cql = "parameter_vocabs='acoustics' OR ai_parameter_vocabs='acoustics' OR " - + "parameter_vocabs='aerosols' OR ai_parameter_vocabs='aerosols' OR " - + "parameter_vocabs='air pressure' OR ai_parameter_vocabs='air pressure'"; + public void parameterVocabFilterEnablesPrioritySort() throws CQLException { + String cql = "(parameter_vocabs='acoustics' OR ai_parameter_vocabs='acoustics') OR " + + "(parameter_vocabs='aerosols' OR ai_parameter_vocabs='aerosols') OR " + + "(parameter_vocabs='air pressure' OR ai_parameter_vocabs='air pressure')"; CQLToElasticFilterFactory factory = newFactory(); Filter filter = CompilerUtil.parseFilter(Language.ECQL, cql, factory); assertTrue(factory.isParameterPrioritySort()); - assertEquals( - Set.of("acoustics", "aerosols", "air pressure"), - factory.getParameterPrioritySortTerms()); assertFalse(factory.isPlatformPrioritySort()); - assertTrue(factory.getPlatformPrioritySortTerms().isEmpty()); OrImpl parameterFilter = assertInstanceOf(OrImpl.class, filter); assertTrue(parameterFilter.getQuery().isBool()); @@ -43,28 +37,34 @@ public void parameterVocabFilterEnablesPrioritySortAndCollectsTerms() throws CQL } @Test - public void platformVocabFilterEnablesPrioritySortAndCollectsTerms() throws CQLException { - CQLToElasticFilterFactory factory = parse( - "platform_vocabs='glider' OR ai_platform_vocabs='glider'"); + public void platformVocabFilterEnablesPrioritySort() throws CQLException { + String cql = "(platform_vocabs='satellite' OR ai_platform_vocabs='satellite') OR " + + "(platform_vocabs='glider' OR ai_platform_vocabs='glider')"; + CQLToElasticFilterFactory factory = newFactory(); + Filter filter = CompilerUtil.parseFilter(Language.ECQL, cql, factory); assertTrue(factory.isPlatformPrioritySort()); - assertEquals(Set.of("glider"), factory.getPlatformPrioritySortTerms()); assertFalse(factory.isParameterPrioritySort()); - assertTrue(factory.getParameterPrioritySortTerms().isEmpty()); + + OrImpl platformFilter = assertInstanceOf(OrImpl.class, filter); + assertTrue(platformFilter.getQuery().isBool()); + assertEquals(4, platformFilter.getQuery().bool().should().size()); + assertTrue( + platformFilter.getQuery().bool().should().stream().noneMatch(query -> query.isBool()), + "Grouped platform vocabulary clauses should be flattened into one should list"); } @Test public void prioritySortMetadataIsCollectedAlongsideQuerySettings() throws CQLException { CQLToElasticFilterFactory factory = parse( "page_size=11 AND " - + "(parameter_vocabs='heat budget' OR ai_parameter_vocabs='heat budget') " - + "AND (platform_vocabs='glider' OR ai_platform_vocabs='glider')"); + + "((parameter_vocabs='heat budget' OR ai_parameter_vocabs='heat budget')) " + + "AND ((platform_vocabs='satellite' OR ai_platform_vocabs='satellite') OR " + + "(platform_vocabs='glider' OR ai_platform_vocabs='glider'))"); assertEquals("11", factory.getQuerySetting().get(CQLElasticSetting.page_size)); assertTrue(factory.isParameterPrioritySort()); - assertEquals(Set.of("heat budget"), factory.getParameterPrioritySortTerms()); assertTrue(factory.isPlatformPrioritySort()); - assertEquals(Set.of("glider"), factory.getPlatformPrioritySortTerms()); } @Test