From 80c646addac94dc557a5eb35290cca8fb84de494 Mon Sep 17 00:00:00 2001 From: Peter-Josef Meisch Date: Wed, 22 Jun 2022 17:10:01 +0200 Subject: [PATCH] suggest handling --- .../client/elc/DocumentAdapters.java | 16 +++ .../elc/SearchDocumentResponseBuilder.java | 131 ++++++++++++++++-- .../core/AbstractElasticsearchTemplate.java | 6 +- ...AbstractReactiveElasticsearchTemplate.java | 6 +- .../SearchDocumentResponseBuilder.java | 4 +- .../response/CompletionSuggestion.java | 2 +- .../suggest/response/PhraseSuggestion.java | 11 +- .../core/suggest/response/Suggest.java | 11 +- .../core/suggest/response/TermSuggestion.java | 9 +- .../data/elasticsearch/support/ScoreDoc.java | 18 ++- .../CompletionELCIntegrationTests.java | 4 +- .../suggest/CompletionIntegrationTests.java | 10 +- .../ReactiveSuggestELCIntegrationTests.java | 2 +- .../ReactiveSuggestIntegrationTests.java | 2 +- 14 files changed, 187 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java index b90ae49e7..3c00f745e 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java @@ -19,6 +19,7 @@ import co.elastic.clients.elasticsearch.core.MgetResponse; import co.elastic.clients.elasticsearch.core.explain.ExplanationDetail; import co.elastic.clients.elasticsearch.core.get.GetResult; +import co.elastic.clients.elasticsearch.core.search.CompletionSuggestOption; import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.core.search.NestedIdentity; import co.elastic.clients.json.JsonData; @@ -142,6 +143,21 @@ public static SearchDocument from(Hit hit, JsonpMapper jsonpMapper) { highlightFields, innerHits, nestedMetaData, explanation, matchedQueries, hit.routing()); } + public static SearchDocument from(CompletionSuggestOption completionSuggestOption) { + + Document document = completionSuggestOption.source() != null ? Document.from(completionSuggestOption.source()) + : Document.create(); + document.setIndex(completionSuggestOption.index()); + + if (completionSuggestOption.id() != null) { + document.setId(completionSuggestOption.id()); + } + + float score = completionSuggestOption.score() != null ? completionSuggestOption.score().floatValue() : Float.NaN; + return new SearchDocumentAdapter(document, score, new Object[] {}, Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), null, null, null, completionSuggestOption.routing()); + } + @Nullable private static Explanation from(@Nullable co.elastic.clients.elasticsearch.core.explain.Explanation explanation) { diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java index bdf3a69ca..4202b7a85 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java @@ -17,24 +17,30 @@ import co.elastic.clients.elasticsearch._types.aggregations.Aggregate; import co.elastic.clients.elasticsearch.core.SearchResponse; -import co.elastic.clients.elasticsearch.core.search.Hit; -import co.elastic.clients.elasticsearch.core.search.HitsMetadata; -import co.elastic.clients.elasticsearch.core.search.ResponseBody; -import co.elastic.clients.elasticsearch.core.search.Suggestion; -import co.elastic.clients.elasticsearch.core.search.TotalHits; +import co.elastic.clients.elasticsearch.core.search.*; import co.elastic.clients.json.JsonpMapper; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.elasticsearch.search.SearchHits; import org.springframework.data.elasticsearch.core.TotalHitsRelation; import org.springframework.data.elasticsearch.core.document.SearchDocument; import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse; +import org.springframework.data.elasticsearch.core.suggest.response.CompletionSuggestion; +import org.springframework.data.elasticsearch.core.suggest.response.PhraseSuggestion; import org.springframework.data.elasticsearch.core.suggest.response.Suggest; +import org.springframework.data.elasticsearch.core.suggest.response.TermSuggestion; +import org.springframework.data.elasticsearch.support.ScoreDoc; import org.springframework.lang.Nullable; import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; /** * Factory class to create {@link SearchDocumentResponse} instances. @@ -43,6 +49,9 @@ * @since 4.4 */ class SearchDocumentResponseBuilder { + + private static final Log LOGGER = LogFactory.getLog(SearchDocumentResponseBuilder.class); + /** * creates a SearchDocumentResponse from the {@link SearchResponse} * @@ -80,7 +89,7 @@ public static SearchDocumentResponse from(ResponseBody response * @return the {@link SearchDocumentResponse} */ public static SearchDocumentResponse from(HitsMetadata hitsMetadata, @Nullable String scrollId, - Map aggregations, Map>> suggestES, + @Nullable Map aggregations, Map>> suggestES, SearchDocumentResponse.EntityCreator entityCreator, JsonpMapper jsonpMapper) { Assert.notNull(hitsMetadata, "hitsMetadata must not be null"); @@ -116,10 +125,116 @@ public static SearchDocumentResponse from(HitsMetadata hitsMetadata, @Nul ElasticsearchAggregations aggregationsContainer = aggregations != null ? new ElasticsearchAggregations(aggregations) : null; - // todo #2154 - Suggest suggest = null; + Suggest suggest = suggestFrom(suggestES, entityCreator); return new SearchDocumentResponse(totalHits, totalHitsRelation, maxScore, scrollId, searchDocuments, aggregationsContainer, suggest); } + + @Nullable + private static Suggest suggestFrom(Map>> suggestES, + SearchDocumentResponse.EntityCreator entityCreator) { + + if (CollectionUtils.isEmpty(suggestES)) { + return null; + } + + List>> suggestions = new ArrayList<>(); + + suggestES.forEach((name, suggestionsES) -> { + + if (!suggestionsES.isEmpty()) { + // take the type from the first entry + switch (suggestionsES.get(0)._kind()) { + case Term: { + suggestions.add(getTermSuggestion(name, suggestionsES)); + break; + } + case Phrase: { + suggestions.add(getPhraseSuggestion(name, suggestionsES)); + break; + } + case Completion: { + suggestions.add(getCompletionSuggestion(name, suggestionsES, entityCreator)); + break; + } + default: + break; + } + } + }); + + // todo: hasScoreDocs checks if any one + boolean hasScoreDocs = false; + + return new Suggest(suggestions, hasScoreDocs); + } + + private static TermSuggestion getTermSuggestion(String name, List> suggestionsES) { + + List entries = new ArrayList<>(); + suggestionsES.forEach(suggestionES -> { + TermSuggest termSuggest = suggestionES.term(); + + TermSuggestOption optionES = termSuggest.options(); + List options = new ArrayList<>(); + options.add(new TermSuggestion.Entry.Option(optionES.text(), null, optionES.score(), null, + Math.toIntExact(optionES.freq()))); + entries.add(new TermSuggestion.Entry(termSuggest.text(), termSuggest.offset(), termSuggest.length(), options)); + }); + return new TermSuggestion(name, suggestionsES.size(), entries, null); + } + + private static PhraseSuggestion getPhraseSuggestion(String name, List> suggestionsES) { + + List entries = new ArrayList<>(); + suggestionsES.forEach(suggestionES -> { + PhraseSuggest phraseSuggest = suggestionES.phrase(); + PhraseSuggestOption optionES = phraseSuggest.options(); + List options = new ArrayList<>(); + options.add(new PhraseSuggestion.Entry.Option(optionES.text(), optionES.highlighted(), null, null)); + entries.add(new PhraseSuggestion.Entry(phraseSuggest.text(), phraseSuggest.offset(), phraseSuggest.length(), + options, null)); + }); + return new PhraseSuggestion(name, suggestionsES.size(), entries); + } + + private static CompletionSuggestion getCompletionSuggestion(String name, + List> suggestionsES, SearchDocumentResponse.EntityCreator entityCreator) { + List> entries = new ArrayList<>(); + suggestionsES.forEach(suggestionES -> { + CompletionSuggest completionSuggest = suggestionES.completion(); + List> options = new ArrayList<>(); + List> optionsES = completionSuggest.options(); + optionsES.forEach(optionES -> { + SearchDocument searchDocument = (optionES.source() != null) ? DocumentAdapters.from(optionES) : null; + T hitEntity = null; + + if (searchDocument != null) { + try { + hitEntity = entityCreator.apply(searchDocument).get(); + } catch (Exception e) { + if (LOGGER.isWarnEnabled()) { + LOGGER.warn("Error creating entity from SearchDocument: " + e.getMessage()); + } + } + } + + Map> contexts = new HashMap<>(); + optionES.contexts().forEach((key, contextList) -> contexts.put(key, + contextList.stream().map(context -> context._get().toString()).collect(Collectors.toSet()))); + + // response from the new client does not have a doc and shardindex as the ScoreDoc from the old client responses + + options.add(new CompletionSuggestion.Entry.Option<>(optionES.text(), null, optionES.score(), + optionES.collateMatch() != null ? optionES.collateMatch() : false, contexts, + new ScoreDoc(optionES.score() != null ? optionES.score() : Double.NaN, null, null), searchDocument, + hitEntity)); + }); + + entries.add(new CompletionSuggestion.Entry<>(completionSuggest.text(), completionSuggest.offset(), + completionSuggest.length(), options)); + }); + return new CompletionSuggestion<>(name, suggestionsES.size(), entries); + } } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java index 07ace4c68..d1dae04b3 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java @@ -745,9 +745,9 @@ public T doWith(@Nullable Document document) { IndexedObjectInformation indexedObjectInformation = IndexedObjectInformation.of( // documentAfterLoad.hasId() ? documentAfterLoad.getId() : null, // - documentAfterLoad.getSeqNo(), // - documentAfterLoad.getPrimaryTerm(), // - documentAfterLoad.getVersion()); // + documentAfterLoad.hasSeqNo() ? documentAfterLoad.getSeqNo() : null, // + documentAfterLoad.hasPrimaryTerm() ? documentAfterLoad.getPrimaryTerm() : null, // + documentAfterLoad.hasVersion() ? documentAfterLoad.getVersion() : null); // entity = updateIndexedObject(entity, indexedObjectInformation); return maybeCallbackAfterConvert(entity, documentAfterLoad, index); diff --git a/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java index 83b834d98..a13d18454 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java @@ -549,9 +549,9 @@ public Mono toEntity(@Nullable Document document) { IndexedObjectInformation indexedObjectInformation = IndexedObjectInformation.of( // documentAfterLoad.hasId() ? documentAfterLoad.getId() : null, // - documentAfterLoad.getSeqNo(), // - documentAfterLoad.getPrimaryTerm(), // - documentAfterLoad.getVersion()); // + documentAfterLoad.hasSeqNo() ? documentAfterLoad.getSeqNo() : null, // + documentAfterLoad.hasPrimaryTerm() ? documentAfterLoad.getPrimaryTerm() : null, // + documentAfterLoad.hasVersion() ? documentAfterLoad.getVersion() : null); // entity = updateIndexedObject(entity, indexedObjectInformation); return maybeCallAfterConvert(entity, documentAfterLoad, index); diff --git a/src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocumentResponseBuilder.java b/src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocumentResponseBuilder.java index bb8b12849..51e1da753 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocumentResponseBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocumentResponseBuilder.java @@ -156,7 +156,7 @@ private static Suggest suggestFrom(@Nullable org.elasticsearch.search.sugges List options = new ArrayList<>(); for (org.elasticsearch.search.suggest.phrase.PhraseSuggestion.Entry.Option optionES : entryES) { options.add(new PhraseSuggestion.Entry.Option(textToString(optionES.getText()), - textToString(optionES.getHighlighted()), optionES.getScore(), optionES.collateMatch())); + textToString(optionES.getHighlighted()), (double) optionES.getScore(), optionES.collateMatch())); } entries.add(new PhraseSuggestion.Entry(textToString(entryES.getText()), entryES.getOffset(), @@ -188,7 +188,7 @@ private static Suggest suggestFrom(@Nullable org.elasticsearch.search.sugges } options.add(new CompletionSuggestion.Entry.Option<>(textToString(optionES.getText()), - textToString(optionES.getHighlighted()), optionES.getScore(), optionES.collateMatch(), + textToString(optionES.getHighlighted()), (double) optionES.getScore(), optionES.collateMatch(), optionES.getContexts(), scoreDocFrom(optionES.getDoc()), searchDocument, hitEntity)); } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/CompletionSuggestion.java b/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/CompletionSuggestion.java index 1963766bb..e7410c499 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/CompletionSuggestion.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/CompletionSuggestion.java @@ -49,7 +49,7 @@ public static class Option extends Suggest.Suggestion.Entry.Option { @Nullable private final T hitEntity; @Nullable private SearchHit searchHit; - public Option(String text, String highlighted, float score, Boolean collateMatch, + public Option(String text, @Nullable String highlighted, @Nullable Double score, Boolean collateMatch, Map> contexts, ScoreDoc scoreDoc, @Nullable SearchDocument searchDocument, @Nullable T hitEntity) { super(text, highlighted, score, collateMatch); diff --git a/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/PhraseSuggestion.java b/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/PhraseSuggestion.java index 4a2999f16..80fc67662 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/PhraseSuggestion.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/PhraseSuggestion.java @@ -15,6 +15,8 @@ */ package org.springframework.data.elasticsearch.core.suggest.response; +import org.springframework.lang.Nullable; + import java.util.List; /** @@ -29,20 +31,21 @@ public PhraseSuggestion(String name, int size, List entries) { public static class Entry extends Suggest.Suggestion.Entry { - private final double cutoffScore; + @Nullable private final Double cutoffScore; - public Entry(String text, int offset, int length, List