[DE-385] 3.10 analyzers

docker/start_db.sh

@@ -101,6 +101,13 @@ for a in ${COORDINATORS[*]} ; do
     curl -u root:test --insecure --fail "$SCHEME://$a/_api/version"
 done
 
+echo ""
+echo ""
+echo "Copying test ML models into containers..."
+for c in $(docker ps -a -f name=adb-.* -q) ; do
+    docker cp "$LOCATION"/foo.bin "$c":/tmp
+done
+
 echo ""
 echo ""
 echo "Done, your deployment is reachable at: "

src/main/java/com/arangodb/entity/InvertedIndexField.java

@@ -18,7 +18,7 @@ public class InvertedIndexField implements Entity {
     private Boolean searchField;
     private Boolean trackListPositions;
     private final Set<AnalyzerFeature> features = new HashSet<>();
-    private final Collection<InvertedIndexField> nested = new ArrayList<>();
+    private Collection<InvertedIndexField> nested;
 
     public String getName() {
         return name;
@@ -79,6 +79,7 @@ public Collection<InvertedIndexField> getNested() {
     }
 
     public InvertedIndexField nested(InvertedIndexField... nested) {
+        if(this.nested == null) this.nested = new ArrayList<>();
         Collections.addAll(this.nested, nested);
         return this;
     }

src/main/java/com/arangodb/entity/arangosearch/AnalyzerType.java

@@ -24,5 +24,20 @@
  * @author Michele Rastelli
  */
 public enum AnalyzerType {
-    identity, delimiter, stem, norm, ngram, text, pipeline, stopwords, aql, geojson, geopoint, segmentation, collation
+    identity,
+    delimiter,
+    stem,
+    norm,
+    ngram,
+    text,
+    pipeline,
+    stopwords,
+    aql,
+    geojson,
+    geopoint,
+    segmentation,
+    collation,
+    classification,
+    nearest_neighbors,
+    minhash
 }

src/main/java/com/arangodb/entity/arangosearch/analyzer/ClassificationAnalyzer.java

@@ -0,0 +1,64 @@
+/*
+ * DISCLAIMER
+ *
+ * Copyright 2016 ArangoDB GmbH, Cologne, Germany
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright holder is ArangoDB GmbH, Cologne, Germany
+ */
+
+package com.arangodb.entity.arangosearch.analyzer;
+
+
+import com.arangodb.entity.arangosearch.AnalyzerType;
+
+import java.util.Objects;
+
+/**
+ * An Analyzer capable of classifying tokens in the input text. It applies a user-provided supervised fastText word
+ * embedding model to classify the input text. It is able to classify individual tokens as well as entire inputs.
+ *
+ * @author Michele Rastelli
+ * @see <a href= "https://www.arangodb.com/docs/stable/analyzers.html#classification">API Documentation</a>
+ * @since ArangoDB 3.10
+ */
+public class ClassificationAnalyzer extends SearchAnalyzer {
+    public ClassificationAnalyzer() {
+        setType(AnalyzerType.classification);
+    }
+
+    private ClassificationAnalyzerProperties properties;
+
+    public ClassificationAnalyzerProperties getProperties() {
+        return properties;
+    }
+
+    public void setProperties(ClassificationAnalyzerProperties properties) {
+        this.properties = properties;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        ClassificationAnalyzer that = (ClassificationAnalyzer) o;
+        return Objects.equals(properties, that.properties);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), properties);
+    }
+}

src/main/java/com/arangodb/entity/arangosearch/analyzer/ClassificationAnalyzerProperties.java

@@ -0,0 +1,78 @@
+/*
+ * DISCLAIMER
+ *
+ * Copyright 2016 ArangoDB GmbH, Cologne, Germany
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright holder is ArangoDB GmbH, Cologne, Germany
+ */
+
+package com.arangodb.entity.arangosearch.analyzer;
+
+
+import com.arangodb.velocypack.annotations.SerializedName;
+
+import java.util.Objects;
+
+/**
+ * @author Michele Rastelli
+ * @since ArangoDB 3.10
+ */
+public class ClassificationAnalyzerProperties {
+
+    @SerializedName("model_location")
+    private String modelLocation;
+
+    @SerializedName("top_k")
+    private Integer topK;
+
+    private Double threshold;
+
+    public String getModelLocation() {
+        return modelLocation;
+    }
+
+    public void setModelLocation(String modelLocation) {
+        this.modelLocation = modelLocation;
+    }
+
+    public Integer getTopK() {
+        return topK;
+    }
+
+    public void setTopK(Integer topK) {
+        this.topK = topK;
+    }
+
+    public Double getThreshold() {
+        return threshold;
+    }
+
+    public void setThreshold(Double threshold) {
+        this.threshold = threshold;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ClassificationAnalyzerProperties that = (ClassificationAnalyzerProperties) o;
+        return Objects.equals(modelLocation, that.modelLocation) && Objects.equals(topK, that.topK) && Objects.equals(threshold, that.threshold);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(modelLocation, topK, threshold);
+    }
+}

src/main/java/com/arangodb/entity/arangosearch/analyzer/MinHashAnalyzer.java

@@ -0,0 +1,64 @@
+/*
+ * DISCLAIMER
+ *
+ * Copyright 2016 ArangoDB GmbH, Cologne, Germany
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright holder is ArangoDB GmbH, Cologne, Germany
+ */
+
+package com.arangodb.entity.arangosearch.analyzer;
+
+
+import com.arangodb.entity.arangosearch.AnalyzerType;
+
+import java.util.Objects;
+
+/**
+ * An Analyzer that computes so called MinHash signatures using a locality-sensitive hash function. It applies an
+ * Analyzer of your choice before the hashing, for example, to break up text into words.
+ *
+ * @author Michele Rastelli
+ * @see <a href= "https://www.arangodb.com/docs/stable/analyzers.html#minhash">API Documentation</a>
+ * @since ArangoDB 3.10
+ */
+public class MinHashAnalyzer extends SearchAnalyzer {
+    public MinHashAnalyzer() {
+        setType(AnalyzerType.minhash);
+    }
+
+    private MinHashAnalyzerProperties properties;
+
+    public MinHashAnalyzerProperties getProperties() {
+        return properties;
+    }
+
+    public void setProperties(MinHashAnalyzerProperties properties) {
+        this.properties = properties;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        MinHashAnalyzer that = (MinHashAnalyzer) o;
+        return Objects.equals(properties, that.properties);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), properties);
+    }
+}

src/main/java/com/arangodb/entity/arangosearch/analyzer/MinHashAnalyzerProperties.java

@@ -0,0 +1,63 @@
+/*
+ * DISCLAIMER
+ *
+ * Copyright 2016 ArangoDB GmbH, Cologne, Germany
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright holder is ArangoDB GmbH, Cologne, Germany
+ */
+
+package com.arangodb.entity.arangosearch.analyzer;
+
+
+import java.util.Objects;
+
+/**
+ * @author Michele Rastelli
+ * @since ArangoDB 3.10
+ */
+public class MinHashAnalyzerProperties {
+
+    private SearchAnalyzer analyzer;
+    private Integer numHashes;
+
+    public SearchAnalyzer getAnalyzer() {
+        return analyzer;
+    }
+
+    public void setAnalyzer(SearchAnalyzer analyzer) {
+        this.analyzer = analyzer;
+    }
+
+    public Integer getNumHashes() {
+        return numHashes;
+    }
+
+    public void setNumHashes(Integer numHashes) {
+        this.numHashes = numHashes;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        MinHashAnalyzerProperties that = (MinHashAnalyzerProperties) o;
+        return Objects.equals(analyzer, that.analyzer) && Objects.equals(numHashes, that.numHashes);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(analyzer, numHashes);
+    }
+}

src/main/java/com/arangodb/entity/arangosearch/analyzer/NearestNeighborsAnalyzer.java

@@ -0,0 +1,66 @@
+/*
+ * DISCLAIMER
+ *
+ * Copyright 2016 ArangoDB GmbH, Cologne, Germany
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright holder is ArangoDB GmbH, Cologne, Germany
+ */
+
+package com.arangodb.entity.arangosearch.analyzer;
+
+
+import com.arangodb.entity.arangosearch.AnalyzerType;
+
+import java.util.Objects;
+
+/**
+ * An Analyzer capable of finding nearest neighbors of tokens in the input. It applies a user-provided supervised
+ * fastText word embedding model to retrieve nearest neighbor tokens in the text. It is able to find neighbors of
+ * individual tokens as well as entire input strings. For entire input strings, the Analyzer will return nearest
+ * neighbors for each token within the input string.
+ *
+ * @author Michele Rastelli
+ * @see <a href= "https://www.arangodb.com/docs/stable/analyzers.html#nearest_neighbors">API Documentation</a>
+ * @since ArangoDB 3.10
+ */
+public class NearestNeighborsAnalyzer extends SearchAnalyzer {
+    public NearestNeighborsAnalyzer() {
+        setType(AnalyzerType.nearest_neighbors);
+    }
+
+    private NearestNeighborsAnalyzerProperties properties;
+
+    public NearestNeighborsAnalyzerProperties getProperties() {
+        return properties;
+    }
+
+    public void setProperties(NearestNeighborsAnalyzerProperties properties) {
+        this.properties = properties;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        NearestNeighborsAnalyzer that = (NearestNeighborsAnalyzer) o;
+        return Objects.equals(properties, that.properties);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), properties);
+    }
+}