Enhanced search logic of board listall, core search and lib search (#1222)

* Remove fuzzy logic from lib search command

* Remove fuzzy logic from core search command

* Remove fuzzy logic from board listall command

* Remove fuzzysearch dependency

Author: silvanocerza

commands/board/listall.go

@@ -20,9 +20,9 @@ import (
 	"errors"
 	"strings"
 
+	"github.com/arduino/arduino-cli/arduino/utils"
 	"github.com/arduino/arduino-cli/commands"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
-	"github.com/lithammer/fuzzysearch/fuzzy"
 )
 
 // maximumSearchDistance is the maximum Levenshtein distance accepted when using fuzzy search.
@@ -36,18 +36,26 @@ func ListAll(ctx context.Context, req *rpc.BoardListAllReq) (*rpc.BoardListAllRe
 		return nil, errors.New("invalid instance")
 	}
 
-	searchArgs := strings.Join(req.SearchArgs, " ")
+	searchArgs := []string{}
+	for _, s := range req.SearchArgs {
+		searchArgs = append(searchArgs, strings.Trim(s, " "))
+	}
 
-	match := func(toTest []string) bool {
+	match := func(toTest []string) (bool, error) {
 		if len(searchArgs) == 0 {
-			return true
+			return true, nil
 		}
-		for _, rank := range fuzzy.RankFindNormalizedFold(searchArgs, toTest) {
-			if rank.Distance < maximumSearchDistance {
-				return true
+
+		for _, t := range toTest {
+			matches, err := utils.Match(t, searchArgs)
+			if err != nil {
+				return false, err
+			}
+			if matches {
+				return matches, nil
 			}
 		}
-		return false
+		return false, nil
 	}
 
 	list := &rpc.BoardListAllResp{Boards: []*rpc.BoardListItem{}}
@@ -90,10 +98,11 @@ func ListAll(ctx context.Context, req *rpc.BoardListAllReq) (*rpc.BoardListAllRe
 					continue
 				}
 
-				toTest := toTest
-				toTest = append(toTest, strings.Split(board.Name(), " ")...)
+				toTest := append(toTest, board.Name())
 				toTest = append(toTest, board.FQBN())
-				if !match(toTest) {
+				if ok, err := match(toTest); err != nil {
+					return nil, err
+				} else if !ok {
 					continue
 				}
 

commands/core/search.go

@@ -21,9 +21,9 @@ import (
 	"strings"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
+	"github.com/arduino/arduino-cli/arduino/utils"
 	"github.com/arduino/arduino-cli/commands"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
-	"github.com/lithammer/fuzzysearch/fuzzy"
 )
 
 // maximumSearchDistance is the maximum Levenshtein distance accepted when using fuzzy search.
@@ -44,6 +44,26 @@ func PlatformSearch(req *rpc.PlatformSearchReq) (*rpc.PlatformSearchResp, error)
 		vid, pid := searchArgs[:4], searchArgs[5:]
 		res = pm.FindPlatformReleaseProvidingBoardsWithVidPid(vid, pid)
 	} else {
+
+		searchArgs := strings.Split(searchArgs, " ")
+
+		match := func(toTest []string) (bool, error) {
+			if len(searchArgs) == 0 {
+				return true, nil
+			}
+
+			for _, t := range toTest {
+				matches, err := utils.Match(t, searchArgs)
+				if err != nil {
+					return false, err
+				}
+				if matches {
+					return matches, nil
+				}
+			}
+			return false, nil
+		}
+
 		for _, targetPackage := range pm.Packages {
 			for _, platform := range targetPackage.Platforms {
 				// discard invalid platforms
@@ -60,15 +80,6 @@ func PlatformSearch(req *rpc.PlatformSearchReq) (*rpc.PlatformSearchResp, error)
 					continue
 				}
 
-				if searchArgs == "" {
-					if allVersions {
-						res = append(res, platform.GetAllReleases()...)
-					} else {
-						res = append(res, platformRelease)
-					}
-					continue
-				}
-
 				// Gather all strings that can be used for searching
 				toTest := []string{
 					platform.String(),
@@ -82,32 +93,18 @@ func PlatformSearch(req *rpc.PlatformSearchReq) (*rpc.PlatformSearchResp, error)
 					toTest = append(toTest, board.Name)
 				}
 
-				// Removes some chars from query strings to enhance results
-				cleanSearchArgs := strings.Map(func(r rune) rune {
-					switch r {
-					case '_':
-					case '-':
-					case ' ':
-						return -1
-					}
-					return r
-				}, searchArgs)
+				// Search
+				if ok, err := match(toTest); err != nil {
+					return nil, err
+				} else if !ok {
+					continue
+				}
 
-				// Fuzzy search
-				for _, arg := range []string{searchArgs, cleanSearchArgs} {
-					for _, rank := range fuzzy.RankFindNormalizedFold(arg, toTest) {
-						// Accepts only results that close to the searched terms
-						if rank.Distance < maximumSearchDistance {
-							if allVersions {
-								res = append(res, platform.GetAllReleases()...)
-							} else {
-								res = append(res, platformRelease)
-							}
-							goto nextPlatform
-						}
-					}
+				if allVersions {
+					res = append(res, platform.GetAllReleases()...)
+				} else {
+					res = append(res, platformRelease)
 				}
-			nextPlatform:
 			}
 		}
 	}

commands/lib/search.go

@@ -22,9 +22,9 @@ import (
 
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesindex"
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
+	"github.com/arduino/arduino-cli/arduino/utils"
 	"github.com/arduino/arduino-cli/commands"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
-	"github.com/lithammer/fuzzysearch/fuzzy"
 	semver "go.bug.st/relaxed-semver"
 )
 
@@ -43,44 +43,33 @@ func searchLibrary(req *rpc.LibrarySearchReq, lm *librariesmanager.LibrariesMana
 	res := []*rpc.SearchedLibrary{}
 	status := rpc.LibrarySearchStatus_success
 
-	// If the query is empty all libraries are returned
-	if strings.Trim(query, " ") == "" {
-		for _, lib := range lm.Index.Libraries {
-			res = append(res, indexLibraryToRPCSearchLibrary(lib))
+	searchArgs := strings.Split(strings.Trim(query, " "), " ")
+
+	match := func(toTest []string) (bool, error) {
+		if len(searchArgs) == 0 {
+			return true, nil
 		}
-		return &rpc.LibrarySearchResp{Libraries: res, Status: status}, nil
-	}
 
-	// maximumSearchDistance is the maximum Levenshtein distance accepted when using fuzzy search.
-	// This value is completely arbitrary and picked randomly.
-	maximumSearchDistance := 150
-	// Use a lower distance for shorter query or the user might be flooded with unrelated results
-	if len(query) <= 4 {
-		maximumSearchDistance = 40
+		for _, t := range toTest {
+			matches, err := utils.Match(t, searchArgs)
+			if err != nil {
+				return false, err
+			}
+			if matches {
+				return matches, nil
+			}
+		}
+		return false, nil
 	}
 
-	// Removes some chars from query strings to enhance results
-	cleanQuery := strings.Map(func(r rune) rune {
-		switch r {
-		case '_':
-		case '-':
-		case ' ':
-			return -1
-		}
-		return r
-	}, query)
 	for _, lib := range lm.Index.Libraries {
-		// Use both uncleaned and cleaned query
-		for _, q := range []string{query, cleanQuery} {
-			toTest := []string{lib.Name, lib.Latest.Paragraph, lib.Latest.Sentence}
-			for _, rank := range fuzzy.RankFindNormalizedFold(q, toTest) {
-				if rank.Distance < maximumSearchDistance {
-					res = append(res, indexLibraryToRPCSearchLibrary(lib))
-					goto nextLib
-				}
-			}
+		toTest := []string{lib.Name, lib.Latest.Paragraph, lib.Latest.Sentence}
+		if ok, err := match(toTest); err != nil {
+			return nil, err
+		} else if !ok {
+			continue
 		}
-	nextLib:
+		res = append(res, indexLibraryToRPCSearchLibrary(lib))
 	}
 
 	return &rpc.LibrarySearchResp{Libraries: res, Status: status}, nil

commands/lib/search_test.go

@@ -39,7 +39,7 @@ func TestSearchLibrarySimilar(t *testing.T) {
 
 	req := &rpc.LibrarySearchReq{
 		Instance: &rpc.Instance{Id: 1},
-		Query:    "ardino",
+		Query:    "arduino",
 	}
 
 	resp, err := searchLibrary(req, lm)

go.mod

@@ -21,7 +21,6 @@ require (
 	github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/leonelquinteros/gotext v1.4.0
-	github.com/lithammer/fuzzysearch v1.1.1
 	github.com/marcinbor85/gohex v0.0.0-20210308104911-55fb1c624d84
 	github.com/mattn/go-colorable v0.1.2
 	github.com/mattn/go-isatty v0.0.8

go.sum

@@ -150,8 +150,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leonelquinteros/gotext v1.4.0 h1:2NHPCto5IoMXbrT0bldPrxj0qM5asOCwtb1aUQZ1tys=
 github.com/leonelquinteros/gotext v1.4.0/go.mod h1:yZGXREmoGTtBvZHNcc+Yfug49G/2spuF/i/Qlsvz1Us=
-github.com/lithammer/fuzzysearch v1.1.1 h1:8F9OAV2xPuYblToVohjanztdnPjbtA0MLgMvDKQ0Z08=
-github.com/lithammer/fuzzysearch v1.1.1/go.mod h1:H2bng+w5gsR7NlfIJM8ElGZI0sX6C/9uzGqicVXGU6c=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=

test/test_board.py

@@ -455,34 +455,6 @@ def test_board_listall_with_manually_installed_platform(run_command, data_dir):
     assert "Arduino SAMD (32-bits ARM Cortex-M0+) Boards" == platform["Name"]
 
 
-def test_board_listall_fuzzy_search(run_command, data_dir):
-    assert run_command("update")
-
-    # Install from platform manager
-    assert run_command("core install arduino:[email protected]")
-
-    # Manually installs a core in sketchbooks hardware folder
-    git_url = "https://github.com/arduino/ArduinoCore-samd.git"
-    repo_dir = Path(data_dir, "hardware", "arduino-beta-development", "samd")
-    assert Repo.clone_from(git_url, repo_dir, multi_options=["-b 1.8.11"])
-
-    res = run_command("board listall --format json samd")
-    assert res.ok
-    data = json.loads(res.stdout)
-    boards = {b["FQBN"]: b for b in data["boards"]}
-    assert len(boards) == 17
-    assert "arduino-beta-development:samd:mkr1000" in boards
-    assert "arduino:avr:uno" not in boards
-
-    res = run_command("board listall --format json avr")
-    assert res.ok
-    data = json.loads(res.stdout)
-    boards = {b["FQBN"]: b for b in data["boards"]}
-    assert len(boards) == 26
-    assert "arduino:avr:uno" in boards
-    assert "arduino-beta-development:samd:mkr1000" not in boards
-
-
 def test_board_details(run_command):
     run_command("core update-index")
     # Download samd core pinned to 1.8.6

test/test_core.py

@@ -76,12 +76,33 @@ def get_platforms(stdout):
     assert "1.0.5" in platforms["Retrokits-RK002:arm"]
     assert "1.0.6" in platforms["Retrokits-RK002:arm"]
 
-    # Search using a board name
+    # Search using board names
     result = run_command(f"core search myboard --all --additional-urls={url} --format json")
     assert result.ok
     platforms = get_platforms(result.stdout)
     assert "1.2.3" in platforms["Package:x86"]
 
+    def run_search(search_args, expected_ids):
+        res = run_command(f"core search --format json {search_args}")
+        assert res.ok
+        data = json.loads(res.stdout)
+        platform_ids = [p["ID"] for p in data]
+        for platform_id in expected_ids:
+            assert platform_id in platform_ids
+
+    run_search("mkr1000", ["arduino:samd"])
+    run_search("mkr 1000", ["arduino:samd"])
+
+    run_search("yún", ["arduino:avr"])
+    run_search("yùn", ["arduino:avr"])
+    run_search("yun", ["arduino:avr"])
+
+    run_search("nano", ["arduino:avr", "arduino:megaavr", "arduino:samd", "arduino:mbed"])
+    run_search("nano 33", ["arduino:samd", "arduino:mbed"])
+    run_search("nano ble", ["arduino:mbed"])
+    run_search("ble", ["arduino:mbed"])
+    run_search("ble nano", ["arduino:mbed"])
+
 
 def test_core_search_no_args(run_command, httpserver):
     """
@@ -146,32 +167,6 @@ def test_core_search_no_args(run_command, httpserver):
     assert len(platforms) == num_platforms
 
 
-def test_core_search_fuzzy(run_command):
-    assert run_command("update")
-
-    def run_fuzzy_search(search_args, expected_ids):
-        res = run_command(f"core search --format json {search_args}")
-        assert res.ok
-        data = json.loads(res.stdout)
-        platform_ids = [p["ID"] for p in data]
-        for platform_id in expected_ids:
-            assert platform_id in platform_ids
-
-    run_fuzzy_search("mkr1000", ["arduino:samd"])
-    run_fuzzy_search("mkr 1000", ["arduino:samd"])
-
-    run_fuzzy_search("yún", ["arduino:avr"])
-    run_fuzzy_search("yùn", ["arduino:avr"])
-    run_fuzzy_search("yun", ["arduino:avr"])
-
-    run_fuzzy_search("nano", ["arduino:avr", "arduino:megaavr", "arduino:samd", "arduino:mbed"])
-    run_fuzzy_search("nano33", ["arduino:samd", "arduino:mbed"])
-    run_fuzzy_search("nano 33", ["arduino:samd", "arduino:mbed"])
-    run_fuzzy_search("nano ble", ["arduino:mbed"])
-    run_fuzzy_search("ble", ["arduino:mbed"])
-    run_fuzzy_search("ble nano", [])
-
-
 def test_core_updateindex_url_not_found(run_command, httpserver):
     assert run_command("core update-index")