diff --git a/properties.go b/properties.go index 933fc1e..7e32eef 100644 --- a/properties.go +++ b/properties.go @@ -77,6 +77,7 @@ import ( "regexp" "runtime" "strings" + "unicode/utf8" "github.com/arduino/go-paths-helper" ) @@ -129,9 +130,23 @@ func NewFromHashmap(orig map[string]string) *Map { return res } +func toUtf8(iso8859_1_buf []byte) string { + buf := make([]rune, len(iso8859_1_buf)) + for i, b := range iso8859_1_buf { + buf[i] = rune(b) + } + return string(buf) +} + // LoadFromBytes reads properties data and makes a Map out of it. func LoadFromBytes(bytes []byte) (*Map, error) { - text := string(bytes) + var text string + if utf8.Valid(bytes) { + text = string(bytes) + } else { + // Assume ISO8859-1 encoding and convert to UTF-8 + text = toUtf8(bytes) + } text = strings.Replace(text, "\r\n", "\n", -1) text = strings.Replace(text, "\r", "\n", -1) diff --git a/properties_test.go b/properties_test.go index 46f2c76..d5fa25e 100644 --- a/properties_test.go +++ b/properties_test.go @@ -384,3 +384,9 @@ func TestExtractSubIndexLists(t *testing.T) { s5 := m.ExtractSubIndexLists("cinque.discovery.required") require.Len(t, s5, 0) } + +func TestLoadingNonUTF8Properties(t *testing.T) { + m, err := LoadFromPath(paths.New("testdata/non-utf8.properties")) + require.NoError(t, err) + require.Equal(t, "Aáa", m.Get("maintainer")) +} diff --git a/testdata/non-utf8.properties b/testdata/non-utf8.properties new file mode 100644 index 0000000..34390ea --- /dev/null +++ b/testdata/non-utf8.properties @@ -0,0 +1 @@ +maintainer=A�a