From bc1e64219e5bb118fb7b9a28e2cf83b1ff16be7c Mon Sep 17 00:00:00 2001
From: M Hightower <27247790+mhightower83@users.noreply.github.com>
Date: Thu, 16 Feb 2023 15:42:03 -0800
Subject: [PATCH 1/8] Moved up os.makedirs for core directory. Strange problem
 creating files with file paths with spaces.

Added try/except catch on main to commit print buffer for context of traceback.
---
 tools/mkbuildoptglobals.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py
index 9b5fb65ceb..7f49774f7b 100644
--- a/tools/mkbuildoptglobals.py
+++ b/tools/mkbuildoptglobals.py
@@ -187,6 +187,7 @@
 import glob
 import os
 import platform
+import traceback
 import sys
 import textwrap
 import time
@@ -305,7 +306,6 @@ def add_include_line(build_opt_fqfn, include_fqfn):
     with open(build_opt_fqfn, 'a', encoding="utf-8") as build_opt:
         build_opt.write('-include "' + include_fqfn.replace('\\', '\\\\') + '"\n')
 
-
 def extract_create_build_opt_file(globals_h_fqfn, file_name, build_opt_fqfn):
     """
     Extract the embedded build.opt from Sketch.ino.globals.h into build
@@ -655,6 +655,10 @@ def main():
     print_dbg(f"first_time:             {first_time}")
     print_dbg(f"use_aggressive_caching_workaround: {use_aggressive_caching_workaround}")
 
+    if not os.path.exists(build_path_core):
+        os.makedirs(build_path_core)
+        print_msg("Clean build, created dir " + build_path_core)
+
     if first_time or \
     not use_aggressive_caching_workaround or \
     not os.path.exists(commonhfile_fqfn):
@@ -667,10 +671,6 @@ def main():
         touch(commonhfile_fqfn)
         print_err(f"Neutralized future timestamp on build file: {commonhfile_fqfn}")
 
-    if not os.path.exists(build_path_core):
-        os.makedirs(build_path_core)
-        print_msg("Clean build, created dir " + build_path_core)
-
     if os.path.exists(source_globals_h_fqfn):
         print_msg("Using global include from " + source_globals_h_fqfn)
 
@@ -750,4 +750,10 @@ def main():
     handle_error(0)   # commit print buffer
 
 if __name__ == '__main__':
-    sys.exit(main())
+    rc = 1
+    try:
+        rc = main()
+    except:
+        print_err(traceback.format_exc())
+        handle_error(0)
+    sys.exit(rc)

From 338159ffaaf06cb0e6d92999d8bebf8584218f5a Mon Sep 17 00:00:00 2001
From: M Hightower <27247790+mhightower83@users.noreply.github.com>
Date: Sun, 19 Feb 2023 21:47:26 -0800
Subject: [PATCH 2/8] Use default locale encode for writing file names in
 build.opt

Continue using "utf-8" to read or write "C/C++" source filess.

Tested on Windows 10 (en-US) with Arduino IDE 2.0.3 Under an
account with a diacritic charcter in the user ID path.

Needs testing on Japanese Windows
---
 tools/mkbuildoptglobals.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py
index 7f49774f7b..a6671e6956 100644
--- a/tools/mkbuildoptglobals.py
+++ b/tools/mkbuildoptglobals.py
@@ -192,6 +192,8 @@
 import textwrap
 import time
 
+import locale
+
 # Need to work on signature line used for match to avoid conflicts with
 # existing embedded documentation methods.
 build_opt_signature = "/*@create-file:build.opt@"
@@ -296,14 +298,22 @@ def copy_create_build_file(source_fqfn, build_target_fqfn):
                 pass
     return True     # file changed
 
-
 def add_include_line(build_opt_fqfn, include_fqfn):
+    # Given that GCC will insert (or process as if interted) these lines into
+    # the command line. I assume that the contents of @file need to be encoded
+    # to match that of the shell running GCC runs. I am not 100% sure this API
+    # gives me that, but it appears to work.
+    #
+    # However, elsewhere when dealing with source code we continue to use 'utf-8',
+    # ref. https://gcc.gnu.org/onlinedocs/gcc-4.1.2/cpp/Character-sets.html
+    shell_encoding = locale.getdefaultlocale()[1]
     if not os.path.exists(include_fqfn):
         # If file is missing, we need an place holder
-        with open(include_fqfn, 'w', encoding="utf-8"):
+        with open(include_fqfn, 'w', encoding=shell_encoding):
             pass
-        print("add_include_line: Created " + include_fqfn)
-    with open(build_opt_fqfn, 'a', encoding="utf-8") as build_opt:
+        print_msg("add_include_line: Created " + include_fqfn)
+
+    with open(build_opt_fqfn, 'a', encoding=shell_encoding) as build_opt:
         build_opt.write('-include "' + include_fqfn.replace('\\', '\\\\') + '"\n')
 
 def extract_create_build_opt_file(globals_h_fqfn, file_name, build_opt_fqfn):
@@ -611,6 +621,9 @@ def main():
     global debug_enabled
     num_include_lines = 1
 
+    def_locale = locale.getdefaultlocale()
+    print_msg(f'default locale: {def_locale}')
+
     args = parse_args()
     debug_enabled = args.debug
     runtime_ide_path = os.path.normpath(args.runtime_ide_path)

From 39bacae8d883ff67251f37406e76665fb48c0b22 Mon Sep 17 00:00:00 2001
From: M Hightower <27247790+mhightower83@users.noreply.github.com>
Date: Mon, 20 Feb 2023 12:09:20 -0800
Subject: [PATCH 3/8] diacritics in paths specified in <Sketch>.ino.globals.h
 should now work.

---
 tools/mkbuildoptglobals.py | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py
index a6671e6956..7844d046bb 100644
--- a/tools/mkbuildoptglobals.py
+++ b/tools/mkbuildoptglobals.py
@@ -193,6 +193,7 @@
 import time
 
 import locale
+import codecs
 
 # Need to work on signature line used for match to avoid conflicts with
 # existing embedded documentation methods.
@@ -204,6 +205,7 @@
 err_print_flag = False
 msg_print_buf = ""
 debug_enabled = False
+default_locale = (None, None)
 
 # Issues trying to address through buffered printing
 # 1. Arduino IDE 2.0 RC5 does not show stderr text in color. Text printed does
@@ -257,6 +259,7 @@ def handle_error(err_no):
     # on err_no != 0, commit print buffer to stderr and sys exist with err_no
     global msg_print_buf
     global err_print_flag
+    global default_locale
     if len(msg_print_buf):
         if err_no or err_print_flag:
             fd = sys.stderr
@@ -299,21 +302,14 @@ def copy_create_build_file(source_fqfn, build_target_fqfn):
     return True     # file changed
 
 def add_include_line(build_opt_fqfn, include_fqfn):
-    # Given that GCC will insert (or process as if interted) these lines into
-    # the command line. I assume that the contents of @file need to be encoded
-    # to match that of the shell running GCC runs. I am not 100% sure this API
-    # gives me that, but it appears to work.
-    #
-    # However, elsewhere when dealing with source code we continue to use 'utf-8',
-    # ref. https://gcc.gnu.org/onlinedocs/gcc-4.1.2/cpp/Character-sets.html
-    shell_encoding = locale.getdefaultlocale()[1]
+    global default_locale
     if not os.path.exists(include_fqfn):
         # If file is missing, we need an place holder
-        with open(include_fqfn, 'w', encoding=shell_encoding):
+        with open(include_fqfn, 'w', encoding=default_locale[1]):
             pass
         print_msg("add_include_line: Created " + include_fqfn)
 
-    with open(build_opt_fqfn, 'a', encoding=shell_encoding) as build_opt:
+    with open(build_opt_fqfn, 'a', encoding=default_locale[1]) as build_opt:
         build_opt.write('-include "' + include_fqfn.replace('\\', '\\\\') + '"\n')
 
 def extract_create_build_opt_file(globals_h_fqfn, file_name, build_opt_fqfn):
@@ -323,8 +319,9 @@ def extract_create_build_opt_file(globals_h_fqfn, file_name, build_opt_fqfn):
     copy of Sketch.ino.globals.h.
     """
     global build_opt_signature
+    global default_locale
 
-    build_opt = open(build_opt_fqfn, 'w', encoding="utf-8")
+    build_opt = open(build_opt_fqfn, 'w', encoding=default_locale[1])
     if not os.path.exists(globals_h_fqfn) or (0 == os.path.getsize(globals_h_fqfn)):
         build_opt.close()
         return False
@@ -619,10 +616,18 @@ def main():
     global build_opt_signature
     global docs_url
     global debug_enabled
+    global default_locale
     num_include_lines = 1
 
-    def_locale = locale.getdefaultlocale()
-    print_msg(f'default locale: {def_locale}')
+    # Given that GCC will handle lines from an @file as if they were on
+    # the command line. I assume that the contents of @file need to be encoded
+    # to match that of the shell running GCC runs. I am not 100% sure this API
+    # gives me that, but it appears to work.
+    #
+    # However, elsewhere when dealing with source code we continue to use 'utf-8',
+    # ref. https://gcc.gnu.org/onlinedocs/gcc-4.1.2/cpp/Character-sets.html
+    default_locale = locale.getdefaultlocale()
+    print_msg(f'default locale: {default_locale}')
 
     args = parse_args()
     debug_enabled = args.debug

From 94a8c4e6befe55d23bffd53d8a077a2e7dc57d79 Mon Sep 17 00:00:00 2001
From: M Hightower <27247790+mhightower83@users.noreply.github.com>
Date: Mon, 20 Feb 2023 20:33:29 -0800
Subject: [PATCH 4/8] Fix diacritics in path problem with sizes.py

---
 tools/sizes.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tools/sizes.py b/tools/sizes.py
index 3bbc537faf..08b5609c33 100755
--- a/tools/sizes.py
+++ b/tools/sizes.py
@@ -72,8 +72,10 @@ def get_segment_sizes(elf, path, mmu):
         (".bss", "BSS"),
     )
 
+    import locale
+    shell_encoding = locale.getdefaultlocale()[1]
     cmd = [os.path.join(path, "xtensa-lx106-elf-size"), "-A", elf]
-    with subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True) as proc:
+    with subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True, encoding=shell_encoding) as proc:
         lines = proc.stdout.readlines()
         for line in lines:
             words = line.split()

From 287053b6595bc3bbccdb95ec7ff2a112c3125c0a Mon Sep 17 00:00:00 2001
From: M Hightower <27247790+mhightower83@users.noreply.github.com>
Date: Tue, 21 Feb 2023 12:51:33 -0800
Subject: [PATCH 5/8] Improve get encoding logic for mkbuildoptglobals.py

---
 tools/mkbuildoptglobals.py | 69 ++++++++++++++++++++++++++++++++------
 1 file changed, 58 insertions(+), 11 deletions(-)

diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py
index 7844d046bb..b5cf3b4520 100644
--- a/tools/mkbuildoptglobals.py
+++ b/tools/mkbuildoptglobals.py
@@ -205,7 +205,7 @@
 err_print_flag = False
 msg_print_buf = ""
 debug_enabled = False
-default_locale = (None, None)
+default_encoding = None
 
 # Issues trying to address through buffered printing
 # 1. Arduino IDE 2.0 RC5 does not show stderr text in color. Text printed does
@@ -259,7 +259,6 @@ def handle_error(err_no):
     # on err_no != 0, commit print buffer to stderr and sys exist with err_no
     global msg_print_buf
     global err_print_flag
-    global default_locale
     if len(msg_print_buf):
         if err_no or err_print_flag:
             fd = sys.stderr
@@ -302,14 +301,14 @@ def copy_create_build_file(source_fqfn, build_target_fqfn):
     return True     # file changed
 
 def add_include_line(build_opt_fqfn, include_fqfn):
-    global default_locale
+    global default_encoding
     if not os.path.exists(include_fqfn):
         # If file is missing, we need an place holder
-        with open(include_fqfn, 'w', encoding=default_locale[1]):
+        with open(include_fqfn, 'w', encoding=default_encoding):
             pass
         print_msg("add_include_line: Created " + include_fqfn)
 
-    with open(build_opt_fqfn, 'a', encoding=default_locale[1]) as build_opt:
+    with open(build_opt_fqfn, 'a', encoding=default_encoding) as build_opt:
         build_opt.write('-include "' + include_fqfn.replace('\\', '\\\\') + '"\n')
 
 def extract_create_build_opt_file(globals_h_fqfn, file_name, build_opt_fqfn):
@@ -319,9 +318,9 @@ def extract_create_build_opt_file(globals_h_fqfn, file_name, build_opt_fqfn):
     copy of Sketch.ino.globals.h.
     """
     global build_opt_signature
-    global default_locale
+    global default_encoding
 
-    build_opt = open(build_opt_fqfn, 'w', encoding=default_locale[1])
+    build_opt = open(build_opt_fqfn, 'w', encoding=default_encoding)
     if not os.path.exists(globals_h_fqfn) or (0 == os.path.getsize(globals_h_fqfn)):
         build_opt.close()
         return False
@@ -612,11 +611,53 @@ def parse_args():
     # ref nargs='*'', https://stackoverflow.com/a/4480202
     # ref no '--n' parameter, https://stackoverflow.com/a/21998252
 
+
+def get_encoding():
+    try:
+        from locale import getencoding
+    except ImportError:
+        # https://stackoverflow.com/a/23743499
+        return locale.getpreferredencoding(False) or locale.getdefaultlocale()[1]
+
+    return locale.getencoding()
+
+
+def show_value(desc, value):
+    print_dbg(f'{desc:<40} {value}')
+    return
+
+def locale_dbg():
+    show_value("get_encoding()", get_encoding())
+    show_value("locale.getdefaultlocale()", locale.getdefaultlocale())
+    show_value('sys.getfilesystemencoding()', sys.getfilesystemencoding())
+    show_value("sys.getdefaultencoding()", sys.getdefaultencoding())
+    show_value("locale.getpreferredencoding(False)", locale.getpreferredencoding(False))
+    try:
+        show_value("locale.getpreferredencoding()", locale.getpreferredencoding())
+    except:
+        pass
+    show_value("sys.stdout.encoding", sys.stdout.encoding)
+
+    # use current setting
+    show_value("locale.setlocale(locale.LC_ALL, None)", locale.setlocale(locale.LC_ALL, None))
+    try:
+        show_value("locale.getencoding()", locale.getencoding())
+    except:
+        pass
+    show_value("locale.getlocale()", locale.getlocale())
+
+    # use user setting
+    show_value("locale.setlocale(locale.LC_ALL, '')", locale.setlocale(locale.LC_ALL, ''))
+    # show_value("locale.getencoding()", locale.getencoding())
+    show_value("locale.getlocale()", locale.getlocale())
+    return
+
+
 def main():
     global build_opt_signature
     global docs_url
     global debug_enabled
-    global default_locale
+    global default_encoding
     num_include_lines = 1
 
     # Given that GCC will handle lines from an @file as if they were on
@@ -625,9 +666,8 @@ def main():
     # gives me that, but it appears to work.
     #
     # However, elsewhere when dealing with source code we continue to use 'utf-8',
-    # ref. https://gcc.gnu.org/onlinedocs/gcc-4.1.2/cpp/Character-sets.html
-    default_locale = locale.getdefaultlocale()
-    print_msg(f'default locale: {default_locale}')
+    # ref. https://gcc.gnu.org/onlinedocs/cpp/Character-sets.html
+    default_encoding = get_encoding()
 
     args = parse_args()
     debug_enabled = args.debug
@@ -641,6 +681,13 @@ def main():
     build_path_core, build_opt_name = os.path.split(build_opt_fqfn)
     globals_h_fqfn = os.path.join(build_path_core, globals_name)
 
+    if debug_enabled:
+        locale_dbg()
+
+    default_locale = locale.getdefaultlocale()
+    print_msg(f'default locale:         {default_locale}')
+    print_msg(f'default_encoding:       {default_encoding}')
+
     print_dbg(f"runtime_ide_path:       {runtime_ide_path}")
     print_dbg(f"runtime_ide_version:    {args.runtime_ide_version}")
     print_dbg(f"build_path:             {build_path}")

From c3360fa1bcdeb5c1125d0796c9aac33c69cdac77 Mon Sep 17 00:00:00 2001
From: M Hightower <27247790+mhightower83@users.noreply.github.com>
Date: Wed, 22 Feb 2023 07:25:18 -0800
Subject: [PATCH 6/8] Update tools/mkbuildoptglobals.py

Co-authored-by: Max Prokhorov <prokhorov.max@outlook.com>
---
 tools/mkbuildoptglobals.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py
index b5cf3b4520..ac0b41ad5d 100644
--- a/tools/mkbuildoptglobals.py
+++ b/tools/mkbuildoptglobals.py
@@ -612,14 +612,13 @@ def parse_args():
     # ref no '--n' parameter, https://stackoverflow.com/a/21998252
 
 
-def get_encoding():
-    try:
-        from locale import getencoding
-    except ImportError:
-        # https://stackoverflow.com/a/23743499
-        return locale.getpreferredencoding(False) or locale.getdefaultlocale()[1]
-
-    return locale.getencoding()
+# retrieve *system* encoding, not the one used by python internally
+if sys.version_info >= (3, 11):
+    def get_encoding():
+        return locale.getencoding()
+else:
+    def get_encoding():
+        return locale.getdefaultlocale()[1]
 
 
 def show_value(desc, value):

From 78d251085229c0217b1ae238a585af252f8f7474 Mon Sep 17 00:00:00 2001
From: M Hightower <27247790+mhightower83@users.noreply.github.com>
Date: Wed, 22 Feb 2023 07:25:30 -0800
Subject: [PATCH 7/8] Update tools/mkbuildoptglobals.py

Co-authored-by: Max Prokhorov <prokhorov.max@outlook.com>
---
 tools/mkbuildoptglobals.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py
index ac0b41ad5d..3348964bce 100644
--- a/tools/mkbuildoptglobals.py
+++ b/tools/mkbuildoptglobals.py
@@ -193,7 +193,6 @@
 import time
 
 import locale
-import codecs
 
 # Need to work on signature line used for match to avoid conflicts with
 # existing embedded documentation methods.

From 5c66f9f4430cd9f1e31e118474f9de6bd6c84b14 Mon Sep 17 00:00:00 2001
From: M Hightower <27247790+mhightower83@users.noreply.github.com>
Date: Wed, 22 Feb 2023 07:33:30 -0800
Subject: [PATCH 8/8] update size.py

---
 tools/sizes.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/tools/sizes.py b/tools/sizes.py
index 08b5609c33..34ebe30755 100755
--- a/tools/sizes.py
+++ b/tools/sizes.py
@@ -21,9 +21,20 @@
 import os
 import subprocess
 import sys
+import locale
 
 sys.stdout = sys.stderr
 
+
+# retrieve *system* encoding, not the one used by python internally
+if sys.version_info >= (3, 11):
+    def get_encoding():
+        return locale.getencoding()
+else:
+    def get_encoding():
+        return locale.getdefaultlocale()[1]
+
+
 def get_segment_sizes(elf, path, mmu):
     iram_size = 0
     iheap_size = 0
@@ -72,10 +83,8 @@ def get_segment_sizes(elf, path, mmu):
         (".bss", "BSS"),
     )
 
-    import locale
-    shell_encoding = locale.getdefaultlocale()[1]
     cmd = [os.path.join(path, "xtensa-lx106-elf-size"), "-A", elf]
-    with subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True, encoding=shell_encoding) as proc:
+    with subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True, encoding=get_encoding()) as proc:
         lines = proc.stdout.readlines()
         for line in lines:
             words = line.split()