Integrated handling for filesystem and gzipped binaries

.gitignore

@@ -26,3 +26,16 @@ boards.local.txt
 .vs/
 __vm/
 *.vcxproj*
+
+#Ignore files generated by boards.txt.py
+*.orig
+tools/sdk/ld/backup/*
+
+#Ignore my extra files
+*.odt*
+tools/Gui.py
+tools/Replace2SpceWith4.txt
+*(Kopie)*
+platform.local.txt
+
+

doc/filesystem.rst

@@ -192,8 +192,53 @@ in the specific subdirectory.  This mimics the POSIX behavior for
 directory traversal most C programmers are used to.
 
 
-Uploading files to file system
-------------------------------
+Uploading files to file system (combined firmware and filesystem)
+-----------------------------------------------------------------
+
+Since PullRequest #8266 Filsystem generation is integrated into the
+“compile and/or upload” action of Arduino.
+There is no need to download any additional tool.
+
+Its configuration has 3 entries (“Upload”, "Filesystem" and "Export") in
+the IDE's "Tools" menu, and is described as follows:
+
+Menu-Entry “Upload” lets You choose between:
+- Sketch
+- Filesystem
+- Both
+This Menu controls which parts are uploaded when "Sketch->Upload" (Ctrl-U)
+is selected from the Arduino Menu.
+
+Menu-Entry “Filesystem” lets You choose between:
+- Off
+- LitteFs
+- SPIFFS
+This Menu controls which Fs will be created when "Sketch->Compile" (Ctrl-R)
+is selected from the Arduino Menu. The Fs will always be created inside the
+"export" dir (see below). 
+
+Menu-Entry “Export” let You choose between:
+- Off
+- .bin & .bin.signed)
+- Create & Export gzipped Binaries too
+This Menu controls export & generation of (extra) Sketch-Binaries.
+
+The "export" dir
+................
+After a (successful) compile, the files will be exported to a subdir
+“bin/{variant}” of Your sketch directory
+(“{variant}” replaced with the name of the board as shown behind
+“Tools -> Board:”).
+Signed variants of all Binaries are generated similar to the
+“automatic signing” done for normal sketch binary.
+
+As valid for the "old" tools mentioned below, it is nessesary to place
+files you want to be inside the generated file system into a directory
+named ``data`` inside Your sketch directory.
+
+
+Uploading files to file system (old tools)
+------------------------------------------
 
 *ESP8266FS* is a tool which integrates into the Arduino IDE. It adds a
 menu item to *Tools* menu for uploading the contents of sketch data

platform.txt

@@ -20,6 +20,7 @@ runtime.tools.makecorever={runtime.platform.path}/tools/makecorever.py
 runtime.tools.mkdir={runtime.platform.path}/tools/mkdir.py
 runtime.tools.cp={runtime.platform.path}/tools/cp.py
 runtime.tools.eboot={runtime.platform.path}/bootloaders/eboot/eboot.elf
+runtime.tools.postbuild={runtime.platform.path}/tools/postbuild.py
 
 compiler.warning_flags=-w -Werror=return-type
 compiler.warning_flags.none=-w -Werror=return-type
@@ -129,9 +130,14 @@ recipe.objcopy.hex.1.pattern="{runtime.tools.python3.path}/python3" -I "{runtime
 recipe.objcopy.hex.2.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.signing}" --mode sign --privatekey "{build.source.path}/private.key" --bin "{build.path}/{build.project_name}.bin" --out "{build.path}/{build.project_name}.bin.signed" --legacy "{build.path}/{build.project_name}.bin.legacy_sig"
 recipe.objcopy.hex.3.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.sizes}" --elf "{build.path}/{build.project_name}.elf" --path "{runtime.tools.xtensa-lx106-elf-gcc.path}/bin" --mmu "{build.mmuflags}"
 
+# Post-Build: Handle new Menu-Enties: Filesystem & Binaries
+recipe.hooks.objcopy.postobjcopy.1.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.postbuild}" -fi "{build.Filesystem}" -ex "{build.Export}" -na "{build.project_name}" -bp "{build.path}" -so "{build.source.path}" -va "{build.variant}" -ss "{build.spiffs_start}" -se "{build.spiffs_end}" -sp "{build.spiffs_pagesize}" -sl "{build.spiffs_blocksize}"
+
 ## Save hex
-recipe.output.tmp_file={build.project_name}.bin
-recipe.output.save_file={build.project_name}.{build.variant}.bin
+recipe.output.tmp_file.1={build.project_name}.bin
+recipe.output.save_file.1=bin/{build.variant}/{build.project_name}.bin
+recipe.output.tmp_file.2={build.project_name}.bin.signed
+recipe.output.save_file.2=bin/{build.variant}/{build.project_name}.bin.signed
 
 ## Compute size
 recipe.size.pattern="{compiler.path}{compiler.size.cmd}" -A "{build.path}/{build.project_name}.elf"
@@ -155,11 +161,8 @@ tools.esptool.upload.protocol=esp
 tools.esptool.upload.params.verbose=
 tools.esptool.upload.params.quiet=
 
-# First, potentially perform an erase or nothing
-# Next, do the binary upload
-# Combined in one rule because Arduino doesn't support upload.1.pattern/upload.3.pattern
-tools.esptool.upload.pattern="{cmd}" -I "{runtime.platform.path}/tools/upload.py" --chip esp8266 --port "{serial.port}" --baud "{upload.speed}" "{upload.verbose}" {upload.erase_cmd} {upload.resetmethod} write_flash 0x0 "{build.path}/{build.project_name}.bin"
-tools.esptool.upload.network_pattern="{network_cmd}" -I "{runtime.platform.path}/tools/espota.py" -i "{serial.port}" -p "{network.port}" "--auth={network.password}" -f "{build.path}/{build.project_name}.bin"
+tools.esptool.upload.pattern="{cmd}" -I "{runtime.platform.path}/tools/upload.py" -fi "{build.Filesystem}" -up "{build.Upload}" --port "{serial.port}" --baud "{upload.speed}" {upload.resetmethod} --sk 0x0 "{build.path}/{build.project_name}.bin" --fs {build.spiffs_start} "{build.path}/{build.project_name}" {upload.erase_cmd} "{upload.verbose}"
+tools.esptool.upload.network_pattern="{network_cmd}" -I "{runtime.platform.path}/tools/netUpload.py" -fi "{build.Filesystem}" -up "{build.Upload}" -i "{serial.port}" -p "{network.port}" "--auth={network.password}" --sk "{build.path}/{build.project_name}.bin" --fs "{build.path}/{build.project_name}"
 
 tools.mkspiffs.cmd=mkspiffs
 tools.mkspiffs.cmd.windows=mkspiffs.exe

tools/boards.txt.py

@@ -1032,6 +1032,27 @@
         ( '.build.spiffs_pagesize', '256' ),
         ( '.build.debug_port', '' ),
         ( '.build.debug_level', '' ),
+
+        ( '.menu.Upload.Sketch', 'Sketch' ),
+        ( '.menu.Upload.Sketch.build.Upload', '1' ),
+        ( '.menu.Upload.Filesystem', 'Filesystem' ),
+        ( '.menu.Upload.Filesystem.build.Upload', '2' ),
+        ( '.menu.Upload.Both', 'Both' ),
+        ( '.menu.Upload.Both.build.Upload', '3' ),
+
+        ( '.menu.Filesystem.Off', 'Off' ),
+        ( '.menu.Filesystem.Off.build.Filesystem', '0' ),
+        ( '.menu.Filesystem.LitteFs', 'LitteFs' ),
+        ( '.menu.Filesystem.LitteFs.build.Filesystem', '1' ),
+        ( '.menu.Filesystem.SPIFFS', 'SPIFFS' ),
+        ( '.menu.Filesystem.SPIFFS.build.Filesystem', '2' ),
+
+        ( '.menu.Export.Off', 'Off' ),
+        ( '.menu.Export.Off.build.Export', '0' ),
+        ( '.menu.Export.Binaries', '.bin & .bin.signed' ),
+        ( '.menu.Export.Binaries.build.Export', '1' ),
+        ( '.menu.Export.GzBinaries', 'Create & Export gzipped Binaries too' ),
+        ( '.menu.Export.GzBinaries.build.Export', '2' ),
         ]),
 
     #######################
@@ -1658,6 +1679,9 @@ def all_boards ():
     print('menu.ssl=SSL Support')
     print('menu.mmu=MMU')
     print('menu.non32xfer=Non-32-Bit Access')
+    print('menu.Upload=Upload')
+    print('menu.Filesystem=Filesystem')
+    print('menu.Export=Export')
     print('')
 
     missingboards = []

tools/netUpload.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+# Network-Upload-Wrapper
+# 2021-08-08: G.N.: Written to support (optional) uploading of sketch & filesystem in one step
+
+import os
+import sys
+ToolsDir = os.path.dirname( os.path.realpath( __file__ ) ).replace( '\\', '/' ) + "/" # convert to UNIX format
+try:
+    sys.path.insert( 0, ToolsDir ) # ToolsDir
+    from utilities import * # If this fails, we can't continue and will bomb below
+except Exception as e:
+    sys.stderr.write( '\nImport of utilities.py failed.\n...Is it not next to this %s tool?\n...Exception was: %s\n' % ( __file__, e )  )
+    sys.exit( 2 )
+
+try:
+    sys.path.insert( 0, ToolsDir ) # Add this dir to search path
+    import espota # If this fails, we can't continue and will bomb below
+except Exception:
+    Msg( '\nespota not found next to this netUpload.py tool.' )
+    sys.exit( 2 )
+
+#Args: (original)
+#  As defined in 'platform.txt' for 'tools.esptool.upload.network_pattern'
+#  -i "{serial.port}"
+#  -p "{network.port}"
+#  "--auth={network.password}"
+#  -f "{build.path}/{build.project_name}.bin"
+
+#Args: (new)
+#  As defined in 'platform.txt' for 'tools.esptool.upload.network_pattern'
+#  |                                             possible values/meaning
+#  |                                             |
+#  -fi "{build.Filesystem}"                      0-4
+#  -i "{serial.port}"
+#  -p "{network.port}"
+#  "--auth={network.password}"
+#  --sk "{build.path}/{build.project_name}.bin"  <filePath for sketch-bin>
+#  --fs "{build.path}/{build.project_name}"      <filePath for fs.bin, without ext>
+
+def parse_args( argsIn ):
+    parser = argparse.ArgumentParser( description = 'Network-Upload-Wrapper for Arduino esp8266' )
+    parser.add_argument( '-fi', '--Filesystem', type = int, default = 0, help = '0: Off, 1: LitteFs: Create & Upload, 2: LitteFs: Create only, 3: SPIFFS: Create & Upload, 4: SPIFFS: Create only' )
+    parser.add_argument( '-i', '--port', type = str, required = True, help = 'path to serial device' )
+    parser.add_argument( '-p', '--netPort', type = str, required = True, help = 'network port' )
+    parser.add_argument( '--auth', type = str, help = 'network password' )
+    parser.add_argument( '--sk', type = str, required = True, help = 'path_path_to_sketch_binary' )
+    parser.add_argument( '--fs', type = str, help = 'path_to_fs_binary_without_ext' )
+    global Args
+    Args = parser.parse_args( argsIn )
+
+def main( argsIn = None ):
+    """
+    Main function for netUpload
+
+    argsIn - Optional override for default arguments parsing (that uses sys.argv), can be a list of custom arguments.
+    Arguments and their values need to be added as individual items to the list e.g. "-b 115200" thus becomes ['-b', '115200'].
+    """
+
+    parse_args( argsIn )
+
+    if not os.path.exists( Args.port ):
+      Msg( "Port: '%s', does not exist!\n...Upload aborted" % Args.port )
+      sys.exit( 1 )
+
+    baseArgs = [ '-i', Args.port, '-p', Args.netPort, '--auth', Args.auth ]
+
+    sketchArgs = baseArgs + [ '-f', Args.sk ]
+
+    Msg( "Uploading Binaries..." )
+
+    #Debug( str( sketchArgs ) )
+    esptool.main( sketchArgs )
+    filesUploaded = [ Args.sk ]
+
+    if Args.Filesystem == 1 or Args.Filesystem == 3:
+        fsArgs = baseArgs + [ '-s', '-f' ]
+        if Args.Filesystem == 1:
+            fsArgs = baseArgs + [ "%s.littlefs" % Args.fs ]
+        elif Args.Filesystem == 3:
+            fsArgs = baseArgs + [ "%s.spiffs" % Args.fs ]
+
+        #Debug( str( fsArgs ) )
+        esptool.main( fsArgs )
+        filesUploaded += [ Args.fs ]
+
+    Msg( '...files uploaded: %s' % ", ".join( filesUploaded ) )
+
+def main_():
+    try:
+        main()
+    except RuntimeError as e:
+        Msg( '\nA RuntimeError error occurred: %s' % e )
+        sys.exit( 2 )
+
+if __name__ == '__main__':
+    main_()
+