Merge pull request #213 from facchinm/archiveCompiledCore

Archive compiled core and use on subsequent compilations

Author: cmaglie

src/arduino.cc/arduino-builder/main.go

@@ -64,6 +64,7 @@ const FLAG_FQBN = "fqbn"
 const FLAG_IDE_VERSION = "ide-version"
 const FLAG_CORE_API_VERSION = "core-api-version"
 const FLAG_BUILD_PATH = "build-path"
+const FLAG_BUILD_CACHE = "build-cache"
 const FLAG_VERBOSE = "verbose"
 const FLAG_QUIET = "quiet"
 const FLAG_DEBUG_LEVEL = "debug-level"
@@ -126,6 +127,7 @@ var fqbnFlag *string
 var coreAPIVersionFlag *string
 var ideVersionFlag *string
 var buildPathFlag *string
+var buildCachePathFlag *string
 var verboseFlag *bool
 var quietFlag *bool
 var debugLevelFlag *int
@@ -148,6 +150,7 @@ func init() {
 	coreAPIVersionFlag = flag.String(FLAG_CORE_API_VERSION, "10600", "version of core APIs (used to populate ARDUINO #define)")
 	ideVersionFlag = flag.String(FLAG_IDE_VERSION, "10600", "[deprecated] use '"+FLAG_CORE_API_VERSION+"' instead")
 	buildPathFlag = flag.String(FLAG_BUILD_PATH, "", "build path")
+	buildCachePathFlag = flag.String(FLAG_BUILD_CACHE, "", "builds of 'core.a' are saved into this folder to be cached and reused")
 	verboseFlag = flag.Bool(FLAG_VERBOSE, false, "if 'true' prints lots of stuff")
 	quietFlag = flag.Bool(FLAG_QUIET, false, "if 'true' doesn't print any warnings or progress or whatever")
 	debugLevelFlag = flag.Int(FLAG_DEBUG_LEVEL, builder.DEFAULT_DEBUG_LEVEL, "Turns on debugging messages. The higher, the chattier")
@@ -256,6 +259,25 @@ func main() {
 	}
 	ctx.BuildPath = buildPath
 
+	// FLAG_BUILD_CACHE
+	buildCachePath, err := gohasissues.Unquote(*buildCachePathFlag)
+	if err != nil {
+		printCompleteError(err)
+	}
+	if buildCachePath != "" {
+		_, err := os.Stat(buildCachePath)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+
+		err = utils.EnsureFolderExists(buildCachePath)
+		if err != nil {
+			printCompleteError(err)
+		}
+	}
+	ctx.BuildCachePath = buildCachePath
+
 	// FLAG_VID_PID
 	if *vidPidFlag != "" {
 		ctx.USBVidPid = *vidPidFlag

src/arduino.cc/builder/add_additional_entries_to_context.go

@@ -30,10 +30,11 @@
 package builder
 
 import (
+	"path/filepath"
+
 	"arduino.cc/builder/constants"
 	"arduino.cc/builder/i18n"
 	"arduino.cc/builder/types"
-	"path/filepath"
 )
 
 type AddAdditionalEntriesToContext struct{}
@@ -64,6 +65,15 @@ func (s *AddAdditionalEntriesToContext) Run(ctx *types.Context) error {
 		ctx.CoreBuildPath = coreBuildPath
 	}
 
+	if ctx.BuildCachePath != "" {
+		coreBuildCachePath, err := filepath.Abs(filepath.Join(ctx.BuildCachePath, constants.FOLDER_CORE))
+		if err != nil {
+			return i18n.WrapError(err)
+		}
+
+		ctx.CoreBuildCachePath = coreBuildCachePath
+	}
+
 	if ctx.WarningsLevel == "" {
 		ctx.WarningsLevel = DEFAULT_WARNINGS_LEVEL
 	}

src/arduino.cc/builder/builder_utils/utils.go

@@ -32,6 +32,7 @@ package builder_utils
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -116,6 +117,34 @@ func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]str
 	return sources, nil
 }
 
+func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) {
+	files, err := utils.ReadDirFiltered(sourcePath, utils.FilterFiles())
+	if err != nil {
+		return nil, i18n.WrapError(err)
+	}
+	var sources []string
+	for _, file := range files {
+		sources = append(sources, filepath.Join(sourcePath, file.Name()))
+	}
+
+	if recurse {
+		folders, err := utils.ReadDirFiltered(sourcePath, utils.FilterDirs)
+		if err != nil {
+			return nil, i18n.WrapError(err)
+		}
+
+		for _, folder := range folders {
+			otherSources, err := findAllFilesInFolder(filepath.Join(sourcePath, folder.Name()), recurse)
+			if err != nil {
+				return nil, i18n.WrapError(err)
+			}
+			sources = append(sources, otherSources...)
+		}
+	}
+
+	return sources, nil
+}
+
 func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
 	for _, source := range sources {
 		objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
@@ -258,6 +287,28 @@ func nonEmptyString(s string) bool {
 	return s != constants.EMPTY_STRING
 }
 
+func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile string) bool {
+
+	targetFileStat, err := os.Stat(targetFile)
+	if err == nil {
+		files, err := findAllFilesInFolder(corePath, true)
+		if err != nil {
+			return true
+		}
+		for _, file := range files {
+			fileStat, err := os.Stat(file)
+			if err != nil || fileStat.ModTime().After(targetFileStat.ModTime()) {
+				return true
+			}
+		}
+		if targetCorePath != constants.EMPTY_STRING && !strings.EqualFold(corePath, targetCorePath) {
+			return CoreOrReferencedCoreHasChanged(targetCorePath, constants.EMPTY_STRING, targetFile)
+		}
+		return false
+	}
+	return true
+}
+
 func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map, verbose bool, logger i18n.Logger) (string, error) {
 	archiveFilePath := filepath.Join(buildPath, archiveFile)
 
@@ -366,3 +417,59 @@ func ExecRecipeCollectStdErr(buildProperties properties.Map, recipe string, remo
 func RemoveHyphenMDDFlagFromGCCCommandLine(buildProperties properties.Map) {
 	buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS] = strings.Replace(buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS], "-MMD", "", -1)
 }
+
+// CopyFile copies the contents of the file named src to the file named
+// by dst. The file will be created if it does not already exist. If the
+// destination file exists, all it's contents will be replaced by the contents
+// of the source file. The file mode will be copied from the source and
+// the copied data is synced/flushed to stable storage.
+func CopyFile(src, dst string) (err error) {
+	in, err := os.Open(src)
+	if err != nil {
+		return
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return
+	}
+	defer func() {
+		if e := out.Close(); e != nil {
+			err = e
+		}
+	}()
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return
+	}
+
+	err = out.Sync()
+	if err != nil {
+		return
+	}
+
+	si, err := os.Stat(src)
+	if err != nil {
+		return
+	}
+	err = os.Chmod(dst, si.Mode())
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// GetCachedCoreArchiveFileName returns the filename to be used to store
+// the global cached core.a.
+func GetCachedCoreArchiveFileName(fqbn, coreFolder string) string {
+	fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1)
+	fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1)
+	if absCoreFolder, err := filepath.Abs(coreFolder); err == nil {
+		coreFolder = absCoreFolder
+	} // silently continue if absolute path can't be detected
+	hash := utils.MD5Sum([]byte(coreFolder))
+	return "core_" + fqbnToUnderscore + "_" + hash + ".a"
+}

src/arduino.cc/builder/constants/constants.go

@@ -55,6 +55,7 @@ const BUILD_PROPERTIES_EXTRA_TIME_DST = "extra.time.dst"
 const BUILD_PROPERTIES_EXTRA_TIME_LOCAL = "extra.time.local"
 const BUILD_PROPERTIES_EXTRA_TIME_UTC = "extra.time.utc"
 const BUILD_PROPERTIES_EXTRA_TIME_ZONE = "extra.time.zone"
+const BUILD_PROPERTIES_FQBN = "build.fqbn"
 const BUILD_PROPERTIES_INCLUDES = "includes"
 const BUILD_PROPERTIES_OBJECT_FILE = "object_file"
 const BUILD_PROPERTIES_OBJECT_FILES = "object_files"

src/arduino.cc/builder/phases/core_builder.go

@@ -30,6 +30,8 @@
 package phases
 
 import (
+	"path/filepath"
+
 	"arduino.cc/builder/builder_utils"
 	"arduino.cc/builder/constants"
 	"arduino.cc/builder/i18n"
@@ -42,6 +44,7 @@ type CoreBuilder struct{}
 
 func (s *CoreBuilder) Run(ctx *types.Context) error {
 	coreBuildPath := ctx.CoreBuildPath
+	coreBuildCachePath := ctx.CoreBuildCachePath
 	buildProperties := ctx.BuildProperties
 	verbose := ctx.Verbose
 	warningsLevel := ctx.WarningsLevel
@@ -52,7 +55,14 @@ func (s *CoreBuilder) Run(ctx *types.Context) error {
 		return i18n.WrapError(err)
 	}
 
-	archiveFile, objectFiles, err := compileCore(coreBuildPath, buildProperties, verbose, warningsLevel, logger)
+	if coreBuildCachePath != "" {
+		err := utils.EnsureFolderExists(coreBuildCachePath)
+		if err != nil {
+			return i18n.WrapError(err)
+		}
+	}
+
+	archiveFile, objectFiles, err := compileCore(coreBuildPath, coreBuildCachePath, buildProperties, verbose, warningsLevel, logger)
 	if err != nil {
 		return i18n.WrapError(err)
 	}
@@ -63,10 +73,12 @@ func (s *CoreBuilder) Run(ctx *types.Context) error {
 	return nil
 }
 
-func compileCore(buildPath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) {
+func compileCore(buildPath string, buildCachePath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) {
 	coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH]
 	variantFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH]
 
+	targetCoreFolder := buildProperties[constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH]
+
 	includes := []string{}
 	includes = append(includes, coreFolder)
 	if variantFolder != constants.EMPTY_STRING {
@@ -84,6 +96,24 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool,
 		}
 	}
 
+	// Recreate the archive if ANY of the core files (including platform.txt) has changed
+	realCoreFolder := utils.GetParentFolder(coreFolder, 2)
+
+	var targetArchivedCore string
+	if buildCachePath != "" {
+		archivedCoreName := builder_utils.GetCachedCoreArchiveFileName(buildProperties[constants.BUILD_PROPERTIES_FQBN], realCoreFolder)
+		targetArchivedCore = filepath.Join(buildCachePath, archivedCoreName)
+		canUseArchivedCore := !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore)
+
+		if canUseArchivedCore {
+			// use archived core
+			if verbose {
+				logger.Println(constants.LOG_LEVEL_INFO, "Using precompiled core")
+			}
+			return targetArchivedCore, variantObjectFiles, nil
+		}
+	}
+
 	coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger)
 	if err != nil {
 		return "", nil, i18n.WrapError(err)
@@ -94,5 +124,11 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool,
 		return "", nil, i18n.WrapError(err)
 	}
 
+	// archive core.a
+	if targetArchivedCore != "" {
+		logger.Println(constants.LOG_LEVEL_DEBUG, "Archiving built core (caching) in: "+targetArchivedCore)
+		builder_utils.CopyFile(archiveFile, targetArchivedCore)
+	}
+
 	return archiveFile, variantObjectFiles, nil
 }

src/arduino.cc/builder/setup_build_properties.go

@@ -70,6 +70,7 @@ func (s *SetupBuildProperties) Run(ctx *types.Context) error {
 	buildProperties[constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH] = targetPlatform.Folder
 	buildProperties[constants.BUILD_PROPERTIES_RUNTIME_HARDWARE_PATH] = filepath.Join(targetPlatform.Folder, "..")
 	buildProperties[constants.BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = ctx.ArduinoAPIVersion
+	buildProperties[constants.BUILD_PROPERTIES_FQBN] = ctx.FQBN
 	buildProperties[constants.IDE_VERSION] = ctx.ArduinoAPIVersion
 	buildProperties[constants.BUILD_PROPERTIES_RUNTIME_OS] = utils.PrettyOSName()