Improved compile speed by running multi-threaded library discovery.

commands/service_compile.go

@@ -165,7 +165,7 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 	if buildPathArg := req.GetBuildPath(); buildPathArg != "" {
 		buildPath = paths.New(req.GetBuildPath()).Canonical()
 		if in, _ := buildPath.IsInsideDir(sk.FullPath); in && buildPath.IsDir() {
-			if sk.AdditionalFiles, err = removeBuildFromSketchFiles(sk.AdditionalFiles, buildPath); err != nil {
+			if sk.AdditionalFiles, err = removeBuildPathFromSketchFiles(sk.AdditionalFiles, buildPath); err != nil {
 				return err
 			}
 		}
@@ -214,10 +214,6 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 		return err
 	}
 
-	actualPlatform := buildPlatform
-	otherLibrariesDirs := paths.NewPathList(req.GetLibraries()...)
-	otherLibrariesDirs.Add(s.settings.LibrariesDir())
-
 	var libsManager *librariesmanager.LibrariesManager
 	if pme.GetProfile() != nil {
 		libsManager = lm
@@ -240,6 +236,11 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 			Message: &rpc.CompileResponse_Progress{Progress: p},
 		})
 	}
+
+	librariesDirs := paths.NewPathList(req.GetLibraries()...) // Array of collection of libraries directories
+	librariesDirs.Add(s.settings.LibrariesDir())
+	libraryDirs := paths.NewPathList(req.GetLibrary()...) // Array of single-library directories
+
 	sketchBuilder, err := builder.NewBuilder(
 		ctx,
 		sk,
@@ -251,16 +252,16 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 		int(req.GetJobs()),
 		req.GetBuildProperties(),
 		s.settings.HardwareDirectories(),
-		otherLibrariesDirs,
+		librariesDirs,
 		s.settings.IDEBuiltinLibrariesDir(),
 		fqbn,
 		req.GetClean(),
 		req.GetSourceOverride(),
 		req.GetCreateCompilationDatabaseOnly(),
-		targetPlatform, actualPlatform,
+		targetPlatform, buildPlatform,
 		req.GetSkipLibrariesDiscovery(),
 		libsManager,
-		paths.NewPathList(req.GetLibrary()...),
+		libraryDirs,
 		outStream, errStream, req.GetVerbose(), req.GetWarnings(),
 		progressCB,
 		pme.GetEnvVarsForSpawnedProcess(),
@@ -434,15 +435,15 @@ func maybePurgeBuildCache(compilationsBeforePurge uint, cacheTTL time.Duration)
 	buildcache.New(paths.TempDir().Join("arduino", "sketches")).Purge(cacheTTL)
 }
 
-// removeBuildFromSketchFiles removes the files contained in the build directory from
+// removeBuildPathFromSketchFiles removes the files contained in the build directory from
 // the list of the sketch files
-func removeBuildFromSketchFiles(files paths.PathList, build *paths.Path) (paths.PathList, error) {
+func removeBuildPathFromSketchFiles(files paths.PathList, build *paths.Path) (paths.PathList, error) {
 	var res paths.PathList
 	ignored := false
 	for _, file := range files {
 		if isInside, _ := file.IsInsideDir(build); !isInside {
-			res = append(res, file)
-		} else if !ignored {
+			res.Add(file)
+		} else {
 			ignored = true
 		}
 	}

commands/service_monitor.go

@@ -27,7 +27,6 @@ import (
 	"github.com/arduino/arduino-cli/internal/arduino/cores"
 	"github.com/arduino/arduino-cli/internal/arduino/cores/packagemanager"
 	pluggableMonitor "github.com/arduino/arduino-cli/internal/arduino/monitor"
-	"github.com/arduino/arduino-cli/internal/i18n"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	"github.com/arduino/go-properties-orderedmap"
 	"github.com/djherbis/buffer"
@@ -264,10 +263,7 @@ func findMonitorAndSettingsForProtocolAndBoard(pme *packagemanager.Explorer, pro
 		} else if recipe, ok := boardPlatform.MonitorsDevRecipes[protocol]; ok {
 			// If we have a recipe we must resolve it
 			cmdLine := boardProperties.ExpandPropsInString(recipe)
-			cmdArgs, err := properties.SplitQuotedString(cmdLine, `"'`, false)
-			if err != nil {
-				return nil, nil, &cmderrors.InvalidArgumentError{Message: i18n.Tr("Invalid recipe in platform.txt"), Cause: err}
-			}
+			cmdArgs, _ := properties.SplitQuotedString(cmdLine, `"'`, false)
 			id := fmt.Sprintf("%s-%s", boardPlatform, protocol)
 			return pluggableMonitor.New(id, cmdArgs...), boardSettings, nil
 		}

commands/service_upload.go

@@ -703,10 +703,7 @@ func runTool(recipeID string, props *properties.Map, outStream, errStream io.Wri
 		return errors.New(i18n.Tr("no upload port provided"))
 	}
 	cmdLine := props.ExpandPropsInString(recipe)
-	cmdArgs, err := properties.SplitQuotedString(cmdLine, `"'`, false)
-	if err != nil {
-		return errors.New(i18n.Tr("invalid recipe '%[1]s': %[2]s", recipe, err))
-	}
+	cmdArgs, _ := properties.SplitQuotedString(cmdLine, `"'`, false)
 
 	// Run Tool
 	logrus.WithField("phase", "upload").Tracef("Executing upload tool: %s", cmdLine)

internal/arduino/builder/builder.go

@@ -58,9 +58,6 @@ type Builder struct {
 	// Parallel processes
 	jobs int
 
-	// Custom build properties defined by user (line by line as "key=value" pairs)
-	customBuildProperties []string
-
 	// core related
 	coreBuildCachePath       *paths.Path
 	extraCoreBuildCachePaths paths.PathList
@@ -88,7 +85,7 @@ type Builder struct {
 	lineOffset int
 
 	targetPlatform *cores.PlatformRelease
-	actualPlatform *cores.PlatformRelease
+	buildPlatform  *cores.PlatformRelease
 
 	buildArtifacts *buildArtifacts
 
@@ -124,19 +121,20 @@ func NewBuilder(
 	coreBuildCachePath *paths.Path,
 	extraCoreBuildCachePaths paths.PathList,
 	jobs int,
-	requestBuildProperties []string,
-	hardwareDirs, otherLibrariesDirs paths.PathList,
+	customBuildProperties []string,
+	hardwareDirs paths.PathList,
+	librariesDirs paths.PathList,
 	builtInLibrariesDirs *paths.Path,
 	fqbn *cores.FQBN,
 	clean bool,
 	sourceOverrides map[string]string,
 	onlyUpdateCompilationDatabase bool,
-	targetPlatform, actualPlatform *cores.PlatformRelease,
+	targetPlatform, buildPlatform *cores.PlatformRelease,
 	useCachedLibrariesResolution bool,
 	librariesManager *librariesmanager.LibrariesManager,
-	libraryDirs paths.PathList,
+	customLibraryDirs paths.PathList,
 	stdout, stderr io.Writer, verbose bool, warningsLevel string,
-	progresCB rpc.TaskProgressCB,
+	progressCB rpc.TaskProgressCB,
 	toolEnv []string,
 ) (*Builder, error) {
 	buildProperties := properties.NewMap()
@@ -145,14 +143,12 @@ func NewBuilder(
 	}
 	if sk != nil {
 		buildProperties.SetPath("sketch_path", sk.FullPath)
+		buildProperties.Set("build.project_name", sk.MainFile.Base())
+		buildProperties.SetPath("build.source.path", sk.FullPath)
 	}
 	if buildPath != nil {
 		buildProperties.SetPath("build.path", buildPath)
 	}
-	if sk != nil {
-		buildProperties.Set("build.project_name", sk.MainFile.Base())
-		buildProperties.SetPath("build.source.path", sk.FullPath)
-	}
 	if optimizeForDebug {
 		if debugFlags, ok := buildProperties.GetOk("compiler.optimization_flags.debug"); ok {
 			buildProperties.Set("compiler.optimization_flags", debugFlags)
@@ -164,12 +160,11 @@ func NewBuilder(
 	}
 
 	// Add user provided custom build properties
-	customBuildProperties, err := properties.LoadFromSlice(requestBuildProperties)
-	if err != nil {
+	if p, err := properties.LoadFromSlice(customBuildProperties); err == nil {
+		buildProperties.Merge(p)
+	} else {
 		return nil, fmt.Errorf("invalid build properties: %w", err)
 	}
-	buildProperties.Merge(customBuildProperties)
-	customBuildPropertiesArgs := append(requestBuildProperties, "build.warn_data_percentage=75")
 
 	sketchBuildPath, err := buildPath.Join("sketch").Abs()
 	if err != nil {
@@ -189,16 +184,20 @@ func NewBuilder(
 	}
 
 	logger := logger.New(stdout, stderr, verbose, warningsLevel)
-	libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader(
-		useCachedLibrariesResolution, librariesManager,
-		builtInLibrariesDirs, libraryDirs, otherLibrariesDirs,
-		actualPlatform, targetPlatform,
+	libsResolver, libsLoadingWarnings, err := detector.LibrariesLoader(
+		useCachedLibrariesResolution,
+		librariesManager,
+		builtInLibrariesDirs,
+		customLibraryDirs,
+		librariesDirs,
+		buildPlatform,
+		targetPlatform,
 	)
 	if err != nil {
 		return nil, err
 	}
 	if logger.Verbose() {
-		logger.Warn(string(verboseOut))
+		logger.Warn(string(libsLoadingWarnings))
 	}
 
 	diagnosticStore := diagnostics.NewStore()
@@ -211,25 +210,26 @@ func NewBuilder(
 		coreBuildPath:                 coreBuildPath,
 		librariesBuildPath:            librariesBuildPath,
 		jobs:                          jobs,
-		customBuildProperties:         customBuildPropertiesArgs,
 		coreBuildCachePath:            coreBuildCachePath,
 		extraCoreBuildCachePaths:      extraCoreBuildCachePaths,
 		logger:                        logger,
 		clean:                         clean,
 		sourceOverrides:               sourceOverrides,
 		onlyUpdateCompilationDatabase: onlyUpdateCompilationDatabase,
 		compilationDatabase:           compilation.NewDatabase(buildPath.Join("compile_commands.json")),
-		Progress:                      progress.New(progresCB),
+		Progress:                      progress.New(progressCB),
 		executableSectionsSize:        []ExecutableSectionSize{},
 		buildArtifacts:                &buildArtifacts{},
 		targetPlatform:                targetPlatform,
-		actualPlatform:                actualPlatform,
+		buildPlatform:                 buildPlatform,
 		toolEnv:                       toolEnv,
 		buildOptions: newBuildOptions(
-			hardwareDirs, otherLibrariesDirs,
-			builtInLibrariesDirs, buildPath,
+			hardwareDirs,
+			librariesDirs,
+			builtInLibrariesDirs,
+			buildPath,
 			sk,
-			customBuildPropertiesArgs,
+			customBuildProperties,
 			fqbn,
 			clean,
 			buildProperties.Get("compiler.optimization_flags"),
@@ -238,7 +238,7 @@ func NewBuilder(
 		),
 		diagnosticStore: diagnosticStore,
 		libsDetector: detector.NewSketchLibrariesDetector(
-			libsManager, libsResolver,
+			libsResolver,
 			useCachedLibrariesResolution,
 			onlyUpdateCompilationDatabase,
 			logger,
@@ -321,10 +321,19 @@ func (b *Builder) preprocess() error {
 		b.librariesBuildPath,
 		b.buildProperties,
 		b.targetPlatform.Platform.Architecture,
+		b.jobs,
 	)
 	if err != nil {
 		return err
 	}
+	if b.libsDetector.IncludeFoldersChanged() && b.librariesBuildPath.Exist() {
+		if b.logger.Verbose() {
+			b.logger.Info(i18n.Tr("The list of included libraries has been changed... rebuilding all libraries."))
+		}
+		if err := b.librariesBuildPath.RemoveAll(); err != nil {
+			return err
+		}
+	}
 	b.Progress.CompleteStep()
 
 	b.warnAboutArchIncompatibleLibraries(b.libsDetector.ImportedLibraries())
@@ -493,29 +502,26 @@ func (b *Builder) prepareCommandForRecipe(buildProperties *properties.Map, recip
 		commandLine = properties.DeleteUnexpandedPropsFromString(commandLine)
 	}
 
-	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
-	if err != nil {
-		return nil, err
-	}
+	args, _ := properties.SplitQuotedString(commandLine, `"'`, false)
 
 	// if the overall commandline is too long for the platform
 	// try reducing the length by making the filenames relative
 	// and changing working directory to build.path
 	var relativePath string
 	if len(commandLine) > 30000 {
 		relativePath = buildProperties.Get("build.path")
-		for i, arg := range parts {
+		for i, arg := range args {
 			if _, err := os.Stat(arg); os.IsNotExist(err) {
 				continue
 			}
 			rel, err := filepath.Rel(relativePath, arg)
 			if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) {
-				parts[i] = rel
+				args[i] = rel
 			}
 		}
 	}
 
-	command, err := paths.NewProcess(b.toolEnv, parts...)
+	command, err := paths.NewProcess(b.toolEnv, args...)
 	if err != nil {
 		return nil, err
 	}