From 6eb7b61e08735208abff327714a5382d4e09fe06 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Thu, 30 May 2024 12:24:07 +0200
Subject: [PATCH 01/28] Removed unused field/parameter

---
 internal/arduino/builder/builder.go                   |  4 ++--
 .../arduino/builder/internal/detector/detector.go     | 11 ++++-------
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/internal/arduino/builder/builder.go b/internal/arduino/builder/builder.go
index 608baead6ee..041d488067f 100644
--- a/internal/arduino/builder/builder.go
+++ b/internal/arduino/builder/builder.go
@@ -189,7 +189,7 @@ func NewBuilder(
 	}
 
 	logger := logger.New(stdout, stderr, verbose, warningsLevel)
-	libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader(
+	libsResolver, verboseOut, err := detector.LibrariesLoader(
 		useCachedLibrariesResolution, librariesManager,
 		builtInLibrariesDirs, libraryDirs, otherLibrariesDirs,
 		actualPlatform, targetPlatform,
@@ -238,7 +238,7 @@ func NewBuilder(
 		),
 		diagnosticStore: diagnosticStore,
 		libsDetector: detector.NewSketchLibrariesDetector(
-			libsManager, libsResolver,
+			libsResolver,
 			useCachedLibrariesResolution,
 			onlyUpdateCompilationDatabase,
 			logger,
diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index 17de9377833..0bf60bf3c3e 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -49,7 +49,6 @@ type libraryResolutionResult struct {
 
 // SketchLibrariesDetector todo
 type SketchLibrariesDetector struct {
-	librariesManager              *librariesmanager.LibrariesManager
 	librariesResolver             *librariesresolver.Cpp
 	useCachedLibrariesResolution  bool
 	onlyUpdateCompilationDatabase bool
@@ -62,7 +61,6 @@ type SketchLibrariesDetector struct {
 
 // NewSketchLibrariesDetector todo
 func NewSketchLibrariesDetector(
-	lm *librariesmanager.LibrariesManager,
 	libsResolver *librariesresolver.Cpp,
 	useCachedLibrariesResolution bool,
 	onlyUpdateCompilationDatabase bool,
@@ -70,7 +68,6 @@ func NewSketchLibrariesDetector(
 	diagnosticStore *diagnostics.Store,
 ) *SketchLibrariesDetector {
 	return &SketchLibrariesDetector{
-		librariesManager:              lm,
 		librariesResolver:             libsResolver,
 		useCachedLibrariesResolution:  useCachedLibrariesResolution,
 		librariesResolutionResults:    map[string]libraryResolutionResult{},
@@ -599,7 +596,7 @@ func LibrariesLoader(
 	librariesManager *librariesmanager.LibrariesManager,
 	builtInLibrariesDirs *paths.Path, libraryDirs, otherLibrariesDirs paths.PathList,
 	actualPlatform, targetPlatform *cores.PlatformRelease,
-) (*librariesmanager.LibrariesManager, *librariesresolver.Cpp, []byte, error) {
+) (*librariesresolver.Cpp, []byte, error) {
 	verboseOut := &bytes.Buffer{}
 	lm := librariesManager
 	if useCachedLibrariesResolution {
@@ -613,7 +610,7 @@ func LibrariesLoader(
 		builtInLibrariesFolders := builtInLibrariesDirs
 		if builtInLibrariesFolders != nil {
 			if err := builtInLibrariesFolders.ToAbs(); err != nil {
-				return nil, nil, nil, err
+				return nil, nil, err
 			}
 			lmb.AddLibrariesDir(librariesmanager.LibrariesDir{
 				Path:     builtInLibrariesFolders,
@@ -636,7 +633,7 @@ func LibrariesLoader(
 
 		librariesFolders := otherLibrariesDirs
 		if err := librariesFolders.ToAbs(); err != nil {
-			return nil, nil, nil, err
+			return nil, nil, err
 		}
 		for _, folder := range librariesFolders {
 			lmb.AddLibrariesDir(librariesmanager.LibrariesDir{
@@ -668,7 +665,7 @@ func LibrariesLoader(
 
 	allLibs := lm.FindAllInstalled()
 	resolver := librariesresolver.NewCppResolver(allLibs, targetPlatform, actualPlatform)
-	return lm, resolver, verboseOut.Bytes(), nil
+	return resolver, verboseOut.Bytes(), nil
 }
 
 type includeCacheEntry struct {

From b4262e09f732af3bd94fe92dbd35a56f900c2fe2 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Thu, 30 May 2024 17:34:39 +0200
Subject: [PATCH 02/28] Moved Result structure into his own package

This change is preparatory for next refactorings
---
 .../builder/internal/detector/detector.go     | 21 +++++++++--------
 .../preprocessor/arduino_preprocessor.go      | 13 ++++++-----
 .../builder/internal/preprocessor/ctags.go    | 23 ++++++++++---------
 .../builder/internal/preprocessor/gcc.go      | 11 +++++----
 .../result.go => runner/task.go}              | 21 ++++-------------
 internal/arduino/builder/preprocess_sketch.go |  6 ++---
 6 files changed, 44 insertions(+), 51 deletions(-)
 rename internal/arduino/builder/internal/{preprocessor/result.go => runner/task.go} (76%)

diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index 0bf60bf3c3e..5bb3dbe8ec3 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -30,6 +30,7 @@ import (
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/diagnostics"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/logger"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/preprocessor"
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/utils"
 	"github.com/arduino/arduino-cli/internal/arduino/cores"
 	"github.com/arduino/arduino-cli/internal/arduino/globals"
@@ -339,7 +340,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 		}
 
 		var preprocErr error
-		var preprocFirstResult preprocessor.Result
+		var preprocFirstResult *runner.Result
 
 		var missingIncludeH string
 		if unchanged && cache.valid {
@@ -350,18 +351,18 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 		} else {
 			preprocFirstResult, preprocErr = preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties)
 			if l.logger.Verbose() {
-				l.logger.WriteStdout(preprocFirstResult.Stdout())
+				l.logger.WriteStdout(preprocFirstResult.Stdout)
 			}
 			// Unwrap error and see if it is an ExitError.
 			var exitErr *exec.ExitError
 			if preprocErr == nil {
 				// Preprocessor successful, done
 				missingIncludeH = ""
-			} else if isExitErr := errors.As(preprocErr, &exitErr); !isExitErr || preprocFirstResult.Stderr() == nil {
+			} else if isExitErr := errors.As(preprocErr, &exitErr); !isExitErr || len(preprocFirstResult.Stderr) == 0 {
 				// Ignore ExitErrors (e.g. gcc returning non-zero status), but bail out on other errors
 				return preprocErr
 			} else {
-				missingIncludeH = IncludesFinderWithRegExp(string(preprocFirstResult.Stderr()))
+				missingIncludeH = IncludesFinderWithRegExp(string(preprocFirstResult.Stderr))
 				if missingIncludeH == "" && l.logger.Verbose() {
 					l.logger.Info(i18n.Tr("Error while detecting libraries included by %[1]s", sourcePath))
 				}
@@ -377,11 +378,11 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 		library := l.resolveLibrary(missingIncludeH, platformArch)
 		if library == nil {
 			// Library could not be resolved, show error
-			if preprocErr == nil || preprocFirstResult.Stderr() == nil {
+			if preprocErr == nil || len(preprocFirstResult.Stderr) == 0 {
 				// Filename came from cache, so run preprocessor to obtain error to show
 				result, err := preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties)
 				if l.logger.Verbose() {
-					l.logger.WriteStdout(result.Stdout())
+					l.logger.WriteStdout(result.Stdout)
 				}
 				if err == nil {
 					// If there is a missing #include in the cache, but running
@@ -390,12 +391,12 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 					// deleted, so hopefully the next compilation will succeed.
 					return errors.New(i18n.Tr("Internal error in cache"))
 				}
-				l.diagnosticStore.Parse(result.Args(), result.Stderr())
-				l.logger.WriteStderr(result.Stderr())
+				l.diagnosticStore.Parse(result.Args, result.Stderr)
+				l.logger.WriteStderr(result.Stderr)
 				return err
 			}
-			l.diagnosticStore.Parse(preprocFirstResult.Args(), preprocFirstResult.Stderr())
-			l.logger.WriteStderr(preprocFirstResult.Stderr())
+			l.diagnosticStore.Parse(preprocFirstResult.Args, preprocFirstResult.Stderr)
+			l.logger.WriteStderr(preprocFirstResult.Stderr)
 			return preprocErr
 		}
 
diff --git a/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go b/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go
index deabc04c4f0..19aa933ea76 100644
--- a/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go
+++ b/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go
@@ -22,6 +22,7 @@ import (
 	"path/filepath"
 	"runtime"
 
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/utils"
 	"github.com/arduino/arduino-cli/internal/arduino/sketch"
 	"github.com/arduino/arduino-cli/internal/i18n"
@@ -35,7 +36,7 @@ func PreprocessSketchWithArduinoPreprocessor(
 	ctx context.Context,
 	sk *sketch.Sketch, buildPath *paths.Path, includeFolders paths.PathList,
 	lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool,
-) (*Result, error) {
+) (*runner.Result, error) {
 	verboseOut := &bytes.Buffer{}
 	normalOut := &bytes.Buffer{}
 	if err := buildPath.Join("preproc").MkdirAll(); err != nil {
@@ -45,8 +46,8 @@ func PreprocessSketchWithArduinoPreprocessor(
 	sourceFile := buildPath.Join("sketch", sk.MainFile.Base()+".cpp")
 	targetFile := buildPath.Join("preproc", "sketch_merged.cpp")
 	gccResult, err := GCC(ctx, sourceFile, targetFile, includeFolders, buildProperties)
-	verboseOut.Write(gccResult.Stdout())
-	verboseOut.Write(gccResult.Stderr())
+	verboseOut.Write(gccResult.Stdout)
+	verboseOut.Write(gccResult.Stderr)
 	if err != nil {
 		return nil, err
 	}
@@ -83,13 +84,13 @@ func PreprocessSketchWithArduinoPreprocessor(
 	commandStdOut, commandStdErr, err := command.RunAndCaptureOutput(ctx)
 	verboseOut.Write(commandStdErr)
 	if err != nil {
-		return &Result{args: gccResult.Args(), stdout: verboseOut.Bytes(), stderr: normalOut.Bytes()}, err
+		return &runner.Result{Args: gccResult.Args, Stdout: verboseOut.Bytes(), Stderr: normalOut.Bytes()}, err
 	}
 	result := utils.NormalizeUTF8(commandStdOut)
 
 	destFile := buildPath.Join(sk.MainFile.Base() + ".cpp")
 	if err := destFile.WriteFile(result); err != nil {
-		return &Result{args: gccResult.Args(), stdout: verboseOut.Bytes(), stderr: normalOut.Bytes()}, err
+		return &runner.Result{Args: gccResult.Args, Stdout: verboseOut.Bytes(), Stderr: normalOut.Bytes()}, err
 	}
-	return &Result{args: gccResult.Args(), stdout: verboseOut.Bytes(), stderr: normalOut.Bytes()}, err
+	return &runner.Result{Args: gccResult.Args, Stdout: verboseOut.Bytes(), Stderr: normalOut.Bytes()}, err
 }
diff --git a/internal/arduino/builder/internal/preprocessor/ctags.go b/internal/arduino/builder/internal/preprocessor/ctags.go
index fe36cfc89e5..a8c22498e98 100644
--- a/internal/arduino/builder/internal/preprocessor/ctags.go
+++ b/internal/arduino/builder/internal/preprocessor/ctags.go
@@ -27,6 +27,7 @@ import (
 
 	"github.com/arduino/arduino-cli/internal/arduino/builder/cpp"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/preprocessor/internal/ctags"
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
 	"github.com/arduino/arduino-cli/internal/arduino/sketch"
 	"github.com/arduino/arduino-cli/internal/i18n"
 	"github.com/arduino/go-paths-helper"
@@ -43,7 +44,7 @@ func PreprocessSketchWithCtags(
 	sketch *sketch.Sketch, buildPath *paths.Path, includes paths.PathList,
 	lineOffset int, buildProperties *properties.Map,
 	onlyUpdateCompilationDatabase, verbose bool,
-) (*Result, error) {
+) (*runner.Result, error) {
 	// Create a temporary working directory
 	tmpDir, err := paths.MkTempDir("", "")
 	if err != nil {
@@ -57,11 +58,11 @@ func PreprocessSketchWithCtags(
 	// Run GCC preprocessor
 	sourceFile := buildPath.Join("sketch", sketch.MainFile.Base()+".cpp")
 	result, err := GCC(ctx, sourceFile, ctagsTarget, includes, buildProperties)
-	stdout.Write(result.Stdout())
-	stderr.Write(result.Stderr())
+	stdout.Write(result.Stdout)
+	stderr.Write(result.Stderr)
 	if err != nil {
 		if !onlyUpdateCompilationDatabase {
-			return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+			return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 		}
 
 		// Do not bail out if we are generating the compile commands database
@@ -69,17 +70,17 @@ func PreprocessSketchWithCtags(
 			i18n.Tr("An error occurred adding prototypes"),
 			i18n.Tr("the compilation database may be incomplete or inaccurate")))
 		if err := sourceFile.CopyTo(ctagsTarget); err != nil {
-			return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+			return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 		}
 	}
 
 	if src, err := ctagsTarget.ReadFile(); err == nil {
 		filteredSource := filterSketchSource(sketch, bytes.NewReader(src), false)
 		if err := ctagsTarget.WriteFile([]byte(filteredSource)); err != nil {
-			return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+			return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 		}
 	} else {
-		return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+		return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 	}
 
 	// Run CTags on gcc-preprocessed source
@@ -88,7 +89,7 @@ func PreprocessSketchWithCtags(
 		stderr.Write(ctagsStdErr)
 	}
 	if err != nil {
-		return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+		return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 	}
 
 	// Parse CTags output
@@ -103,13 +104,13 @@ func PreprocessSketchWithCtags(
 	if sourceData, err := sourceFile.ReadFile(); err == nil {
 		source = string(sourceData)
 	} else {
-		return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+		return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 	}
 	source = strings.ReplaceAll(source, "\r\n", "\n")
 	source = strings.ReplaceAll(source, "\r", "\n")
 	sourceRows := strings.Split(source, "\n")
 	if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) {
-		return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, nil
+		return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, nil
 	}
 
 	insertionLine := firstFunctionLine + lineOffset - 1
@@ -135,7 +136,7 @@ func PreprocessSketchWithCtags(
 
 	// Write back arduino-preprocess output to the sourceFile
 	err = sourceFile.WriteFile([]byte(preprocessedSource))
-	return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+	return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 }
 
 func composePrototypeSection(line int, prototypes []*ctags.Prototype) string {
diff --git a/internal/arduino/builder/internal/preprocessor/gcc.go b/internal/arduino/builder/internal/preprocessor/gcc.go
index bfcc9512e0c..96ea729a42d 100644
--- a/internal/arduino/builder/internal/preprocessor/gcc.go
+++ b/internal/arduino/builder/internal/preprocessor/gcc.go
@@ -23,6 +23,7 @@ import (
 
 	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/cpp"
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
 	"github.com/arduino/arduino-cli/internal/i18n"
 	"github.com/arduino/go-paths-helper"
 	"github.com/arduino/go-properties-orderedmap"
@@ -34,7 +35,7 @@ func GCC(
 	ctx context.Context,
 	sourceFilePath, targetFilePath *paths.Path,
 	includes paths.PathList, buildProperties *properties.Map,
-) (Result, error) {
+) (*runner.Result, error) {
 	gccBuildProperties := properties.NewMap()
 	gccBuildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC")
 	gccBuildProperties.Merge(buildProperties)
@@ -59,14 +60,14 @@ func GCC(
 
 	pattern := gccBuildProperties.Get(gccPreprocRecipeProperty)
 	if pattern == "" {
-		return Result{}, errors.New(i18n.Tr("%s pattern is missing", gccPreprocRecipeProperty))
+		return nil, errors.New(i18n.Tr("%s pattern is missing", gccPreprocRecipeProperty))
 	}
 
 	commandLine := gccBuildProperties.ExpandPropsInString(pattern)
 	commandLine = properties.DeleteUnexpandedPropsFromString(commandLine)
 	args, err := properties.SplitQuotedString(commandLine, `"'`, false)
 	if err != nil {
-		return Result{}, err
+		return nil, err
 	}
 
 	// Remove -MMD argument if present. Leaving it will make gcc try
@@ -75,12 +76,12 @@ func GCC(
 
 	proc, err := paths.NewProcess(nil, args...)
 	if err != nil {
-		return Result{}, err
+		return nil, err
 	}
 	stdout, stderr, err := proc.RunAndCaptureOutput(ctx)
 
 	// Append gcc arguments to stdout
 	stdout = append([]byte(fmt.Sprintln(strings.Join(args, " "))), stdout...)
 
-	return Result{args: proc.GetArgs(), stdout: stdout, stderr: stderr}, err
+	return &runner.Result{Args: proc.GetArgs(), Stdout: stdout, Stderr: stderr}, err
 }
diff --git a/internal/arduino/builder/internal/preprocessor/result.go b/internal/arduino/builder/internal/runner/task.go
similarity index 76%
rename from internal/arduino/builder/internal/preprocessor/result.go
rename to internal/arduino/builder/internal/runner/task.go
index 3fa45e40974..05d5f63105a 100644
--- a/internal/arduino/builder/internal/preprocessor/result.go
+++ b/internal/arduino/builder/internal/runner/task.go
@@ -13,22 +13,11 @@
 // Arduino software without disclosing the source code of your own applications.
 // To purchase a commercial license, send an email to license@arduino.cc.
 
-package preprocessor
+package runner
 
+// Result contains the output of a command execution
 type Result struct {
-	args   []string
-	stdout []byte
-	stderr []byte
-}
-
-func (r Result) Args() []string {
-	return r.args
-}
-
-func (r Result) Stdout() []byte {
-	return r.stdout
-}
-
-func (r Result) Stderr() []byte {
-	return r.stderr
+	Args   []string
+	Stdout []byte
+	Stderr []byte
 }
diff --git a/internal/arduino/builder/preprocess_sketch.go b/internal/arduino/builder/preprocess_sketch.go
index 86d7bd7e7e9..b4508304aa3 100644
--- a/internal/arduino/builder/preprocess_sketch.go
+++ b/internal/arduino/builder/preprocess_sketch.go
@@ -30,10 +30,10 @@ func (b *Builder) preprocessSketch(includes paths.PathList) error {
 	)
 	if result != nil {
 		if b.logger.Verbose() {
-			b.logger.WriteStdout(result.Stdout())
+			b.logger.WriteStdout(result.Stdout)
 		}
-		b.logger.WriteStdout(result.Stderr())
-		b.diagnosticStore.Parse(result.Args(), result.Stderr())
+		b.logger.WriteStdout(result.Stderr)
+		b.diagnosticStore.Parse(result.Args, result.Stderr)
 	}
 
 	return err

From 3daa69f7673c0c7c25a44c4973da00b66644a148 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Fri, 31 May 2024 15:03:05 +0200
Subject: [PATCH 03/28] Moving sourceFile object in his own file

---
 .../builder/internal/detector/detector.go     | 91 ------------------
 .../builder/internal/detector/source_file.go  | 92 +++++++++++++++++++
 2 files changed, 92 insertions(+), 91 deletions(-)
 create mode 100644 internal/arduino/builder/internal/detector/source_file.go

diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index 5bb3dbe8ec3..57a2d3bcc86 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -500,97 +500,6 @@ func findIncludeForOldCompilers(source string) string {
 	return ""
 }
 
-type sourceFile struct {
-	// Path to the source file within the sketch/library root folder
-	relativePath *paths.Path
-
-	// ExtraIncludePath contains an extra include path that must be
-	// used to compile this source file.
-	// This is mainly used for source files that comes from old-style libraries
-	// (Arduino IDE <1.5) requiring an extra include path to the "utility" folder.
-	extraIncludePath *paths.Path
-
-	// The source root for the given origin, where its source files
-	// can be found. Prepending this to SourceFile.RelativePath will give
-	// the full path to that source file.
-	sourceRoot *paths.Path
-
-	// The build root for the given origin, where build products will
-	// be placed. Any directories inside SourceFile.RelativePath will be
-	// appended here.
-	buildRoot *paths.Path
-}
-
-// Equals fixdoc
-func (f *sourceFile) Equals(g *sourceFile) bool {
-	return f.relativePath.EqualsTo(g.relativePath) &&
-		f.buildRoot.EqualsTo(g.buildRoot) &&
-		f.sourceRoot.EqualsTo(g.sourceRoot)
-}
-
-// makeSourceFile containing the given source file path within the
-// given origin. The given path can be absolute, or relative within the
-// origin's root source folder
-func makeSourceFile(
-	sourceDir *paths.Path,
-	buildDir *paths.Path,
-	sourceFilePath *paths.Path,
-	extraIncludePath ...*paths.Path,
-) (*sourceFile, error) {
-	res := &sourceFile{
-		buildRoot:  buildDir,
-		sourceRoot: sourceDir,
-	}
-
-	if len(extraIncludePath) > 1 {
-		panic("only one extra include path allowed")
-	}
-	if len(extraIncludePath) > 0 {
-		res.extraIncludePath = extraIncludePath[0]
-	}
-	// switch o := origin.(type) {
-	// case *sketch.Sketch:
-	// 	res.buildRoot = sketchBuildPath
-	// 	res.sourceRoot = sketchBuildPath
-	// case *libraries.Library:
-	// 	res.buildRoot = librariesBuildPath.Join(o.DirName)
-	// 	res.sourceRoot = o.SourceDir
-	// 	res.extraIncludePath = o.UtilityDir
-	// default:
-	// 	panic("Unexpected origin for SourceFile: " + fmt.Sprint(origin))
-	// }
-
-	if sourceFilePath.IsAbs() {
-		var err error
-		sourceFilePath, err = res.sourceRoot.RelTo(sourceFilePath)
-		if err != nil {
-			return nil, err
-		}
-	}
-	res.relativePath = sourceFilePath
-	return res, nil
-}
-
-// ExtraIncludePath fixdoc
-func (f *sourceFile) ExtraIncludePath() *paths.Path {
-	return f.extraIncludePath
-}
-
-// SourcePath fixdoc
-func (f *sourceFile) SourcePath() *paths.Path {
-	return f.sourceRoot.JoinPath(f.relativePath)
-}
-
-// ObjectPath fixdoc
-func (f *sourceFile) ObjectPath() *paths.Path {
-	return f.buildRoot.Join(f.relativePath.String() + ".o")
-}
-
-// DepfilePath fixdoc
-func (f *sourceFile) DepfilePath() *paths.Path {
-	return f.buildRoot.Join(f.relativePath.String() + ".d")
-}
-
 // LibrariesLoader todo
 func LibrariesLoader(
 	useCachedLibrariesResolution bool,
diff --git a/internal/arduino/builder/internal/detector/source_file.go b/internal/arduino/builder/internal/detector/source_file.go
new file mode 100644
index 00000000000..92d1f93e938
--- /dev/null
+++ b/internal/arduino/builder/internal/detector/source_file.go
@@ -0,0 +1,92 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2024 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package detector
+
+import "github.com/arduino/go-paths-helper"
+
+type sourceFile struct {
+	// Path to the source file within the sketch/library root folder
+	relativePath *paths.Path
+
+	// ExtraIncludePath contains an extra include path that must be
+	// used to compile this source file.
+	// This is mainly used for source files that comes from old-style libraries
+	// (Arduino IDE <1.5) requiring an extra include path to the "utility" folder.
+	extraIncludePath *paths.Path
+
+	// The source root for the given origin, where its source files
+	// can be found. Prepending this to SourceFile.RelativePath will give
+	// the full path to that source file.
+	sourceRoot *paths.Path
+
+	// The build root for the given origin, where build products will
+	// be placed. Any directories inside SourceFile.RelativePath will be
+	// appended here.
+	buildRoot *paths.Path
+}
+
+// Equals fixdoc
+func (f *sourceFile) Equals(g *sourceFile) bool {
+	return f.relativePath.EqualsTo(g.relativePath) &&
+		f.buildRoot.EqualsTo(g.buildRoot) &&
+		f.sourceRoot.EqualsTo(g.sourceRoot)
+}
+
+// makeSourceFile create a sourceFile object for the given source file path.
+// The given sourceFilePath can be absolute, or relative within the sourceRoot root folder.
+func makeSourceFile(sourceRoot, buildRoot, sourceFilePath *paths.Path, extraIncludePath ...*paths.Path) (*sourceFile, error) {
+	res := &sourceFile{
+		buildRoot:  buildRoot,
+		sourceRoot: sourceRoot,
+	}
+
+	if len(extraIncludePath) > 1 {
+		panic("only one extra include path allowed")
+	}
+	if len(extraIncludePath) > 0 {
+		res.extraIncludePath = extraIncludePath[0]
+	}
+
+	if sourceFilePath.IsAbs() {
+		var err error
+		sourceFilePath, err = res.sourceRoot.RelTo(sourceFilePath)
+		if err != nil {
+			return nil, err
+		}
+	}
+	res.relativePath = sourceFilePath
+	return res, nil
+}
+
+// ExtraIncludePath returns the extra include path required to build the source file.
+func (f *sourceFile) ExtraIncludePath() *paths.Path {
+	return f.extraIncludePath
+}
+
+// SourcePath return the full path to the source file.
+func (f *sourceFile) SourcePath() *paths.Path {
+	return f.sourceRoot.JoinPath(f.relativePath)
+}
+
+// ObjectPath return the full path to the object file.
+func (f *sourceFile) ObjectPath() *paths.Path {
+	return f.buildRoot.Join(f.relativePath.String() + ".o")
+}
+
+// DepfilePath return the full path to the dependency file.
+func (f *sourceFile) DepfilePath() *paths.Path {
+	return f.buildRoot.Join(f.relativePath.String() + ".d")
+}

From 5fc9a8489ef3bc90860f4c107c7278f01009f580 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Fri, 31 May 2024 15:07:04 +0200
Subject: [PATCH 04/28] Moved uniqueSourceFileQueue in his own file

---
 .../builder/internal/detector/detector.go     | 32 +++--------------
 .../builder/internal/detector/source_file.go  | 34 ++++++++++++++++++-
 2 files changed, 37 insertions(+), 29 deletions(-)

diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index 57a2d3bcc86..c7e983ac39b 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -23,7 +23,6 @@ import (
 	"fmt"
 	"os/exec"
 	"regexp"
-	"slices"
 	"strings"
 	"time"
 
@@ -259,7 +258,7 @@ func (l *SketchLibrariesDetector) findIncludes(
 		if err != nil {
 			return err
 		}
-		sourceFileQueue.push(mergedfile)
+		sourceFileQueue.Push(mergedfile)
 
 		l.queueSourceFilesFromFolder(sourceFileQueue, sketchBuildPath, false /* recurse */, sketchBuildPath, sketchBuildPath)
 		srcSubfolderPath := sketchBuildPath.Join("src")
@@ -267,7 +266,7 @@ func (l *SketchLibrariesDetector) findIncludes(
 			l.queueSourceFilesFromFolder(sourceFileQueue, srcSubfolderPath, true /* recurse */, sketchBuildPath, sketchBuildPath)
 		}
 
-		for !sourceFileQueue.empty() {
+		for !sourceFileQueue.Empty() {
 			err := l.findIncludesUntilDone(ctx, cache, sourceFileQueue, buildProperties, librariesBuildPath, platformArch)
 			if err != nil {
 				cachePath.Remove()
@@ -303,7 +302,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 	librariesBuildPath *paths.Path,
 	platformArch string,
 ) error {
-	sourceFile := sourceFileQueue.pop()
+	sourceFile := sourceFileQueue.Pop()
 	sourcePath := sourceFile.SourcePath()
 	targetFilePath := paths.NullPath()
 	depPath := sourceFile.DepfilePath()
@@ -443,7 +442,7 @@ func (l *SketchLibrariesDetector) queueSourceFilesFromFolder(
 		if err != nil {
 			return err
 		}
-		sourceFileQueue.push(sourceFile)
+		sourceFileQueue.Push(sourceFile)
 	}
 
 	return nil
@@ -688,26 +687,3 @@ func writeCache(cache *includeCache, path *paths.Path) error {
 	}
 	return nil
 }
-
-type uniqueSourceFileQueue []*sourceFile
-
-func (queue *uniqueSourceFileQueue) push(value *sourceFile) {
-	if !queue.contains(value) {
-		*queue = append(*queue, value)
-	}
-}
-
-func (queue uniqueSourceFileQueue) contains(target *sourceFile) bool {
-	return slices.ContainsFunc(queue, target.Equals)
-}
-
-func (queue *uniqueSourceFileQueue) pop() *sourceFile {
-	old := *queue
-	x := old[0]
-	*queue = old[1:]
-	return x
-}
-
-func (queue uniqueSourceFileQueue) empty() bool {
-	return len(queue) == 0
-}
diff --git a/internal/arduino/builder/internal/detector/source_file.go b/internal/arduino/builder/internal/detector/source_file.go
index 92d1f93e938..3ebb52d0387 100644
--- a/internal/arduino/builder/internal/detector/source_file.go
+++ b/internal/arduino/builder/internal/detector/source_file.go
@@ -15,7 +15,11 @@
 
 package detector
 
-import "github.com/arduino/go-paths-helper"
+import (
+	"slices"
+
+	"github.com/arduino/go-paths-helper"
+)
 
 type sourceFile struct {
 	// Path to the source file within the sketch/library root folder
@@ -90,3 +94,31 @@ func (f *sourceFile) ObjectPath() *paths.Path {
 func (f *sourceFile) DepfilePath() *paths.Path {
 	return f.buildRoot.Join(f.relativePath.String() + ".d")
 }
+
+// uniqueSourceFileQueue is a queue of source files that does not allow duplicates.
+type uniqueSourceFileQueue []*sourceFile
+
+// Push adds a source file to the queue if it is not already present.
+func (queue *uniqueSourceFileQueue) Push(value *sourceFile) {
+	if !queue.Contains(value) {
+		*queue = append(*queue, value)
+	}
+}
+
+// Contains checks if the queue Contains a source file.
+func (queue uniqueSourceFileQueue) Contains(target *sourceFile) bool {
+	return slices.ContainsFunc(queue, target.Equals)
+}
+
+// Pop removes and returns the first element of the queue.
+func (queue *uniqueSourceFileQueue) Pop() *sourceFile {
+	old := *queue
+	x := old[0]
+	*queue = old[1:]
+	return x
+}
+
+// Empty returns true if the queue is Empty.
+func (queue uniqueSourceFileQueue) Empty() bool {
+	return len(queue) == 0
+}

From 7ff78b3d70f9b1728c93d914093b3e36d6528ab4 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Fri, 31 May 2024 15:50:30 +0200
Subject: [PATCH 05/28] Moved includeCache in his own file and made it a field
 of detector

---
 .../builder/internal/detector/detector.go     | 121 ++----------------
 .../internal/detector/include_cache.go        | 117 +++++++++++++++++
 2 files changed, 130 insertions(+), 108 deletions(-)
 create mode 100644 internal/arduino/builder/internal/detector/include_cache.go

diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index c7e983ac39b..6476d3a23df 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -51,6 +51,7 @@ type libraryResolutionResult struct {
 type SketchLibrariesDetector struct {
 	librariesResolver             *librariesresolver.Cpp
 	useCachedLibrariesResolution  bool
+	cache                         *includeCache
 	onlyUpdateCompilationDatabase bool
 	importedLibraries             libraries.List
 	librariesResolutionResults    map[string]libraryResolutionResult
@@ -181,13 +182,12 @@ func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList {
 // and should be the empty string for the default include folders, like
 // the core or variant.
 func (l *SketchLibrariesDetector) appendIncludeFolder(
-	cache *includeCache,
 	sourceFilePath *paths.Path,
 	include string,
 	folder *paths.Path,
 ) {
 	l.includeFolders = append(l.includeFolders, folder)
-	cache.ExpectEntry(sourceFilePath, include, folder)
+	l.cache.ExpectEntry(sourceFilePath, include, folder)
 }
 
 // FindIncludes todo
@@ -243,11 +243,11 @@ func (l *SketchLibrariesDetector) findIncludes(
 	}
 
 	cachePath := buildPath.Join("includes.cache")
-	cache := readCache(cachePath)
+	l.cache = readCache(cachePath)
 
-	l.appendIncludeFolder(cache, nil, "", buildCorePath)
+	l.appendIncludeFolder(nil, "", buildCorePath)
 	if buildVariantPath != nil {
-		l.appendIncludeFolder(cache, nil, "", buildVariantPath)
+		l.appendIncludeFolder(nil, "", buildVariantPath)
 	}
 
 	sourceFileQueue := &uniqueSourceFileQueue{}
@@ -267,7 +267,7 @@ func (l *SketchLibrariesDetector) findIncludes(
 		}
 
 		for !sourceFileQueue.Empty() {
-			err := l.findIncludesUntilDone(ctx, cache, sourceFileQueue, buildProperties, librariesBuildPath, platformArch)
+			err := l.findIncludesUntilDone(ctx, sourceFileQueue, buildProperties, librariesBuildPath, platformArch)
 			if err != nil {
 				cachePath.Remove()
 				return err
@@ -275,8 +275,8 @@ func (l *SketchLibrariesDetector) findIncludes(
 		}
 
 		// Finalize the cache
-		cache.ExpectEnd()
-		if err := writeCache(cache, cachePath); err != nil {
+		l.cache.ExpectEnd()
+		if err := l.cache.write(cachePath); err != nil {
 			return err
 		}
 	}
@@ -296,7 +296,6 @@ func (l *SketchLibrariesDetector) findIncludes(
 
 func (l *SketchLibrariesDetector) findIncludesUntilDone(
 	ctx context.Context,
-	cache *includeCache,
 	sourceFileQueue *uniqueSourceFileQueue,
 	buildProperties *properties.Map,
 	librariesBuildPath *paths.Path,
@@ -327,7 +326,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 
 	first := true
 	for {
-		cache.ExpectFile(sourcePath)
+		l.cache.ExpectFile(sourcePath)
 
 		// Libraries may require the "utility" directory to be added to the include
 		// search path, but only for the source code of the library, so we temporary
@@ -342,8 +341,8 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 		var preprocFirstResult *runner.Result
 
 		var missingIncludeH string
-		if unchanged && cache.valid {
-			missingIncludeH = cache.Next().Include
+		if unchanged && l.cache.valid {
+			missingIncludeH = l.cache.Next().Include
 			if first && l.logger.Verbose() {
 				l.logger.Info(i18n.Tr("Using cached library dependencies for file: %[1]s", sourcePath))
 			}
@@ -370,7 +369,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 
 		if missingIncludeH == "" {
 			// No missing includes found, we're done
-			cache.ExpectEntry(sourcePath, "", nil)
+			l.cache.ExpectEntry(sourcePath, "", nil)
 			return nil
 		}
 
@@ -403,7 +402,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 		// include path and queue its source files for further
 		// include scanning
 		l.AppendImportedLibraries(library)
-		l.appendIncludeFolder(cache, sourcePath, missingIncludeH, library.SourceDir)
+		l.appendIncludeFolder(sourcePath, missingIncludeH, library.SourceDir)
 
 		if library.Precompiled && library.PrecompiledWithSources {
 			// Fully precompiled libraries should have no dependencies to avoid ABI breakage
@@ -593,97 +592,3 @@ func (entry *includeCacheEntry) String() string {
 func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool {
 	return entry.String() == other.String()
 }
-
-type includeCache struct {
-	// Are the cache contents valid so far?
-	valid bool
-	// Index into entries of the next entry to be processed. Unused
-	// when the cache is invalid.
-	next    int
-	entries []*includeCacheEntry
-}
-
-// Next Return the next cache entry. Should only be called when the cache is
-// valid and a next entry is available (the latter can be checked with
-// ExpectFile). Does not advance the cache.
-func (cache *includeCache) Next() *includeCacheEntry {
-	return cache.entries[cache.next]
-}
-
-// ExpectFile check that the next cache entry is about the given file. If it is
-// not, or no entry is available, the cache is invalidated. Does not
-// advance the cache.
-func (cache *includeCache) ExpectFile(sourcefile *paths.Path) {
-	if cache.valid && (cache.next >= len(cache.entries) || !cache.Next().Sourcefile.EqualsTo(sourcefile)) {
-		cache.valid = false
-		cache.entries = cache.entries[:cache.next]
-	}
-}
-
-// ExpectEntry check that the next entry matches the given values. If so, advance
-// the cache. If not, the cache is invalidated. If the cache is
-// invalidated, or was already invalid, an entry with the given values
-// is appended.
-func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, librarypath *paths.Path) {
-	entry := &includeCacheEntry{Sourcefile: sourcefile, Include: include, Includepath: librarypath}
-	if cache.valid {
-		if cache.next < len(cache.entries) && cache.Next().Equals(entry) {
-			cache.next++
-		} else {
-			cache.valid = false
-			cache.entries = cache.entries[:cache.next]
-		}
-	}
-
-	if !cache.valid {
-		cache.entries = append(cache.entries, entry)
-	}
-}
-
-// ExpectEnd check that the cache is completely consumed. If not, the cache is
-// invalidated.
-func (cache *includeCache) ExpectEnd() {
-	if cache.valid && cache.next < len(cache.entries) {
-		cache.valid = false
-		cache.entries = cache.entries[:cache.next]
-	}
-}
-
-// Read the cache from the given file
-func readCache(path *paths.Path) *includeCache {
-	bytes, err := path.ReadFile()
-	if err != nil {
-		// Return an empty, invalid cache
-		return &includeCache{}
-	}
-	result := &includeCache{}
-	err = json.Unmarshal(bytes, &result.entries)
-	if err != nil {
-		// Return an empty, invalid cache
-		return &includeCache{}
-	}
-	result.valid = true
-	return result
-}
-
-// Write the given cache to the given file if it is invalidated. If the
-// cache is still valid, just update the timestamps of the file.
-func writeCache(cache *includeCache, path *paths.Path) error {
-	// If the cache was still valid all the way, just touch its file
-	// (in case any source file changed without influencing the
-	// includes). If it was invalidated, overwrite the cache with
-	// the new contents.
-	if cache.valid {
-		path.Chtimes(time.Now(), time.Now())
-	} else {
-		bytes, err := json.MarshalIndent(cache.entries, "", "  ")
-		if err != nil {
-			return err
-		}
-		err = path.WriteFile(bytes)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
diff --git a/internal/arduino/builder/internal/detector/include_cache.go b/internal/arduino/builder/internal/detector/include_cache.go
new file mode 100644
index 00000000000..7e6937162eb
--- /dev/null
+++ b/internal/arduino/builder/internal/detector/include_cache.go
@@ -0,0 +1,117 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package detector
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/arduino/go-paths-helper"
+)
+
+type includeCache struct {
+	// Are the cache contents valid so far?
+	valid bool
+	// Index into entries of the next entry to be processed. Unused
+	// when the cache is invalid.
+	next    int
+	entries []*includeCacheEntry
+}
+
+// Next Return the next cache entry. Should only be called when the cache is
+// valid and a next entry is available (the latter can be checked with
+// ExpectFile). Does not advance the cache.
+func (cache *includeCache) Next() *includeCacheEntry {
+	return cache.entries[cache.next]
+}
+
+// ExpectFile check that the next cache entry is about the given file. If it is
+// not, or no entry is available, the cache is invalidated. Does not
+// advance the cache.
+func (cache *includeCache) ExpectFile(sourcefile *paths.Path) {
+	if cache.valid && (cache.next >= len(cache.entries) || !cache.Next().Sourcefile.EqualsTo(sourcefile)) {
+		cache.valid = false
+		cache.entries = cache.entries[:cache.next]
+	}
+}
+
+// ExpectEntry check that the next entry matches the given values. If so, advance
+// the cache. If not, the cache is invalidated. If the cache is
+// invalidated, or was already invalid, an entry with the given values
+// is appended.
+func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, librarypath *paths.Path) {
+	entry := &includeCacheEntry{Sourcefile: sourcefile, Include: include, Includepath: librarypath}
+	if cache.valid {
+		if cache.next < len(cache.entries) && cache.Next().Equals(entry) {
+			cache.next++
+		} else {
+			cache.valid = false
+			cache.entries = cache.entries[:cache.next]
+		}
+	}
+
+	if !cache.valid {
+		cache.entries = append(cache.entries, entry)
+	}
+}
+
+// ExpectEnd check that the cache is completely consumed. If not, the cache is
+// invalidated.
+func (cache *includeCache) ExpectEnd() {
+	if cache.valid && cache.next < len(cache.entries) {
+		cache.valid = false
+		cache.entries = cache.entries[:cache.next]
+	}
+}
+
+// Read the cache from the given file
+func readCache(path *paths.Path) *includeCache {
+	bytes, err := path.ReadFile()
+	if err != nil {
+		// Return an empty, invalid cache
+		return &includeCache{}
+	}
+	result := &includeCache{}
+	err = json.Unmarshal(bytes, &result.entries)
+	if err != nil {
+		// Return an empty, invalid cache
+		return &includeCache{}
+	}
+	result.valid = true
+	return result
+}
+
+// Write the given cache to the given file if it is invalidated. If the
+// cache is still valid, just update the timestamps of the file.
+func (cache *includeCache) write(path *paths.Path) error {
+	// If the cache was still valid all the way, just touch its file
+	// (in case any source file changed without influencing the
+	// includes). If it was invalidated, overwrite the cache with
+	// the new contents.
+	if cache.valid {
+		path.Chtimes(time.Now(), time.Now())
+	} else {
+		bytes, err := json.MarshalIndent(cache.entries, "", "  ")
+		if err != nil {
+			return err
+		}
+		err = path.WriteFile(bytes)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

From 68a97486bd579c6f235a9e462acb68687d96254e Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Fri, 31 May 2024 19:03:53 +0200
Subject: [PATCH 06/28] Fixed comment

---
 internal/arduino/builder/internal/detector/detector.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index 6476d3a23df..01cbf33945d 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -167,9 +167,8 @@ func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool)
 	time.Sleep(100 * time.Millisecond)
 }
 
-// IncludeFolders fixdoc
+// IncludeFolders returns the list of include folders detected as needed.
 func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList {
-	// TODO should we do a deep copy?
 	return l.includeFolders
 }
 

From db0db78b6786065240dee3d95d6e0fd8d6eb7ec3 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Wed, 5 Jun 2024 10:40:10 +0200
Subject: [PATCH 07/28] Renamed variable (typo)

---
 .../preprocessor/arduino_preprocessor.go      | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go b/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go
index 19aa933ea76..e69adeae154 100644
--- a/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go
+++ b/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go
@@ -52,20 +52,20 @@ func PreprocessSketchWithArduinoPreprocessor(
 		return nil, err
 	}
 
-	arduiniPreprocessorProperties := properties.NewMap()
-	arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}")
-	arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor")
-	arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.pattern", `"{cmd.path}" "{source_file}" -- -std=gnu++11`)
-	arduiniPreprocessorProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC")
-	arduiniPreprocessorProperties.Merge(buildProperties)
-	arduiniPreprocessorProperties.Merge(arduiniPreprocessorProperties.SubTree("tools").SubTree("arduino-preprocessor"))
-	arduiniPreprocessorProperties.SetPath("source_file", targetFile)
-	pattern := arduiniPreprocessorProperties.Get("pattern")
+	arduinoPreprocessorProperties := properties.NewMap()
+	arduinoPreprocessorProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}")
+	arduinoPreprocessorProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor")
+	arduinoPreprocessorProperties.Set("tools.arduino-preprocessor.pattern", `"{cmd.path}" "{source_file}" -- -std=gnu++11`)
+	arduinoPreprocessorProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC")
+	arduinoPreprocessorProperties.Merge(buildProperties)
+	arduinoPreprocessorProperties.Merge(arduinoPreprocessorProperties.SubTree("tools").SubTree("arduino-preprocessor"))
+	arduinoPreprocessorProperties.SetPath("source_file", targetFile)
+	pattern := arduinoPreprocessorProperties.Get("pattern")
 	if pattern == "" {
 		return nil, errors.New(i18n.Tr("arduino-preprocessor pattern is missing"))
 	}
 
-	commandLine := arduiniPreprocessorProperties.ExpandPropsInString(pattern)
+	commandLine := arduinoPreprocessorProperties.ExpandPropsInString(pattern)
 	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
 	if err != nil {
 		return nil, err

From e937db81069bccaa37f40c302243945e1c104430 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Fri, 31 May 2024 19:13:21 +0200
Subject: [PATCH 08/28] Simplified handling of de-duplication of cache hit
 messages

This equivalent change moves the 'first' variable closer to his definition and use.
---
 internal/arduino/builder/internal/detector/detector.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index 01cbf33945d..0deb0194661 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -345,6 +345,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 			if first && l.logger.Verbose() {
 				l.logger.Info(i18n.Tr("Using cached library dependencies for file: %[1]s", sourcePath))
 			}
+			first = false
 		} else {
 			preprocFirstResult, preprocErr = preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties)
 			if l.logger.Verbose() {
@@ -414,7 +415,6 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 					library.SourceDir, librariesBuildPath.Join(library.DirName), library.UtilityDir)
 			}
 		}
-		first = false
 	}
 }
 

From e4f9927131abe4b20a78523324f2d1eedf1f96f1 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Fri, 31 May 2024 19:03:35 +0200
Subject: [PATCH 09/28] Implemented a better include-cache

---
 .../builder/internal/detector/cache.go        | 122 ++++++++++++++++++
 .../builder/internal/detector/detector.go     |  46 +++----
 2 files changed, 141 insertions(+), 27 deletions(-)
 create mode 100644 internal/arduino/builder/internal/detector/cache.go

diff --git a/internal/arduino/builder/internal/detector/cache.go b/internal/arduino/builder/internal/detector/cache.go
new file mode 100644
index 00000000000..247a1e8de30
--- /dev/null
+++ b/internal/arduino/builder/internal/detector/cache.go
@@ -0,0 +1,122 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2024 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package detector
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/arduino/go-paths-helper"
+)
+
+type detectorCache struct {
+	curr    int
+	entries []*detectorCacheEntry
+}
+
+type detectorCacheEntry struct {
+	AddedIncludePath    *paths.Path `json:"added_include_path,omitempty"`
+	CompilingSourcePath *paths.Path `json:"compiling_source_path,omitempty"`
+	MissingIncludeH     *string     `json:"missing_include_h,omitempty"`
+}
+
+func (e *detectorCacheEntry) String() string {
+	if e.AddedIncludePath != nil {
+		return "Added include path: " + e.AddedIncludePath.String()
+	}
+	if e.CompilingSourcePath != nil {
+		return "Compiling source path: " + e.CompilingSourcePath.String()
+	}
+	if e.MissingIncludeH != nil {
+		if *e.MissingIncludeH == "" {
+			return "No missing include files detected"
+		}
+		return "Missing include file: " + *e.MissingIncludeH
+	}
+	return "No operation"
+}
+
+func (e *detectorCacheEntry) Equals(entry *detectorCacheEntry) bool {
+	return e.String() == entry.String()
+}
+
+func newDetectorCache() *detectorCache {
+	return &detectorCache{}
+}
+
+func (c *detectorCache) String() string {
+	res := ""
+	for _, entry := range c.entries {
+		res += fmt.Sprintln(entry)
+	}
+	return res
+}
+
+// Load reads a saved cache from the given file.
+// If the file do not exists, it does nothing.
+func (c *detectorCache) Load(cacheFile *paths.Path) error {
+	if exist, err := cacheFile.ExistCheck(); err != nil {
+		return err
+	} else if !exist {
+		return nil
+	}
+	data, err := cacheFile.ReadFile()
+	if err != nil {
+		return err
+	}
+	var entries []*detectorCacheEntry
+	if err := json.Unmarshal(data, &entries); err != nil {
+		return err
+	}
+	c.curr = 0
+	c.entries = entries
+	return nil
+}
+
+// Expect adds an entry to the cache and checks if it matches the next expected entry.
+func (c *detectorCache) Expect(entry *detectorCacheEntry) {
+	if c.curr < len(c.entries) {
+		if c.entries[c.curr].Equals(entry) {
+			// Cache hit, move to the next entry
+			c.curr++
+			return
+		}
+		// Cache mismatch, invalidate and cut the remainder of the cache
+		c.entries = c.entries[:c.curr]
+	}
+	c.curr++
+	c.entries = append(c.entries, entry)
+}
+
+// Peek returns the next cache entry to be expected or nil if the cache is fully consumed.
+func (c *detectorCache) Peek() *detectorCacheEntry {
+	if c.curr < len(c.entries) {
+		return c.entries[c.curr]
+	}
+	return nil
+}
+
+// Save writes the current cache to the given file.
+func (c *detectorCache) Save(cacheFile *paths.Path) error {
+	// Cut off the cache if it is not fully consumed
+	c.entries = c.entries[:c.curr]
+
+	data, err := json.MarshalIndent(c.entries, "", "  ")
+	if err != nil {
+		return err
+	}
+	return cacheFile.WriteFile(data)
+}
diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index 0deb0194661..ed505ece296 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -51,7 +51,7 @@ type libraryResolutionResult struct {
 type SketchLibrariesDetector struct {
 	librariesResolver             *librariesresolver.Cpp
 	useCachedLibrariesResolution  bool
-	cache                         *includeCache
+	cache                         *detectorCache
 	onlyUpdateCompilationDatabase bool
 	importedLibraries             libraries.List
 	librariesResolutionResults    map[string]libraryResolutionResult
@@ -71,6 +71,7 @@ func NewSketchLibrariesDetector(
 	return &SketchLibrariesDetector{
 		librariesResolver:             libsResolver,
 		useCachedLibrariesResolution:  useCachedLibrariesResolution,
+		cache:                         newDetectorCache(),
 		librariesResolutionResults:    map[string]libraryResolutionResult{},
 		importedLibraries:             libraries.List{},
 		includeFolders:                paths.PathList{},
@@ -172,21 +173,10 @@ func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList {
 	return l.includeFolders
 }
 
-// appendIncludeFolder todo should rename this, probably after refactoring the
-// container_find_includes command.
-// Original comment:
-// Append the given folder to the include path and match or append it to
-// the cache. sourceFilePath and include indicate the source of this
-// include (e.g. what #include line in what file it was resolved from)
-// and should be the empty string for the default include folders, like
-// the core or variant.
-func (l *SketchLibrariesDetector) appendIncludeFolder(
-	sourceFilePath *paths.Path,
-	include string,
-	folder *paths.Path,
-) {
+// addIncludeFolder add the given folder to the include path.
+func (l *SketchLibrariesDetector) addIncludeFolder(folder *paths.Path) {
 	l.includeFolders = append(l.includeFolders, folder)
-	l.cache.ExpectEntry(sourceFilePath, include, folder)
+	l.cache.Expect(&detectorCacheEntry{AddedIncludePath: folder})
 }
 
 // FindIncludes todo
@@ -242,11 +232,13 @@ func (l *SketchLibrariesDetector) findIncludes(
 	}
 
 	cachePath := buildPath.Join("includes.cache")
-	l.cache = readCache(cachePath)
+	if err := l.cache.Load(cachePath); err != nil {
+		l.logger.Warn(i18n.Tr("Failed to load library discovery cache: %[1]s", err))
+	}
 
-	l.appendIncludeFolder(nil, "", buildCorePath)
+	l.addIncludeFolder(buildCorePath)
 	if buildVariantPath != nil {
-		l.appendIncludeFolder(nil, "", buildVariantPath)
+		l.addIncludeFolder(buildVariantPath)
 	}
 
 	sourceFileQueue := &uniqueSourceFileQueue{}
@@ -266,7 +258,7 @@ func (l *SketchLibrariesDetector) findIncludes(
 		}
 
 		for !sourceFileQueue.Empty() {
-			err := l.findIncludesUntilDone(ctx, sourceFileQueue, buildProperties, librariesBuildPath, platformArch)
+			err := l.findMissingIncludesInCompilationUnit(ctx, sourceFileQueue, buildProperties, librariesBuildPath, platformArch)
 			if err != nil {
 				cachePath.Remove()
 				return err
@@ -274,8 +266,7 @@ func (l *SketchLibrariesDetector) findIncludes(
 		}
 
 		// Finalize the cache
-		l.cache.ExpectEnd()
-		if err := l.cache.write(cachePath); err != nil {
+		if err := l.cache.Save(cachePath); err != nil {
 			return err
 		}
 	}
@@ -293,7 +284,7 @@ func (l *SketchLibrariesDetector) findIncludes(
 	return nil
 }
 
-func (l *SketchLibrariesDetector) findIncludesUntilDone(
+func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 	ctx context.Context,
 	sourceFileQueue *uniqueSourceFileQueue,
 	buildProperties *properties.Map,
@@ -325,7 +316,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 
 	first := true
 	for {
-		l.cache.ExpectFile(sourcePath)
+		l.cache.Expect(&detectorCacheEntry{CompilingSourcePath: sourcePath})
 
 		// Libraries may require the "utility" directory to be added to the include
 		// search path, but only for the source code of the library, so we temporary
@@ -340,8 +331,8 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 		var preprocFirstResult *runner.Result
 
 		var missingIncludeH string
-		if unchanged && l.cache.valid {
-			missingIncludeH = l.cache.Next().Include
+		if entry := l.cache.Peek(); unchanged && entry != nil && entry.MissingIncludeH != nil {
+			missingIncludeH = *entry.MissingIncludeH
 			if first && l.logger.Verbose() {
 				l.logger.Info(i18n.Tr("Using cached library dependencies for file: %[1]s", sourcePath))
 			}
@@ -367,9 +358,10 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 			}
 		}
 
+		l.cache.Expect(&detectorCacheEntry{MissingIncludeH: &missingIncludeH})
+
 		if missingIncludeH == "" {
 			// No missing includes found, we're done
-			l.cache.ExpectEntry(sourcePath, "", nil)
 			return nil
 		}
 
@@ -402,7 +394,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 		// include path and queue its source files for further
 		// include scanning
 		l.AppendImportedLibraries(library)
-		l.appendIncludeFolder(sourcePath, missingIncludeH, library.SourceDir)
+		l.addIncludeFolder(library.SourceDir)
 
 		if library.Precompiled && library.PrecompiledWithSources {
 			// Fully precompiled libraries should have no dependencies to avoid ABI breakage

From 80ab8058f93c0b54d0485cf445047628d184dfbd Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Fri, 31 May 2024 19:05:24 +0200
Subject: [PATCH 10/28] Remove the old, no longer used, includeCache

---
 .../builder/internal/detector/detector.go     |  17 ---
 .../internal/detector/include_cache.go        | 117 ------------------
 2 files changed, 134 deletions(-)
 delete mode 100644 internal/arduino/builder/internal/detector/include_cache.go

diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index ed505ece296..d8a257a68b7 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -566,20 +566,3 @@ func LibrariesLoader(
 	resolver := librariesresolver.NewCppResolver(allLibs, targetPlatform, actualPlatform)
 	return resolver, verboseOut.Bytes(), nil
 }
-
-type includeCacheEntry struct {
-	Sourcefile  *paths.Path
-	Include     string
-	Includepath *paths.Path
-}
-
-// String fixdoc
-func (entry *includeCacheEntry) String() string {
-	return fmt.Sprintf("SourceFile: %s; Include: %s; IncludePath: %s",
-		entry.Sourcefile, entry.Include, entry.Includepath)
-}
-
-// Equals fixdoc
-func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool {
-	return entry.String() == other.String()
-}
diff --git a/internal/arduino/builder/internal/detector/include_cache.go b/internal/arduino/builder/internal/detector/include_cache.go
deleted file mode 100644
index 7e6937162eb..00000000000
--- a/internal/arduino/builder/internal/detector/include_cache.go
+++ /dev/null
@@ -1,117 +0,0 @@
-// This file is part of arduino-cli.
-//
-// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
-//
-// This software is released under the GNU General Public License version 3,
-// which covers the main part of arduino-cli.
-// The terms of this license can be found at:
-// https://www.gnu.org/licenses/gpl-3.0.en.html
-//
-// You can be released from the requirements of the above licenses by purchasing
-// a commercial license. Buying such a license is mandatory if you want to
-// modify or otherwise use the software for commercial activities involving the
-// Arduino software without disclosing the source code of your own applications.
-// To purchase a commercial license, send an email to license@arduino.cc.
-
-package detector
-
-import (
-	"encoding/json"
-	"time"
-
-	"github.com/arduino/go-paths-helper"
-)
-
-type includeCache struct {
-	// Are the cache contents valid so far?
-	valid bool
-	// Index into entries of the next entry to be processed. Unused
-	// when the cache is invalid.
-	next    int
-	entries []*includeCacheEntry
-}
-
-// Next Return the next cache entry. Should only be called when the cache is
-// valid and a next entry is available (the latter can be checked with
-// ExpectFile). Does not advance the cache.
-func (cache *includeCache) Next() *includeCacheEntry {
-	return cache.entries[cache.next]
-}
-
-// ExpectFile check that the next cache entry is about the given file. If it is
-// not, or no entry is available, the cache is invalidated. Does not
-// advance the cache.
-func (cache *includeCache) ExpectFile(sourcefile *paths.Path) {
-	if cache.valid && (cache.next >= len(cache.entries) || !cache.Next().Sourcefile.EqualsTo(sourcefile)) {
-		cache.valid = false
-		cache.entries = cache.entries[:cache.next]
-	}
-}
-
-// ExpectEntry check that the next entry matches the given values. If so, advance
-// the cache. If not, the cache is invalidated. If the cache is
-// invalidated, or was already invalid, an entry with the given values
-// is appended.
-func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, librarypath *paths.Path) {
-	entry := &includeCacheEntry{Sourcefile: sourcefile, Include: include, Includepath: librarypath}
-	if cache.valid {
-		if cache.next < len(cache.entries) && cache.Next().Equals(entry) {
-			cache.next++
-		} else {
-			cache.valid = false
-			cache.entries = cache.entries[:cache.next]
-		}
-	}
-
-	if !cache.valid {
-		cache.entries = append(cache.entries, entry)
-	}
-}
-
-// ExpectEnd check that the cache is completely consumed. If not, the cache is
-// invalidated.
-func (cache *includeCache) ExpectEnd() {
-	if cache.valid && cache.next < len(cache.entries) {
-		cache.valid = false
-		cache.entries = cache.entries[:cache.next]
-	}
-}
-
-// Read the cache from the given file
-func readCache(path *paths.Path) *includeCache {
-	bytes, err := path.ReadFile()
-	if err != nil {
-		// Return an empty, invalid cache
-		return &includeCache{}
-	}
-	result := &includeCache{}
-	err = json.Unmarshal(bytes, &result.entries)
-	if err != nil {
-		// Return an empty, invalid cache
-		return &includeCache{}
-	}
-	result.valid = true
-	return result
-}
-
-// Write the given cache to the given file if it is invalidated. If the
-// cache is still valid, just update the timestamps of the file.
-func (cache *includeCache) write(path *paths.Path) error {
-	// If the cache was still valid all the way, just touch its file
-	// (in case any source file changed without influencing the
-	// includes). If it was invalidated, overwrite the cache with
-	// the new contents.
-	if cache.valid {
-		path.Chtimes(time.Now(), time.Now())
-	} else {
-		bytes, err := json.MarshalIndent(cache.entries, "", "  ")
-		if err != nil {
-			return err
-		}
-		err = path.WriteFile(bytes)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}

From 94b4b3d716658c1af155ce05be6df028093b36d8 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Fri, 31 May 2024 19:12:01 +0200
Subject: [PATCH 11/28] Simplified error reporting in library detection

There is no need to duplicate the preprocessResult/Err variables.
This also simplifies naming making it more straighforward.
---
 .../builder/internal/detector/detector.go     | 28 +++++++++----------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index d8a257a68b7..15f2a0772c0 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -328,7 +328,7 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 		}
 
 		var preprocErr error
-		var preprocFirstResult *runner.Result
+		var preprocResult *runner.Result
 
 		var missingIncludeH string
 		if entry := l.cache.Peek(); unchanged && entry != nil && entry.MissingIncludeH != nil {
@@ -338,20 +338,20 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 			}
 			first = false
 		} else {
-			preprocFirstResult, preprocErr = preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties)
+			preprocResult, preprocErr = preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties)
 			if l.logger.Verbose() {
-				l.logger.WriteStdout(preprocFirstResult.Stdout)
+				l.logger.WriteStdout(preprocResult.Stdout)
 			}
 			// Unwrap error and see if it is an ExitError.
 			var exitErr *exec.ExitError
 			if preprocErr == nil {
 				// Preprocessor successful, done
 				missingIncludeH = ""
-			} else if isExitErr := errors.As(preprocErr, &exitErr); !isExitErr || len(preprocFirstResult.Stderr) == 0 {
+			} else if isExitErr := errors.As(preprocErr, &exitErr); !isExitErr || len(preprocResult.Stderr) == 0 {
 				// Ignore ExitErrors (e.g. gcc returning non-zero status), but bail out on other errors
 				return preprocErr
 			} else {
-				missingIncludeH = IncludesFinderWithRegExp(string(preprocFirstResult.Stderr))
+				missingIncludeH = IncludesFinderWithRegExp(string(preprocResult.Stderr))
 				if missingIncludeH == "" && l.logger.Verbose() {
 					l.logger.Info(i18n.Tr("Error while detecting libraries included by %[1]s", sourcePath))
 				}
@@ -368,25 +368,23 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 		library := l.resolveLibrary(missingIncludeH, platformArch)
 		if library == nil {
 			// Library could not be resolved, show error
-			if preprocErr == nil || len(preprocFirstResult.Stderr) == 0 {
-				// Filename came from cache, so run preprocessor to obtain error to show
-				result, err := preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties)
+
+			// If preprocess result came from cache, run the preprocessor to obtain the actual error to show
+			if preprocErr == nil || len(preprocResult.Stderr) == 0 {
+				preprocResult, preprocErr = preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties)
 				if l.logger.Verbose() {
-					l.logger.WriteStdout(result.Stdout)
+					l.logger.WriteStdout(preprocResult.Stdout)
 				}
-				if err == nil {
+				if preprocErr == nil {
 					// If there is a missing #include in the cache, but running
 					// gcc does not reproduce that, there is something wrong.
 					// Returning an error here will cause the cache to be
 					// deleted, so hopefully the next compilation will succeed.
 					return errors.New(i18n.Tr("Internal error in cache"))
 				}
-				l.diagnosticStore.Parse(result.Args, result.Stderr)
-				l.logger.WriteStderr(result.Stderr)
-				return err
 			}
-			l.diagnosticStore.Parse(preprocFirstResult.Args, preprocFirstResult.Stderr)
-			l.logger.WriteStderr(preprocFirstResult.Stderr)
+			l.diagnosticStore.Parse(preprocResult.Args, preprocResult.Stderr)
+			l.logger.WriteStderr(preprocResult.Stderr)
 			return preprocErr
 		}
 

From fc7779f1c94d5f555e8bd7d822a71abad2f4dd17 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Sat, 1 Jun 2024 18:10:00 +0200
Subject: [PATCH 12/28] Remove useless targetFilePath variable

---
 internal/arduino/builder/internal/detector/detector.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index 15f2a0772c0..7bb84473fb0 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -293,7 +293,6 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 ) error {
 	sourceFile := sourceFileQueue.Pop()
 	sourcePath := sourceFile.SourcePath()
-	targetFilePath := paths.NullPath()
 	depPath := sourceFile.DepfilePath()
 	objPath := sourceFile.ObjectPath()
 
@@ -338,7 +337,7 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 			}
 			first = false
 		} else {
-			preprocResult, preprocErr = preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties)
+			preprocResult, preprocErr = preprocessor.GCC(ctx, sourcePath, paths.NullPath(), includeFolders, buildProperties)
 			if l.logger.Verbose() {
 				l.logger.WriteStdout(preprocResult.Stdout)
 			}
@@ -371,7 +370,7 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 
 			// If preprocess result came from cache, run the preprocessor to obtain the actual error to show
 			if preprocErr == nil || len(preprocResult.Stderr) == 0 {
-				preprocResult, preprocErr = preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties)
+				preprocResult, preprocErr = preprocessor.GCC(ctx, sourcePath, paths.NullPath(), includeFolders, buildProperties)
 				if l.logger.Verbose() {
 					l.logger.WriteStdout(preprocResult.Stdout)
 				}

From d072f10aa41a9d78e7850739c7d9a98d72382ad9 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Sun, 2 Jun 2024 18:51:18 +0200
Subject: [PATCH 13/28] Slight improvement of removeBuildFromSketchFiles

---
 commands/service_compile.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/commands/service_compile.go b/commands/service_compile.go
index 69865f3522c..05e9b0c9cd2 100644
--- a/commands/service_compile.go
+++ b/commands/service_compile.go
@@ -165,7 +165,7 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 	if buildPathArg := req.GetBuildPath(); buildPathArg != "" {
 		buildPath = paths.New(req.GetBuildPath()).Canonical()
 		if in, _ := buildPath.IsInsideDir(sk.FullPath); in && buildPath.IsDir() {
-			if sk.AdditionalFiles, err = removeBuildFromSketchFiles(sk.AdditionalFiles, buildPath); err != nil {
+			if sk.AdditionalFiles, err = removeBuildPathFromSketchFiles(sk.AdditionalFiles, buildPath); err != nil {
 				return err
 			}
 		}
@@ -434,15 +434,15 @@ func maybePurgeBuildCache(compilationsBeforePurge uint, cacheTTL time.Duration)
 	buildcache.New(paths.TempDir().Join("arduino", "sketches")).Purge(cacheTTL)
 }
 
-// removeBuildFromSketchFiles removes the files contained in the build directory from
+// removeBuildPathFromSketchFiles removes the files contained in the build directory from
 // the list of the sketch files
-func removeBuildFromSketchFiles(files paths.PathList, build *paths.Path) (paths.PathList, error) {
+func removeBuildPathFromSketchFiles(files paths.PathList, build *paths.Path) (paths.PathList, error) {
 	var res paths.PathList
 	ignored := false
 	for _, file := range files {
 		if isInside, _ := file.IsInsideDir(build); !isInside {
-			res = append(res, file)
-		} else if !ignored {
+			res.Add(file)
+		} else {
 			ignored = true
 		}
 	}

From 55021c8207bc8f0406ac2c84016b7c54ba41f55e Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Sun, 2 Jun 2024 19:09:40 +0200
Subject: [PATCH 14/28] Rename variables for clarity

---
 commands/service_compile.go         | 15 ++++++++-------
 internal/arduino/builder/builder.go | 21 +++++++++++----------
 internal/arduino/builder/core.go    |  2 +-
 3 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/commands/service_compile.go b/commands/service_compile.go
index 05e9b0c9cd2..6a739c30a96 100644
--- a/commands/service_compile.go
+++ b/commands/service_compile.go
@@ -214,10 +214,6 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 		return err
 	}
 
-	actualPlatform := buildPlatform
-	otherLibrariesDirs := paths.NewPathList(req.GetLibraries()...)
-	otherLibrariesDirs.Add(s.settings.LibrariesDir())
-
 	var libsManager *librariesmanager.LibrariesManager
 	if pme.GetProfile() != nil {
 		libsManager = lm
@@ -240,6 +236,11 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 			Message: &rpc.CompileResponse_Progress{Progress: p},
 		})
 	}
+
+	librariesDirs := paths.NewPathList(req.GetLibraries()...) // Array of collection of libraries directories
+	librariesDirs.Add(s.settings.LibrariesDir())
+	libraryDirs := paths.NewPathList(req.GetLibrary()...) // Array of single-library directories
+
 	sketchBuilder, err := builder.NewBuilder(
 		ctx,
 		sk,
@@ -251,16 +252,16 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 		int(req.GetJobs()),
 		req.GetBuildProperties(),
 		s.settings.HardwareDirectories(),
-		otherLibrariesDirs,
+		librariesDirs,
 		s.settings.IDEBuiltinLibrariesDir(),
 		fqbn,
 		req.GetClean(),
 		req.GetSourceOverride(),
 		req.GetCreateCompilationDatabaseOnly(),
-		targetPlatform, actualPlatform,
+		targetPlatform, buildPlatform,
 		req.GetSkipLibrariesDiscovery(),
 		libsManager,
-		paths.NewPathList(req.GetLibrary()...),
+		libraryDirs,
 		outStream, errStream, req.GetVerbose(), req.GetWarnings(),
 		progressCB,
 		pme.GetEnvVarsForSpawnedProcess(),
diff --git a/internal/arduino/builder/builder.go b/internal/arduino/builder/builder.go
index 041d488067f..714f51f2801 100644
--- a/internal/arduino/builder/builder.go
+++ b/internal/arduino/builder/builder.go
@@ -88,7 +88,7 @@ type Builder struct {
 	lineOffset int
 
 	targetPlatform *cores.PlatformRelease
-	actualPlatform *cores.PlatformRelease
+	buildPlatform  *cores.PlatformRelease
 
 	buildArtifacts *buildArtifacts
 
@@ -125,18 +125,19 @@ func NewBuilder(
 	extraCoreBuildCachePaths paths.PathList,
 	jobs int,
 	requestBuildProperties []string,
-	hardwareDirs, otherLibrariesDirs paths.PathList,
+	hardwareDirs paths.PathList,
+	librariesDirs paths.PathList,
 	builtInLibrariesDirs *paths.Path,
 	fqbn *cores.FQBN,
 	clean bool,
 	sourceOverrides map[string]string,
 	onlyUpdateCompilationDatabase bool,
-	targetPlatform, actualPlatform *cores.PlatformRelease,
+	targetPlatform, buildPlatform *cores.PlatformRelease,
 	useCachedLibrariesResolution bool,
 	librariesManager *librariesmanager.LibrariesManager,
-	libraryDirs paths.PathList,
+	customLibraryDirs paths.PathList,
 	stdout, stderr io.Writer, verbose bool, warningsLevel string,
-	progresCB rpc.TaskProgressCB,
+	progressCB rpc.TaskProgressCB,
 	toolEnv []string,
 ) (*Builder, error) {
 	buildProperties := properties.NewMap()
@@ -191,8 +192,8 @@ func NewBuilder(
 	logger := logger.New(stdout, stderr, verbose, warningsLevel)
 	libsResolver, verboseOut, err := detector.LibrariesLoader(
 		useCachedLibrariesResolution, librariesManager,
-		builtInLibrariesDirs, libraryDirs, otherLibrariesDirs,
-		actualPlatform, targetPlatform,
+		builtInLibrariesDirs, customLibraryDirs, librariesDirs,
+		buildPlatform, targetPlatform,
 	)
 	if err != nil {
 		return nil, err
@@ -219,14 +220,14 @@ func NewBuilder(
 		sourceOverrides:               sourceOverrides,
 		onlyUpdateCompilationDatabase: onlyUpdateCompilationDatabase,
 		compilationDatabase:           compilation.NewDatabase(buildPath.Join("compile_commands.json")),
-		Progress:                      progress.New(progresCB),
+		Progress:                      progress.New(progressCB),
 		executableSectionsSize:        []ExecutableSectionSize{},
 		buildArtifacts:                &buildArtifacts{},
 		targetPlatform:                targetPlatform,
-		actualPlatform:                actualPlatform,
+		buildPlatform:                 buildPlatform,
 		toolEnv:                       toolEnv,
 		buildOptions: newBuildOptions(
-			hardwareDirs, otherLibrariesDirs,
+			hardwareDirs, librariesDirs,
 			builtInLibrariesDirs, buildPath,
 			sk,
 			customBuildPropertiesArgs,
diff --git a/internal/arduino/builder/core.go b/internal/arduino/builder/core.go
index f407ed60c41..c3e7d4bb04b 100644
--- a/internal/arduino/builder/core.go
+++ b/internal/arduino/builder/core.go
@@ -163,7 +163,7 @@ func (b *Builder) compileCore() (*paths.Path, paths.PathList, error) {
 				b.logger.Info(i18n.Tr("Archiving built core (caching) in: %[1]s", targetArchivedCore))
 			} else if os.IsNotExist(err) {
 				b.logger.Info(i18n.Tr("Unable to cache built core, please tell %[1]s maintainers to follow %[2]s",
-					b.actualPlatform,
+					b.buildPlatform,
 					"https://arduino.github.io/arduino-cli/latest/platform-specification/#recipes-to-build-the-corea-archive-file"))
 			} else {
 				b.logger.Info(i18n.Tr("Error archiving built core (caching) in %[1]s: %[2]s", targetArchivedCore, err))

From 39f200029f9790e61c27b716626b66335bfca613 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Sun, 2 Jun 2024 19:28:41 +0200
Subject: [PATCH 15/28] Removed hardcoded build.warn_data_percentage in
 build.options file

Also fixed the "low memory" warning printer.
---
 internal/arduino/builder/builder.go                |  7 +------
 internal/arduino/builder/sizer.go                  | 14 ++++++++------
 internal/integrationtest/compile_4/compile_test.go |  9 ++++-----
 3 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/internal/arduino/builder/builder.go b/internal/arduino/builder/builder.go
index 714f51f2801..fb3e555f823 100644
--- a/internal/arduino/builder/builder.go
+++ b/internal/arduino/builder/builder.go
@@ -58,9 +58,6 @@ type Builder struct {
 	// Parallel processes
 	jobs int
 
-	// Custom build properties defined by user (line by line as "key=value" pairs)
-	customBuildProperties []string
-
 	// core related
 	coreBuildCachePath       *paths.Path
 	extraCoreBuildCachePaths paths.PathList
@@ -170,7 +167,6 @@ func NewBuilder(
 		return nil, fmt.Errorf("invalid build properties: %w", err)
 	}
 	buildProperties.Merge(customBuildProperties)
-	customBuildPropertiesArgs := append(requestBuildProperties, "build.warn_data_percentage=75")
 
 	sketchBuildPath, err := buildPath.Join("sketch").Abs()
 	if err != nil {
@@ -212,7 +208,6 @@ func NewBuilder(
 		coreBuildPath:                 coreBuildPath,
 		librariesBuildPath:            librariesBuildPath,
 		jobs:                          jobs,
-		customBuildProperties:         customBuildPropertiesArgs,
 		coreBuildCachePath:            coreBuildCachePath,
 		extraCoreBuildCachePaths:      extraCoreBuildCachePaths,
 		logger:                        logger,
@@ -230,7 +225,7 @@ func NewBuilder(
 			hardwareDirs, librariesDirs,
 			builtInLibrariesDirs, buildPath,
 			sk,
-			customBuildPropertiesArgs,
+			requestBuildProperties,
 			fqbn,
 			clean,
 			buildProperties.Get("compiler.optimization_flags"),
diff --git a/internal/arduino/builder/sizer.go b/internal/arduino/builder/sizer.go
index 84cf8012a32..87ec1f24870 100644
--- a/internal/arduino/builder/sizer.go
+++ b/internal/arduino/builder/sizer.go
@@ -196,15 +196,17 @@ func (b *Builder) checkSize() (ExecutablesFileSections, error) {
 		return executableSectionsSize, errors.New(i18n.Tr("data section exceeds available space in board"))
 	}
 
+	warnDataPercentage := 75
 	if w := properties.Get("build.warn_data_percentage"); w != "" {
-		warnDataPercentage, err := strconv.Atoi(w)
-		if err != nil {
-			return executableSectionsSize, err
-		}
-		if maxDataSize > 0 && dataSize > maxDataSize*warnDataPercentage/100 {
-			b.logger.Warn(i18n.Tr("Low memory available, stability problems may occur."))
+		if p, err := strconv.Atoi(w); err == nil {
+			warnDataPercentage = p
+		} else {
+			b.logger.Warn(i18n.Tr("Invalid value for build.warn_data_percentage: %s", w))
 		}
 	}
+	if maxDataSize > 0 && dataSize > maxDataSize*warnDataPercentage/100 {
+		b.logger.Warn(i18n.Tr("Low memory available, stability problems may occur."))
+	}
 
 	return executableSectionsSize, nil
 }
diff --git a/internal/integrationtest/compile_4/compile_test.go b/internal/integrationtest/compile_4/compile_test.go
index a909cab3ab8..5ae8b6b10a4 100644
--- a/internal/integrationtest/compile_4/compile_test.go
+++ b/internal/integrationtest/compile_4/compile_test.go
@@ -1038,15 +1038,14 @@ func TestBuildOptionsFile(t *testing.T) {
 		"sketchLocation"
 	]`)
 	requirejson.Query(t, buildOptionsBytes, ".fqbn", `"arduino:avr:uno"`)
-	requirejson.Query(t, buildOptionsBytes, ".customBuildProperties", `"build.warn_data_percentage=75"`)
 
 	// Recompiling a second time should provide the same result
 	_, _, err = cli.Run("compile", "-b", "arduino:avr:uno", "--build-path", buildPath.String(), sketchPath.String())
 	require.NoError(t, err)
 
-	buildOptionsBytes, err = buildPath.Join("build.options.json").ReadFile()
+	buildOptionsBytes2, err := buildPath.Join("build.options.json").ReadFile()
 	require.NoError(t, err)
-	requirejson.Query(t, buildOptionsBytes, ".customBuildProperties", `"build.warn_data_percentage=75"`)
+	require.Equal(t, buildOptionsBytes, buildOptionsBytes2)
 
 	// Recompiling with a new build option must produce a new `build.options.json`
 	_, _, err = cli.Run("compile", "-b", "arduino:avr:uno", "--build-path", buildPath.String(),
@@ -1055,7 +1054,7 @@ func TestBuildOptionsFile(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	buildOptionsBytes, err = buildPath.Join("build.options.json").ReadFile()
+	buildOptionsBytes3, err := buildPath.Join("build.options.json").ReadFile()
 	require.NoError(t, err)
-	requirejson.Query(t, buildOptionsBytes, ".customBuildProperties", `"custom=prop,build.warn_data_percentage=75"`)
+	require.NotEqual(t, buildOptionsBytes, buildOptionsBytes3)
 }

From 9c19adf743c12657a6c95282ee526dc76168a367 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Sun, 2 Jun 2024 19:32:00 +0200
Subject: [PATCH 16/28] Renamed variables for clarity

---
 internal/arduino/builder/builder.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/internal/arduino/builder/builder.go b/internal/arduino/builder/builder.go
index fb3e555f823..73dc9259a66 100644
--- a/internal/arduino/builder/builder.go
+++ b/internal/arduino/builder/builder.go
@@ -121,7 +121,7 @@ func NewBuilder(
 	coreBuildCachePath *paths.Path,
 	extraCoreBuildCachePaths paths.PathList,
 	jobs int,
-	requestBuildProperties []string,
+	customBuildProperties []string,
 	hardwareDirs paths.PathList,
 	librariesDirs paths.PathList,
 	builtInLibrariesDirs *paths.Path,
@@ -162,11 +162,11 @@ func NewBuilder(
 	}
 
 	// Add user provided custom build properties
-	customBuildProperties, err := properties.LoadFromSlice(requestBuildProperties)
-	if err != nil {
+	if p, err := properties.LoadFromSlice(customBuildProperties); err == nil {
+		buildProperties.Merge(p)
+	} else {
 		return nil, fmt.Errorf("invalid build properties: %w", err)
 	}
-	buildProperties.Merge(customBuildProperties)
 
 	sketchBuildPath, err := buildPath.Join("sketch").Abs()
 	if err != nil {
@@ -225,7 +225,7 @@ func NewBuilder(
 			hardwareDirs, librariesDirs,
 			builtInLibrariesDirs, buildPath,
 			sk,
-			requestBuildProperties,
+			customBuildProperties,
 			fqbn,
 			clean,
 			buildProperties.Get("compiler.optimization_flags"),

From 4f069011bebd6918c00bed16d827ff220c2f9a45 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Sun, 2 Jun 2024 19:48:19 +0200
Subject: [PATCH 17/28] Renamed variables for clarity

---
 internal/arduino/builder/builder.go           | 26 ++++++++-------
 .../builder/internal/detector/detector.go     | 32 ++++++++-----------
 2 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/internal/arduino/builder/builder.go b/internal/arduino/builder/builder.go
index 73dc9259a66..d267b9ec415 100644
--- a/internal/arduino/builder/builder.go
+++ b/internal/arduino/builder/builder.go
@@ -143,14 +143,12 @@ func NewBuilder(
 	}
 	if sk != nil {
 		buildProperties.SetPath("sketch_path", sk.FullPath)
+		buildProperties.Set("build.project_name", sk.MainFile.Base())
+		buildProperties.SetPath("build.source.path", sk.FullPath)
 	}
 	if buildPath != nil {
 		buildProperties.SetPath("build.path", buildPath)
 	}
-	if sk != nil {
-		buildProperties.Set("build.project_name", sk.MainFile.Base())
-		buildProperties.SetPath("build.source.path", sk.FullPath)
-	}
 	if optimizeForDebug {
 		if debugFlags, ok := buildProperties.GetOk("compiler.optimization_flags.debug"); ok {
 			buildProperties.Set("compiler.optimization_flags", debugFlags)
@@ -186,16 +184,20 @@ func NewBuilder(
 	}
 
 	logger := logger.New(stdout, stderr, verbose, warningsLevel)
-	libsResolver, verboseOut, err := detector.LibrariesLoader(
-		useCachedLibrariesResolution, librariesManager,
-		builtInLibrariesDirs, customLibraryDirs, librariesDirs,
-		buildPlatform, targetPlatform,
+	libsResolver, libsLoadingWarnings, err := detector.LibrariesLoader(
+		useCachedLibrariesResolution,
+		librariesManager,
+		builtInLibrariesDirs,
+		customLibraryDirs,
+		librariesDirs,
+		buildPlatform,
+		targetPlatform,
 	)
 	if err != nil {
 		return nil, err
 	}
 	if logger.Verbose() {
-		logger.Warn(string(verboseOut))
+		logger.Warn(string(libsLoadingWarnings))
 	}
 
 	diagnosticStore := diagnostics.NewStore()
@@ -222,8 +224,10 @@ func NewBuilder(
 		buildPlatform:                 buildPlatform,
 		toolEnv:                       toolEnv,
 		buildOptions: newBuildOptions(
-			hardwareDirs, librariesDirs,
-			builtInLibrariesDirs, buildPath,
+			hardwareDirs,
+			librariesDirs,
+			builtInLibrariesDirs,
+			buildPath,
 			sk,
 			customBuildProperties,
 			fqbn,
diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index 7bb84473fb0..d93dcaae41c 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -490,8 +490,11 @@ func findIncludeForOldCompilers(source string) string {
 func LibrariesLoader(
 	useCachedLibrariesResolution bool,
 	librariesManager *librariesmanager.LibrariesManager,
-	builtInLibrariesDirs *paths.Path, libraryDirs, otherLibrariesDirs paths.PathList,
-	actualPlatform, targetPlatform *cores.PlatformRelease,
+	builtInLibrariesDir *paths.Path,
+	customLibraryDirs paths.PathList,
+	librariesDirs paths.PathList,
+	buildPlatform *cores.PlatformRelease,
+	targetPlatform *cores.PlatformRelease,
 ) (*librariesresolver.Cpp, []byte, error) {
 	verboseOut := &bytes.Buffer{}
 	lm := librariesManager
@@ -503,21 +506,20 @@ func LibrariesLoader(
 	if librariesManager == nil {
 		lmb := librariesmanager.NewBuilder()
 
-		builtInLibrariesFolders := builtInLibrariesDirs
-		if builtInLibrariesFolders != nil {
-			if err := builtInLibrariesFolders.ToAbs(); err != nil {
+		if builtInLibrariesDir != nil {
+			if err := builtInLibrariesDir.ToAbs(); err != nil {
 				return nil, nil, err
 			}
 			lmb.AddLibrariesDir(librariesmanager.LibrariesDir{
-				Path:     builtInLibrariesFolders,
+				Path:     builtInLibrariesDir,
 				Location: libraries.IDEBuiltIn,
 			})
 		}
 
-		if actualPlatform != targetPlatform {
+		if buildPlatform != targetPlatform {
 			lmb.AddLibrariesDir(librariesmanager.LibrariesDir{
-				PlatformRelease: actualPlatform,
-				Path:            actualPlatform.GetLibrariesDir(),
+				PlatformRelease: buildPlatform,
+				Path:            buildPlatform.GetLibrariesDir(),
 				Location:        libraries.ReferencedPlatformBuiltIn,
 			})
 		}
@@ -527,7 +529,7 @@ func LibrariesLoader(
 			Location:        libraries.PlatformBuiltIn,
 		})
 
-		librariesFolders := otherLibrariesDirs
+		librariesFolders := librariesDirs
 		if err := librariesFolders.ToAbs(); err != nil {
 			return nil, nil, err
 		}
@@ -538,7 +540,7 @@ func LibrariesLoader(
 			})
 		}
 
-		for _, dir := range libraryDirs {
+		for _, dir := range customLibraryDirs {
 			lmb.AddLibrariesDir(librariesmanager.LibrariesDir{
 				Path:            dir,
 				Location:        libraries.Unmanaged,
@@ -548,18 +550,12 @@ func LibrariesLoader(
 
 		newLm, libsLoadingWarnings := lmb.Build()
 		for _, status := range libsLoadingWarnings {
-			// With the refactoring of the initialization step of the CLI we changed how
-			// errors are returned when loading platforms and libraries, that meant returning a list of
-			// errors instead of a single one to enhance the experience for the user.
-			// I have no intention right now to start a refactoring of the legacy package too, so
-			// here's this shitty solution for now.
-			// When we're gonna refactor the legacy package this will be gone.
 			verboseOut.Write([]byte(status.Message()))
 		}
 		lm = newLm
 	}
 
 	allLibs := lm.FindAllInstalled()
-	resolver := librariesresolver.NewCppResolver(allLibs, targetPlatform, actualPlatform)
+	resolver := librariesresolver.NewCppResolver(allLibs, targetPlatform, buildPlatform)
 	return resolver, verboseOut.Bytes(), nil
 }

From a808c176b045d59cbde891e8488136cc7fbea704 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Mon, 3 Jun 2024 10:31:06 +0200
Subject: [PATCH 18/28] Pre-compute sourceFile fields, and save the in the
 includes.cache

---
 .../builder/internal/detector/cache.go        | 10 +--
 .../builder/internal/detector/detector.go     | 10 +--
 .../builder/internal/detector/source_file.go  | 79 ++++++++-----------
 3 files changed, 42 insertions(+), 57 deletions(-)

diff --git a/internal/arduino/builder/internal/detector/cache.go b/internal/arduino/builder/internal/detector/cache.go
index 247a1e8de30..d2ca6b526f0 100644
--- a/internal/arduino/builder/internal/detector/cache.go
+++ b/internal/arduino/builder/internal/detector/cache.go
@@ -28,17 +28,17 @@ type detectorCache struct {
 }
 
 type detectorCacheEntry struct {
-	AddedIncludePath    *paths.Path `json:"added_include_path,omitempty"`
-	CompilingSourcePath *paths.Path `json:"compiling_source_path,omitempty"`
-	MissingIncludeH     *string     `json:"missing_include_h,omitempty"`
+	AddedIncludePath *paths.Path `json:"added_include_path,omitempty"`
+	Compile          *sourceFile `json:"compile,omitempty"`
+	MissingIncludeH  *string     `json:"missing_include_h,omitempty"`
 }
 
 func (e *detectorCacheEntry) String() string {
 	if e.AddedIncludePath != nil {
 		return "Added include path: " + e.AddedIncludePath.String()
 	}
-	if e.CompilingSourcePath != nil {
-		return "Compiling source path: " + e.CompilingSourcePath.String()
+	if e.Compile != nil {
+		return "Compiling: " + e.Compile.String()
 	}
 	if e.MissingIncludeH != nil {
 		if *e.MissingIncludeH == "" {
diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index d93dcaae41c..a6e796da4b6 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -292,9 +292,9 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 	platformArch string,
 ) error {
 	sourceFile := sourceFileQueue.Pop()
-	sourcePath := sourceFile.SourcePath()
-	depPath := sourceFile.DepfilePath()
-	objPath := sourceFile.ObjectPath()
+	sourcePath := sourceFile.SourcePath
+	depPath := sourceFile.DepfilePath
+	objPath := sourceFile.ObjectPath
 
 	// TODO: This should perhaps also compare against the
 	// include.cache file timestamp. Now, it only checks if the file
@@ -315,14 +315,14 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 
 	first := true
 	for {
-		l.cache.Expect(&detectorCacheEntry{CompilingSourcePath: sourcePath})
+		l.cache.Expect(&detectorCacheEntry{Compile: sourceFile})
 
 		// Libraries may require the "utility" directory to be added to the include
 		// search path, but only for the source code of the library, so we temporary
 		// copy the current search path list and add the library' utility directory
 		// if needed.
 		includeFolders := l.includeFolders
-		if extraInclude := sourceFile.ExtraIncludePath(); extraInclude != nil {
+		if extraInclude := sourceFile.ExtraIncludePath; extraInclude != nil {
 			includeFolders = append(includeFolders, extraInclude)
 		}
 
diff --git a/internal/arduino/builder/internal/detector/source_file.go b/internal/arduino/builder/internal/detector/source_file.go
index 3ebb52d0387..f5900b7529c 100644
--- a/internal/arduino/builder/internal/detector/source_file.go
+++ b/internal/arduino/builder/internal/detector/source_file.go
@@ -16,85 +16,70 @@
 package detector
 
 import (
+	"fmt"
 	"slices"
 
 	"github.com/arduino/go-paths-helper"
 )
 
 type sourceFile struct {
-	// Path to the source file within the sketch/library root folder
-	relativePath *paths.Path
+	// SourcePath is the path to the source file
+	SourcePath *paths.Path `json:"source_path"`
+
+	// ObjectPath is the path to the object file that will be generated
+	ObjectPath *paths.Path `json:"object_path"`
+
+	// DepfilePath is the path to the dependency file that will be generated
+	DepfilePath *paths.Path `json:"depfile_path"`
 
 	// ExtraIncludePath contains an extra include path that must be
 	// used to compile this source file.
 	// This is mainly used for source files that comes from old-style libraries
 	// (Arduino IDE <1.5) requiring an extra include path to the "utility" folder.
-	extraIncludePath *paths.Path
-
-	// The source root for the given origin, where its source files
-	// can be found. Prepending this to SourceFile.RelativePath will give
-	// the full path to that source file.
-	sourceRoot *paths.Path
+	ExtraIncludePath *paths.Path `json:"extra_include_path,omitempty"`
+}
 
-	// The build root for the given origin, where build products will
-	// be placed. Any directories inside SourceFile.RelativePath will be
-	// appended here.
-	buildRoot *paths.Path
+func (f *sourceFile) String() string {
+	return fmt.Sprintf("SourcePath:%s SourceRoot:%s BuildRoot:%s ExtraInclude:%s",
+		f.SourcePath, f.ObjectPath, f.DepfilePath, f.ExtraIncludePath)
 }
 
-// Equals fixdoc
+// Equals checks if a sourceFile is equal to another.
 func (f *sourceFile) Equals(g *sourceFile) bool {
-	return f.relativePath.EqualsTo(g.relativePath) &&
-		f.buildRoot.EqualsTo(g.buildRoot) &&
-		f.sourceRoot.EqualsTo(g.sourceRoot)
+	return f.SourcePath.EqualsTo(g.SourcePath) &&
+		f.ObjectPath.EqualsTo(g.ObjectPath) &&
+		f.DepfilePath.EqualsTo(g.DepfilePath) &&
+		((f.ExtraIncludePath == nil && g.ExtraIncludePath == nil) ||
+			(f.ExtraIncludePath != nil && g.ExtraIncludePath != nil && f.ExtraIncludePath.EqualsTo(g.ExtraIncludePath)))
 }
 
 // makeSourceFile create a sourceFile object for the given source file path.
 // The given sourceFilePath can be absolute, or relative within the sourceRoot root folder.
-func makeSourceFile(sourceRoot, buildRoot, sourceFilePath *paths.Path, extraIncludePath ...*paths.Path) (*sourceFile, error) {
-	res := &sourceFile{
-		buildRoot:  buildRoot,
-		sourceRoot: sourceRoot,
-	}
-
-	if len(extraIncludePath) > 1 {
+func makeSourceFile(sourceRoot, buildRoot, sourceFilePath *paths.Path, extraIncludePaths ...*paths.Path) (*sourceFile, error) {
+	if len(extraIncludePaths) > 1 {
 		panic("only one extra include path allowed")
 	}
-	if len(extraIncludePath) > 0 {
-		res.extraIncludePath = extraIncludePath[0]
+	var extraIncludePath *paths.Path
+	if len(extraIncludePaths) > 0 {
+		extraIncludePath = extraIncludePaths[0]
 	}
 
 	if sourceFilePath.IsAbs() {
 		var err error
-		sourceFilePath, err = res.sourceRoot.RelTo(sourceFilePath)
+		sourceFilePath, err = sourceRoot.RelTo(sourceFilePath)
 		if err != nil {
 			return nil, err
 		}
 	}
-	res.relativePath = sourceFilePath
+	res := &sourceFile{
+		SourcePath:       sourceRoot.JoinPath(sourceFilePath),
+		ObjectPath:       buildRoot.Join(sourceFilePath.String() + ".o"),
+		DepfilePath:      buildRoot.Join(sourceFilePath.String() + ".d"),
+		ExtraIncludePath: extraIncludePath,
+	}
 	return res, nil
 }
 
-// ExtraIncludePath returns the extra include path required to build the source file.
-func (f *sourceFile) ExtraIncludePath() *paths.Path {
-	return f.extraIncludePath
-}
-
-// SourcePath return the full path to the source file.
-func (f *sourceFile) SourcePath() *paths.Path {
-	return f.sourceRoot.JoinPath(f.relativePath)
-}
-
-// ObjectPath return the full path to the object file.
-func (f *sourceFile) ObjectPath() *paths.Path {
-	return f.buildRoot.Join(f.relativePath.String() + ".o")
-}
-
-// DepfilePath return the full path to the dependency file.
-func (f *sourceFile) DepfilePath() *paths.Path {
-	return f.buildRoot.Join(f.relativePath.String() + ".d")
-}
-
 // uniqueSourceFileQueue is a queue of source files that does not allow duplicates.
 type uniqueSourceFileQueue []*sourceFile
 

From f1f8327b07933a19cd57a755a5087d3b68a8078b Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Mon, 3 Jun 2024 10:34:52 +0200
Subject: [PATCH 19/28] Added ObjFileIsUpToDate method to sourceFile

---
 internal/arduino/builder/internal/detector/detector.go    | 4 +---
 internal/arduino/builder/internal/detector/source_file.go | 6 ++++++
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index a6e796da4b6..8e36d8167d2 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -293,8 +293,6 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 ) error {
 	sourceFile := sourceFileQueue.Pop()
 	sourcePath := sourceFile.SourcePath
-	depPath := sourceFile.DepfilePath
-	objPath := sourceFile.ObjectPath
 
 	// TODO: This should perhaps also compare against the
 	// include.cache file timestamp. Now, it only checks if the file
@@ -308,7 +306,7 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 	// TODO: This reads the dependency file, but the actual building
 	// does it again. Should the result be somehow cached? Perhaps
 	// remove the object file if it is found to be stale?
-	unchanged, err := utils.ObjFileIsUpToDate(sourcePath, objPath, depPath)
+	unchanged, err := sourceFile.ObjFileIsUpToDate()
 	if err != nil {
 		return err
 	}
diff --git a/internal/arduino/builder/internal/detector/source_file.go b/internal/arduino/builder/internal/detector/source_file.go
index f5900b7529c..d99cb1d862a 100644
--- a/internal/arduino/builder/internal/detector/source_file.go
+++ b/internal/arduino/builder/internal/detector/source_file.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"slices"
 
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/utils"
 	"github.com/arduino/go-paths-helper"
 )
 
@@ -80,6 +81,11 @@ func makeSourceFile(sourceRoot, buildRoot, sourceFilePath *paths.Path, extraIncl
 	return res, nil
 }
 
+// ObjFileIsUpToDate checks if the compile object file is up to date.
+func (f *sourceFile) ObjFileIsUpToDate() (unchanged bool, err error) {
+	return utils.ObjFileIsUpToDate(f.SourcePath, f.ObjectPath, f.DepfilePath)
+}
+
 // uniqueSourceFileQueue is a queue of source files that does not allow duplicates.
 type uniqueSourceFileQueue []*sourceFile
 

From 86bdc5fda47c6b2537b25e5a0699932722bba9f1 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Wed, 5 Jun 2024 00:25:20 +0200
Subject: [PATCH 20/28] Implemented parallel task runner

---
 .../arduino/builder/internal/runner/runner.go | 112 ++++++++++++++++++
 .../builder/internal/runner/runner_test.go    |  53 +++++++++
 .../arduino/builder/internal/runner/task.go   |  37 ++++++
 3 files changed, 202 insertions(+)
 create mode 100644 internal/arduino/builder/internal/runner/runner.go
 create mode 100644 internal/arduino/builder/internal/runner/runner_test.go

diff --git a/internal/arduino/builder/internal/runner/runner.go b/internal/arduino/builder/internal/runner/runner.go
new file mode 100644
index 00000000000..d34d3df3b28
--- /dev/null
+++ b/internal/arduino/builder/internal/runner/runner.go
@@ -0,0 +1,112 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2024 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package runner
+
+import (
+	"context"
+	"runtime"
+	"sync"
+)
+
+// Runner is a helper to run commands in a queue, the commands are immediately exectuded
+// in a goroutine as they are enqueued. The runner can be stopped by calling Cancel.
+type Runner struct {
+	lock      sync.Mutex
+	queue     chan<- *enqueuedCommand
+	results   map[string]<-chan *Result
+	ctx       context.Context
+	ctxCancel func()
+	wg        sync.WaitGroup
+}
+
+type enqueuedCommand struct {
+	task   *Task
+	accept func(*Result)
+}
+
+func (cmd *enqueuedCommand) String() string {
+	return cmd.task.String()
+}
+
+func New(inCtx context.Context) *Runner {
+	ctx, cancel := context.WithCancel(inCtx)
+	queue := make(chan *enqueuedCommand, 1000)
+	r := &Runner{
+		ctx:       ctx,
+		ctxCancel: cancel,
+		queue:     queue,
+		results:   map[string]<-chan *Result{},
+	}
+
+	// Spawn workers
+	for i := 0; i < runtime.NumCPU(); i++ {
+		r.wg.Add(1)
+		go func() {
+			worker(ctx, queue)
+			r.wg.Done()
+		}()
+	}
+
+	return r
+}
+
+func worker(ctx context.Context, queue <-chan *enqueuedCommand) {
+	done := ctx.Done()
+	for {
+		select {
+		case <-done:
+			return
+		default:
+		}
+
+		select {
+		case <-done:
+			return
+		case cmd := <-queue:
+			result := cmd.task.Run(ctx)
+			cmd.accept(result)
+		}
+	}
+}
+
+func (r *Runner) Enqueue(task *Task) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	result := make(chan *Result, 1)
+	r.results[task.String()] = result
+	r.queue <- &enqueuedCommand{
+		task: task,
+		accept: func(res *Result) {
+			result <- res
+		},
+	}
+}
+
+func (r *Runner) Results(task *Task) *Result {
+	r.lock.Lock()
+	result, ok := r.results[task.String()]
+	r.lock.Unlock()
+	if !ok {
+		return nil
+	}
+	return <-result
+}
+
+func (r *Runner) Cancel() {
+	r.ctxCancel()
+	r.wg.Wait()
+}
diff --git a/internal/arduino/builder/internal/runner/runner_test.go b/internal/arduino/builder/internal/runner/runner_test.go
new file mode 100644
index 00000000000..90a72d4654a
--- /dev/null
+++ b/internal/arduino/builder/internal/runner/runner_test.go
@@ -0,0 +1,53 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2024 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package runner_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRunMultipleTask(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	r := runner.New(ctx)
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 1 ; echo -n 0"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 2 ; echo -n 1"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 3 ; echo -n 2"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 4 ; echo -n 3"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 5 ; echo -n 4"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 6 ; echo -n 5"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 7 ; echo -n 6"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 8 ; echo -n 7"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 9 ; echo -n 8"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 10 ; echo -n 9"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 11 ; echo -n 10"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 12 ; echo -n 11"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 13 ; echo -n 12"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 14 ; echo -n 13"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 15 ; echo -n 14"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 16 ; echo -n 15"))
+	require.Nil(t, r.Results(runner.NewTask("bash", "-c", "echo -n 5")))
+	fmt.Println(string(r.Results(runner.NewTask("bash", "-c", "sleep 3 ; echo -n 2")).Stdout))
+	fmt.Println("Cancelling")
+	r.Cancel()
+	fmt.Println("Runner completed")
+}
diff --git a/internal/arduino/builder/internal/runner/task.go b/internal/arduino/builder/internal/runner/task.go
index 05d5f63105a..dd44b2fa332 100644
--- a/internal/arduino/builder/internal/runner/task.go
+++ b/internal/arduino/builder/internal/runner/task.go
@@ -15,9 +15,46 @@
 
 package runner
 
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/arduino/go-paths-helper"
+)
+
+// Task is a command to be executed
+type Task struct {
+	Args []string `json:"args"`
+}
+
+// NewTask creates a new Task
+func NewTask(args ...string) *Task {
+	return &Task{Args: args}
+}
+
+func (t *Task) String() string {
+	return strings.Join(t.Args, " ")
+}
+
 // Result contains the output of a command execution
 type Result struct {
 	Args   []string
 	Stdout []byte
 	Stderr []byte
+	Error  error
+}
+
+// Run executes the command and returns the result
+func (t *Task) Run(ctx context.Context) *Result {
+	proc, err := paths.NewProcess(nil, t.Args...)
+	if err != nil {
+		return &Result{Args: t.Args, Error: err}
+	}
+	stdout, stderr, err := proc.RunAndCaptureOutput(ctx)
+
+	// Append arguments to stdout
+	stdout = append([]byte(fmt.Sprintln(t)), stdout...)
+
+	return &Result{Args: proc.GetArgs(), Stdout: stdout, Stderr: stderr, Error: err}
 }

From bfe6d8cd2a1200206ad60d65fa82915826e7e31d Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Wed, 5 Jun 2024 00:46:48 +0200
Subject: [PATCH 21/28] Simplify use of properties.SplitQuotedString

The new release of the library allow ignoring the returned error.

https://github.com/arduino/go-properties-orderedmap/pull/42
---
 commands/service_monitor.go                           |  6 +-----
 commands/service_upload.go                            |  5 +----
 internal/arduino/builder/builder.go                   | 11 ++++-------
 .../internal/preprocessor/arduino_preprocessor.go     | 10 +++-------
 .../arduino/builder/internal/preprocessor/ctags.go    | 10 +++-------
 internal/arduino/builder/internal/preprocessor/gcc.go |  5 +----
 internal/arduino/cores/packagemanager/loader.go       |  7 ++-----
 7 files changed, 15 insertions(+), 39 deletions(-)

diff --git a/commands/service_monitor.go b/commands/service_monitor.go
index 7c47e201ada..d384a05b885 100644
--- a/commands/service_monitor.go
+++ b/commands/service_monitor.go
@@ -27,7 +27,6 @@ import (
 	"github.com/arduino/arduino-cli/internal/arduino/cores"
 	"github.com/arduino/arduino-cli/internal/arduino/cores/packagemanager"
 	pluggableMonitor "github.com/arduino/arduino-cli/internal/arduino/monitor"
-	"github.com/arduino/arduino-cli/internal/i18n"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	"github.com/arduino/go-properties-orderedmap"
 	"github.com/djherbis/buffer"
@@ -264,10 +263,7 @@ func findMonitorAndSettingsForProtocolAndBoard(pme *packagemanager.Explorer, pro
 		} else if recipe, ok := boardPlatform.MonitorsDevRecipes[protocol]; ok {
 			// If we have a recipe we must resolve it
 			cmdLine := boardProperties.ExpandPropsInString(recipe)
-			cmdArgs, err := properties.SplitQuotedString(cmdLine, `"'`, false)
-			if err != nil {
-				return nil, nil, &cmderrors.InvalidArgumentError{Message: i18n.Tr("Invalid recipe in platform.txt"), Cause: err}
-			}
+			cmdArgs, _ := properties.SplitQuotedString(cmdLine, `"'`, false)
 			id := fmt.Sprintf("%s-%s", boardPlatform, protocol)
 			return pluggableMonitor.New(id, cmdArgs...), boardSettings, nil
 		}
diff --git a/commands/service_upload.go b/commands/service_upload.go
index 330decc3c5e..0365cdd794b 100644
--- a/commands/service_upload.go
+++ b/commands/service_upload.go
@@ -703,10 +703,7 @@ func runTool(recipeID string, props *properties.Map, outStream, errStream io.Wri
 		return errors.New(i18n.Tr("no upload port provided"))
 	}
 	cmdLine := props.ExpandPropsInString(recipe)
-	cmdArgs, err := properties.SplitQuotedString(cmdLine, `"'`, false)
-	if err != nil {
-		return errors.New(i18n.Tr("invalid recipe '%[1]s': %[2]s", recipe, err))
-	}
+	cmdArgs, _ := properties.SplitQuotedString(cmdLine, `"'`, false)
 
 	// Run Tool
 	logrus.WithField("phase", "upload").Tracef("Executing upload tool: %s", cmdLine)
diff --git a/internal/arduino/builder/builder.go b/internal/arduino/builder/builder.go
index d267b9ec415..37ddeae2332 100644
--- a/internal/arduino/builder/builder.go
+++ b/internal/arduino/builder/builder.go
@@ -493,10 +493,7 @@ func (b *Builder) prepareCommandForRecipe(buildProperties *properties.Map, recip
 		commandLine = properties.DeleteUnexpandedPropsFromString(commandLine)
 	}
 
-	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
-	if err != nil {
-		return nil, err
-	}
+	args, _ := properties.SplitQuotedString(commandLine, `"'`, false)
 
 	// if the overall commandline is too long for the platform
 	// try reducing the length by making the filenames relative
@@ -504,18 +501,18 @@ func (b *Builder) prepareCommandForRecipe(buildProperties *properties.Map, recip
 	var relativePath string
 	if len(commandLine) > 30000 {
 		relativePath = buildProperties.Get("build.path")
-		for i, arg := range parts {
+		for i, arg := range args {
 			if _, err := os.Stat(arg); os.IsNotExist(err) {
 				continue
 			}
 			rel, err := filepath.Rel(relativePath, arg)
 			if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) {
-				parts[i] = rel
+				args[i] = rel
 			}
 		}
 	}
 
-	command, err := paths.NewProcess(b.toolEnv, parts...)
+	command, err := paths.NewProcess(b.toolEnv, args...)
 	if err != nil {
 		return nil, err
 	}
diff --git a/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go b/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go
index e69adeae154..af8f6a5a2a4 100644
--- a/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go
+++ b/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go
@@ -66,18 +66,14 @@ func PreprocessSketchWithArduinoPreprocessor(
 	}
 
 	commandLine := arduinoPreprocessorProperties.ExpandPropsInString(pattern)
-	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
-	if err != nil {
-		return nil, err
-	}
-
-	command, err := paths.NewProcess(nil, parts...)
+	args, _ := properties.SplitQuotedString(commandLine, `"'`, false)
+	command, err := paths.NewProcess(nil, args...)
 	if err != nil {
 		return nil, err
 	}
 	if runtime.GOOS == "windows" {
 		// chdir in the uppermost directory to avoid UTF-8 bug in clang (https://github.com/arduino/arduino-preprocessor/issues/2)
-		command.SetDir(filepath.VolumeName(parts[0]) + "/")
+		command.SetDir(filepath.VolumeName(args[0]) + "/")
 	}
 
 	verboseOut.WriteString(commandLine)
diff --git a/internal/arduino/builder/internal/preprocessor/ctags.go b/internal/arduino/builder/internal/preprocessor/ctags.go
index a8c22498e98..0df774d83c0 100644
--- a/internal/arduino/builder/internal/preprocessor/ctags.go
+++ b/internal/arduino/builder/internal/preprocessor/ctags.go
@@ -194,19 +194,15 @@ func RunCTags(ctx context.Context, sourceFile *paths.Path, buildProperties *prop
 	}
 
 	commandLine := ctagsBuildProperties.ExpandPropsInString(pattern)
-	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
-	if err != nil {
-		return nil, nil, err
-	}
-	proc, err := paths.NewProcess(nil, parts...)
+	args, _ := properties.SplitQuotedString(commandLine, `"'`, false)
+	proc, err := paths.NewProcess(nil, args...)
 	if err != nil {
 		return nil, nil, err
 	}
 	stdout, stderr, err := proc.RunAndCaptureOutput(ctx)
 
 	// Append ctags arguments to stderr
-	args := fmt.Sprintln(strings.Join(parts, " "))
-	stderr = append([]byte(args), stderr...)
+	stderr = append([]byte(fmt.Sprintln(strings.Join(args, " "))), stderr...)
 	return stdout, stderr, err
 }
 
diff --git a/internal/arduino/builder/internal/preprocessor/gcc.go b/internal/arduino/builder/internal/preprocessor/gcc.go
index 96ea729a42d..51a971b1500 100644
--- a/internal/arduino/builder/internal/preprocessor/gcc.go
+++ b/internal/arduino/builder/internal/preprocessor/gcc.go
@@ -65,10 +65,7 @@ func GCC(
 
 	commandLine := gccBuildProperties.ExpandPropsInString(pattern)
 	commandLine = properties.DeleteUnexpandedPropsFromString(commandLine)
-	args, err := properties.SplitQuotedString(commandLine, `"'`, false)
-	if err != nil {
-		return nil, err
-	}
+	args, _ := properties.SplitQuotedString(commandLine, `"'`, false)
 
 	// Remove -MMD argument if present. Leaving it will make gcc try
 	// to create a /dev/null.d dependency file, which won't work.
diff --git a/internal/arduino/cores/packagemanager/loader.go b/internal/arduino/cores/packagemanager/loader.go
index 14e1d6df912..e6067cfb125 100644
--- a/internal/arduino/cores/packagemanager/loader.go
+++ b/internal/arduino/cores/packagemanager/loader.go
@@ -721,11 +721,8 @@ func (pme *Explorer) loadDiscoveries(release *cores.PlatformRelease) []error {
 		}
 
 		cmd := configuration.ExpandPropsInString(pattern)
-		if cmdArgs, err := properties.SplitQuotedString(cmd, `"'`, true); err != nil {
-			merr = append(merr, err)
-		} else {
-			pme.discoveryManager.Add(discoveryID, cmdArgs...)
-		}
+		cmdArgs, _ := properties.SplitQuotedString(cmd, `"'`, true)
+		pme.discoveryManager.Add(discoveryID, cmdArgs...)
 	}
 
 	return merr

From f81fdbc61c8a1eb1edd43ba8ad3884e70e42ff3f Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Wed, 5 Jun 2024 11:02:29 +0200
Subject: [PATCH 22/28] Use runner.Task in GCC preprocessor

It slightly simplifies code, but also provide the basis for the next
commits.
---
 .../builder/internal/detector/detector.go     | 29 ++++++++++++-------
 .../preprocessor/arduino_preprocessor.go      |  6 ++--
 .../builder/internal/preprocessor/ctags.go    |  4 +--
 .../builder/internal/preprocessor/gcc.go      | 23 ++-------------
 4 files changed, 25 insertions(+), 37 deletions(-)

diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index 8e36d8167d2..6f1057357e8 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -284,6 +284,19 @@ func (l *SketchLibrariesDetector) findIncludes(
 	return nil
 }
 
+func (l *SketchLibrariesDetector) gccPreprocessTask(sourceFile *sourceFile, buildProperties *properties.Map) *runner.Task {
+	// Libraries may require the "utility" directory to be added to the include
+	// search path, but only for the source code of the library, so we temporary
+	// copy the current search path list and add the library' utility directory
+	// if needed.
+	includeFolders := l.includeFolders
+	if extraInclude := sourceFile.ExtraIncludePath; extraInclude != nil {
+		includeFolders = append(includeFolders, extraInclude)
+	}
+
+	return preprocessor.GCC(sourceFile.SourcePath, paths.NullPath(), includeFolders, buildProperties)
+}
+
 func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 	ctx context.Context,
 	sourceFileQueue *uniqueSourceFileQueue,
@@ -315,15 +328,7 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 	for {
 		l.cache.Expect(&detectorCacheEntry{Compile: sourceFile})
 
-		// Libraries may require the "utility" directory to be added to the include
-		// search path, but only for the source code of the library, so we temporary
-		// copy the current search path list and add the library' utility directory
-		// if needed.
-		includeFolders := l.includeFolders
-		if extraInclude := sourceFile.ExtraIncludePath; extraInclude != nil {
-			includeFolders = append(includeFolders, extraInclude)
-		}
-
+		preprocTask := l.gccPreprocessTask(sourceFile, buildProperties)
 		var preprocErr error
 		var preprocResult *runner.Result
 
@@ -335,7 +340,8 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 			}
 			first = false
 		} else {
-			preprocResult, preprocErr = preprocessor.GCC(ctx, sourcePath, paths.NullPath(), includeFolders, buildProperties)
+			preprocResult = preprocTask.Run(ctx)
+			preprocErr = preprocResult.Error
 			if l.logger.Verbose() {
 				l.logger.WriteStdout(preprocResult.Stdout)
 			}
@@ -368,7 +374,8 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 
 			// If preprocess result came from cache, run the preprocessor to obtain the actual error to show
 			if preprocErr == nil || len(preprocResult.Stderr) == 0 {
-				preprocResult, preprocErr = preprocessor.GCC(ctx, sourcePath, paths.NullPath(), includeFolders, buildProperties)
+				preprocResult = preprocTask.Run(ctx)
+				preprocErr = preprocResult.Error
 				if l.logger.Verbose() {
 					l.logger.WriteStdout(preprocResult.Stdout)
 				}
diff --git a/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go b/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go
index af8f6a5a2a4..dd847c26eda 100644
--- a/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go
+++ b/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go
@@ -45,11 +45,11 @@ func PreprocessSketchWithArduinoPreprocessor(
 
 	sourceFile := buildPath.Join("sketch", sk.MainFile.Base()+".cpp")
 	targetFile := buildPath.Join("preproc", "sketch_merged.cpp")
-	gccResult, err := GCC(ctx, sourceFile, targetFile, includeFolders, buildProperties)
+	gccResult := GCC(sourceFile, targetFile, includeFolders, buildProperties).Run(ctx)
 	verboseOut.Write(gccResult.Stdout)
 	verboseOut.Write(gccResult.Stderr)
-	if err != nil {
-		return nil, err
+	if gccResult.Error != nil {
+		return nil, gccResult.Error
 	}
 
 	arduinoPreprocessorProperties := properties.NewMap()
diff --git a/internal/arduino/builder/internal/preprocessor/ctags.go b/internal/arduino/builder/internal/preprocessor/ctags.go
index 0df774d83c0..6393cd86e3d 100644
--- a/internal/arduino/builder/internal/preprocessor/ctags.go
+++ b/internal/arduino/builder/internal/preprocessor/ctags.go
@@ -57,10 +57,10 @@ func PreprocessSketchWithCtags(
 
 	// Run GCC preprocessor
 	sourceFile := buildPath.Join("sketch", sketch.MainFile.Base()+".cpp")
-	result, err := GCC(ctx, sourceFile, ctagsTarget, includes, buildProperties)
+	result := GCC(sourceFile, ctagsTarget, includes, buildProperties).Run(ctx)
 	stdout.Write(result.Stdout)
 	stderr.Write(result.Stderr)
-	if err != nil {
+	if err := result.Error; err != nil {
 		if !onlyUpdateCompilationDatabase {
 			return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 		}
diff --git a/internal/arduino/builder/internal/preprocessor/gcc.go b/internal/arduino/builder/internal/preprocessor/gcc.go
index 51a971b1500..59500c3522d 100644
--- a/internal/arduino/builder/internal/preprocessor/gcc.go
+++ b/internal/arduino/builder/internal/preprocessor/gcc.go
@@ -16,15 +16,11 @@
 package preprocessor
 
 import (
-	"context"
-	"errors"
-	"fmt"
 	"strings"
 
 	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/cpp"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
-	"github.com/arduino/arduino-cli/internal/i18n"
 	"github.com/arduino/go-paths-helper"
 	"github.com/arduino/go-properties-orderedmap"
 )
@@ -32,10 +28,9 @@ import (
 // GCC performs a run of the gcc preprocess (macro/includes expansion). The function outputs the result
 // to targetFilePath. Returns the stdout/stderr of gcc if any.
 func GCC(
-	ctx context.Context,
 	sourceFilePath, targetFilePath *paths.Path,
 	includes paths.PathList, buildProperties *properties.Map,
-) (*runner.Result, error) {
+) *runner.Task {
 	gccBuildProperties := properties.NewMap()
 	gccBuildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC")
 	gccBuildProperties.Merge(buildProperties)
@@ -59,10 +54,6 @@ func GCC(
 	}
 
 	pattern := gccBuildProperties.Get(gccPreprocRecipeProperty)
-	if pattern == "" {
-		return nil, errors.New(i18n.Tr("%s pattern is missing", gccPreprocRecipeProperty))
-	}
-
 	commandLine := gccBuildProperties.ExpandPropsInString(pattern)
 	commandLine = properties.DeleteUnexpandedPropsFromString(commandLine)
 	args, _ := properties.SplitQuotedString(commandLine, `"'`, false)
@@ -70,15 +61,5 @@ func GCC(
 	// Remove -MMD argument if present. Leaving it will make gcc try
 	// to create a /dev/null.d dependency file, which won't work.
 	args = f.Filter(args, f.NotEquals("-MMD"))
-
-	proc, err := paths.NewProcess(nil, args...)
-	if err != nil {
-		return nil, err
-	}
-	stdout, stderr, err := proc.RunAndCaptureOutput(ctx)
-
-	// Append gcc arguments to stdout
-	stdout = append([]byte(fmt.Sprintln(strings.Join(args, " "))), stdout...)
-
-	return &runner.Result{Args: proc.GetArgs(), Stdout: stdout, Stderr: stderr}, err
+	return runner.NewTask(args...)
 }

From 8f3155fea4757a6a853ef6cd34e0da37fd1f4d79 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Wed, 5 Jun 2024 11:12:01 +0200
Subject: [PATCH 23/28] Parallelize library discovery phase in compile

---
 .../builder/internal/detector/cache.go        | 20 ++++++--
 .../builder/internal/detector/detector.go     | 49 +++++++++++++++++--
 2 files changed, 60 insertions(+), 9 deletions(-)

diff --git a/internal/arduino/builder/internal/detector/cache.go b/internal/arduino/builder/internal/detector/cache.go
index d2ca6b526f0..62b3d355678 100644
--- a/internal/arduino/builder/internal/detector/cache.go
+++ b/internal/arduino/builder/internal/detector/cache.go
@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
 	"github.com/arduino/go-paths-helper"
 )
 
@@ -28,17 +29,18 @@ type detectorCache struct {
 }
 
 type detectorCacheEntry struct {
-	AddedIncludePath *paths.Path `json:"added_include_path,omitempty"`
-	Compile          *sourceFile `json:"compile,omitempty"`
-	MissingIncludeH  *string     `json:"missing_include_h,omitempty"`
+	AddedIncludePath *paths.Path  `json:"added_include_path,omitempty"`
+	Compile          *sourceFile  `json:"compile,omitempty"`
+	CompileTask      *runner.Task `json:"compile_task,omitempty"`
+	MissingIncludeH  *string      `json:"missing_include_h,omitempty"`
 }
 
 func (e *detectorCacheEntry) String() string {
 	if e.AddedIncludePath != nil {
 		return "Added include path: " + e.AddedIncludePath.String()
 	}
-	if e.Compile != nil {
-		return "Compiling: " + e.Compile.String()
+	if e.Compile != nil && e.CompileTask != nil {
+		return "Compiling: " + e.Compile.String() + " / " + e.CompileTask.String()
 	}
 	if e.MissingIncludeH != nil {
 		if *e.MissingIncludeH == "" {
@@ -109,6 +111,14 @@ func (c *detectorCache) Peek() *detectorCacheEntry {
 	return nil
 }
 
+// EntriesAhead returns the entries that are ahead of the current cache position.
+func (c *detectorCache) EntriesAhead() []*detectorCacheEntry {
+	if c.curr < len(c.entries) {
+		return c.entries[c.curr:]
+	}
+	return nil
+}
+
 // Save writes the current cache to the given file.
 func (c *detectorCache) Save(cacheFile *paths.Path) error {
 	// Cut off the cache if it is not fully consumed
diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index 6f1057357e8..c2b3b70315f 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -58,6 +58,7 @@ type SketchLibrariesDetector struct {
 	includeFolders                paths.PathList
 	logger                        *logger.BuilderLogger
 	diagnosticStore               *diagnostics.Store
+	preRunner                     *runner.Runner
 }
 
 // NewSketchLibrariesDetector todo
@@ -236,6 +237,18 @@ func (l *SketchLibrariesDetector) findIncludes(
 		l.logger.Warn(i18n.Tr("Failed to load library discovery cache: %[1]s", err))
 	}
 
+	// Pre-run cache entries
+	l.preRunner = runner.New(ctx)
+	for _, entry := range l.cache.EntriesAhead() {
+		if entry.Compile != nil && entry.CompileTask != nil {
+			upToDate, _ := entry.Compile.ObjFileIsUpToDate()
+			if !upToDate {
+				l.preRunner.Enqueue(entry.CompileTask)
+			}
+		}
+	}
+	defer l.preRunner.Cancel()
+
 	l.addIncludeFolder(buildCorePath)
 	if buildVariantPath != nil {
 		l.addIncludeFolder(buildVariantPath)
@@ -263,6 +276,15 @@ func (l *SketchLibrariesDetector) findIncludes(
 				cachePath.Remove()
 				return err
 			}
+
+			// Create a new pre-runner if the previous one was cancelled
+			if l.preRunner == nil {
+				l.preRunner = runner.New(ctx)
+				// Push in the remainder of the queue
+				for _, sourceFile := range *sourceFileQueue {
+					l.preRunner.Enqueue(l.gccPreprocessTask(sourceFile, buildProperties))
+				}
+			}
 		}
 
 		// Finalize the cache
@@ -326,9 +348,9 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 
 	first := true
 	for {
-		l.cache.Expect(&detectorCacheEntry{Compile: sourceFile})
-
 		preprocTask := l.gccPreprocessTask(sourceFile, buildProperties)
+		l.cache.Expect(&detectorCacheEntry{Compile: sourceFile, CompileTask: preprocTask})
+
 		var preprocErr error
 		var preprocResult *runner.Result
 
@@ -340,8 +362,27 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 			}
 			first = false
 		} else {
-			preprocResult = preprocTask.Run(ctx)
-			preprocErr = preprocResult.Error
+			if l.preRunner != nil {
+				if r := l.preRunner.Results(preprocTask); r != nil {
+					preprocResult = r
+					preprocErr = preprocResult.Error
+				}
+			}
+			if preprocResult == nil {
+				// The pre-runner missed this task, maybe the cache is outdated
+				// or maybe the source code changed.
+
+				// Stop the pre-runner
+				if l.preRunner != nil {
+					preRunner := l.preRunner
+					l.preRunner = nil
+					go preRunner.Cancel()
+				}
+
+				// Run the actual preprocessor
+				preprocResult = preprocTask.Run(ctx)
+				preprocErr = preprocResult.Error
+			}
 			if l.logger.Verbose() {
 				l.logger.WriteStdout(preprocResult.Stdout)
 			}

From 8833b280eb6158cba7f6d4ed83844dd967e2e9a2 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Tue, 11 Jun 2024 11:16:55 +0200
Subject: [PATCH 24/28] The number of jobs in library detection now follows
 --jobs flag

---
 internal/arduino/builder/builder.go                     | 1 +
 internal/arduino/builder/internal/detector/detector.go  | 8 +++++---
 internal/arduino/builder/internal/runner/runner.go      | 9 +++++++--
 internal/arduino/builder/internal/runner/runner_test.go | 2 +-
 4 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/internal/arduino/builder/builder.go b/internal/arduino/builder/builder.go
index 37ddeae2332..4116b9a4739 100644
--- a/internal/arduino/builder/builder.go
+++ b/internal/arduino/builder/builder.go
@@ -321,6 +321,7 @@ func (b *Builder) preprocess() error {
 		b.librariesBuildPath,
 		b.buildProperties,
 		b.targetPlatform.Platform.Architecture,
+		b.jobs,
 	)
 	if err != nil {
 		return err
diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index c2b3b70315f..17a70ebf359 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -191,8 +191,9 @@ func (l *SketchLibrariesDetector) FindIncludes(
 	librariesBuildPath *paths.Path,
 	buildProperties *properties.Map,
 	platformArch string,
+	jobs int,
 ) error {
-	err := l.findIncludes(ctx, buildPath, buildCorePath, buildVariantPath, sketchBuildPath, sketch, librariesBuildPath, buildProperties, platformArch)
+	err := l.findIncludes(ctx, buildPath, buildCorePath, buildVariantPath, sketchBuildPath, sketch, librariesBuildPath, buildProperties, platformArch, jobs)
 	if err != nil && l.onlyUpdateCompilationDatabase {
 		l.logger.Info(
 			fmt.Sprintf(
@@ -216,6 +217,7 @@ func (l *SketchLibrariesDetector) findIncludes(
 	librariesBuildPath *paths.Path,
 	buildProperties *properties.Map,
 	platformArch string,
+	jobs int,
 ) error {
 	librariesResolutionCache := buildPath.Join("libraries.cache")
 	if l.useCachedLibrariesResolution && librariesResolutionCache.Exist() {
@@ -238,7 +240,7 @@ func (l *SketchLibrariesDetector) findIncludes(
 	}
 
 	// Pre-run cache entries
-	l.preRunner = runner.New(ctx)
+	l.preRunner = runner.New(ctx, jobs)
 	for _, entry := range l.cache.EntriesAhead() {
 		if entry.Compile != nil && entry.CompileTask != nil {
 			upToDate, _ := entry.Compile.ObjFileIsUpToDate()
@@ -279,7 +281,7 @@ func (l *SketchLibrariesDetector) findIncludes(
 
 			// Create a new pre-runner if the previous one was cancelled
 			if l.preRunner == nil {
-				l.preRunner = runner.New(ctx)
+				l.preRunner = runner.New(ctx, jobs)
 				// Push in the remainder of the queue
 				for _, sourceFile := range *sourceFileQueue {
 					l.preRunner.Enqueue(l.gccPreprocessTask(sourceFile, buildProperties))
diff --git a/internal/arduino/builder/internal/runner/runner.go b/internal/arduino/builder/internal/runner/runner.go
index d34d3df3b28..89565bdbc4c 100644
--- a/internal/arduino/builder/internal/runner/runner.go
+++ b/internal/arduino/builder/internal/runner/runner.go
@@ -41,7 +41,9 @@ func (cmd *enqueuedCommand) String() string {
 	return cmd.task.String()
 }
 
-func New(inCtx context.Context) *Runner {
+// New creates a new Runner with the given number of workers.
+// If workers is 0, the number of workers will be the number of available CPUs.
+func New(inCtx context.Context, workers int) *Runner {
 	ctx, cancel := context.WithCancel(inCtx)
 	queue := make(chan *enqueuedCommand, 1000)
 	r := &Runner{
@@ -52,7 +54,10 @@ func New(inCtx context.Context) *Runner {
 	}
 
 	// Spawn workers
-	for i := 0; i < runtime.NumCPU(); i++ {
+	if workers == 0 {
+		workers = runtime.NumCPU()
+	}
+	for i := 0; i < workers; i++ {
 		r.wg.Add(1)
 		go func() {
 			worker(ctx, queue)
diff --git a/internal/arduino/builder/internal/runner/runner_test.go b/internal/arduino/builder/internal/runner/runner_test.go
index 90a72d4654a..ed499299912 100644
--- a/internal/arduino/builder/internal/runner/runner_test.go
+++ b/internal/arduino/builder/internal/runner/runner_test.go
@@ -28,7 +28,7 @@ import (
 func TestRunMultipleTask(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	r := runner.New(ctx)
+	r := runner.New(ctx, 0)
 	r.Enqueue(runner.NewTask("bash", "-c", "sleep 1 ; echo -n 0"))
 	r.Enqueue(runner.NewTask("bash", "-c", "sleep 2 ; echo -n 1"))
 	r.Enqueue(runner.NewTask("bash", "-c", "sleep 3 ; echo -n 2"))

From bc97f3df8ac4859c84185c0ece47d30a6f6036f6 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Thu, 13 Jun 2024 15:33:37 +0200
Subject: [PATCH 25/28] Reordered properties construction for clarity

---
 internal/arduino/builder/compilation.go | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/internal/arduino/builder/compilation.go b/internal/arduino/builder/compilation.go
index d3a1459da25..8d8e3934cad 100644
--- a/internal/arduino/builder/compilation.go
+++ b/internal/arduino/builder/compilation.go
@@ -118,20 +118,13 @@ func (b *Builder) compileFileWithRecipe(
 	includes []string,
 	recipe string,
 ) (*paths.Path, error) {
-	properties := b.buildProperties.Clone()
-	properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+b.logger.WarningsLevel()))
-	properties.Set("includes", strings.Join(includes, " "))
-	properties.SetPath("source_file", source)
 	relativeSource, err := sourcePath.RelTo(source)
 	if err != nil {
 		return nil, err
 	}
 	depsFile := buildPath.Join(relativeSource.String() + ".d")
 	objectFile := buildPath.Join(relativeSource.String() + ".o")
-
-	properties.SetPath("object_file", objectFile)
-	err = objectFile.Parent().MkdirAll()
-	if err != nil {
+	if err := objectFile.Parent().MkdirAll(); err != nil {
 		return nil, err
 	}
 
@@ -140,6 +133,11 @@ func (b *Builder) compileFileWithRecipe(
 		return nil, err
 	}
 
+	properties := b.buildProperties.Clone()
+	properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+b.logger.WarningsLevel()))
+	properties.Set("includes", strings.Join(includes, " "))
+	properties.SetPath("source_file", source)
+	properties.SetPath("object_file", objectFile)
 	command, err := b.prepareCommandForRecipe(properties, recipe, false)
 	if err != nil {
 		return nil, err

From 60e6c8f47e0b914641172c4dc05a3d36834e5ebe Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Thu, 13 Jun 2024 15:39:32 +0200
Subject: [PATCH 26/28] Reordered compileFileWithRecipe for clarity

---
 internal/arduino/builder/compilation.go | 71 +++++++++++++------------
 1 file changed, 37 insertions(+), 34 deletions(-)

diff --git a/internal/arduino/builder/compilation.go b/internal/arduino/builder/compilation.go
index 8d8e3934cad..f3d85908ea0 100644
--- a/internal/arduino/builder/compilation.go
+++ b/internal/arduino/builder/compilation.go
@@ -110,7 +110,6 @@ func (b *Builder) compileFiles(
 	return objectFiles, nil
 }
 
-// CompileFilesRecursive fixdoc
 func (b *Builder) compileFileWithRecipe(
 	sourcePath *paths.Path,
 	source *paths.Path,
@@ -128,11 +127,6 @@ func (b *Builder) compileFileWithRecipe(
 		return nil, err
 	}
 
-	objIsUpToDate, err := utils.ObjFileIsUpToDate(source, objectFile, depsFile)
-	if err != nil {
-		return nil, err
-	}
-
 	properties := b.buildProperties.Clone()
 	properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+b.logger.WarningsLevel()))
 	properties.Set("includes", strings.Join(includes, " "))
@@ -145,41 +139,50 @@ func (b *Builder) compileFileWithRecipe(
 	if b.compilationDatabase != nil {
 		b.compilationDatabase.Add(source, command)
 	}
-	if !objIsUpToDate && !b.onlyUpdateCompilationDatabase {
-		commandStdout, commandStderr := &bytes.Buffer{}, &bytes.Buffer{}
-		command.RedirectStdoutTo(commandStdout)
-		command.RedirectStderrTo(commandStderr)
 
+	objIsUpToDate, err := utils.ObjFileIsUpToDate(source, objectFile, depsFile)
+	if err != nil {
+		return nil, err
+	}
+	if objIsUpToDate {
 		if b.logger.Verbose() {
-			b.logger.Info(utils.PrintableCommand(command.GetArgs()))
-		}
-		// Since this compile could be multithreaded, we first capture the command output
-		if err := command.Start(); err != nil {
-			return nil, err
+			b.logger.Info(i18n.Tr("Using previously compiled file: %[1]s", objectFile))
 		}
-		err := command.Wait()
-		// and transfer all at once at the end...
+		return objectFile, nil
+	}
+	if b.onlyUpdateCompilationDatabase {
 		if b.logger.Verbose() {
-			b.logger.WriteStdout(commandStdout.Bytes())
+			b.logger.Info(i18n.Tr("Skipping compile of: %[1]s", objectFile))
 		}
-		b.logger.WriteStderr(commandStderr.Bytes())
+		return objectFile, nil
+	}
 
-		// Parse the output of the compiler to gather errors and warnings...
-		if b.diagnosticStore != nil {
-			b.diagnosticStore.Parse(command.GetArgs(), commandStdout.Bytes())
-			b.diagnosticStore.Parse(command.GetArgs(), commandStderr.Bytes())
-		}
+	commandStdout, commandStderr := &bytes.Buffer{}, &bytes.Buffer{}
+	command.RedirectStdoutTo(commandStdout)
+	command.RedirectStderrTo(commandStderr)
+	if b.logger.Verbose() {
+		b.logger.Info(utils.PrintableCommand(command.GetArgs()))
+	}
+	// Since this compile could be multithreaded, we first capture the command output
+	if err := command.Start(); err != nil {
+		return nil, err
+	}
+	err = command.Wait()
+	// and transfer all at once at the end...
+	if b.logger.Verbose() {
+		b.logger.WriteStdout(commandStdout.Bytes())
+	}
+	b.logger.WriteStderr(commandStderr.Bytes())
 
-		// ...and then return the error
-		if err != nil {
-			return nil, err
-		}
-	} else if b.logger.Verbose() {
-		if objIsUpToDate {
-			b.logger.Info(i18n.Tr("Using previously compiled file: %[1]s", objectFile))
-		} else {
-			b.logger.Info(i18n.Tr("Skipping compile of: %[1]s", objectFile))
-		}
+	// Parse the output of the compiler to gather errors and warnings...
+	if b.diagnosticStore != nil {
+		b.diagnosticStore.Parse(command.GetArgs(), commandStdout.Bytes())
+		b.diagnosticStore.Parse(command.GetArgs(), commandStderr.Bytes())
+	}
+
+	// ...and then return the error
+	if err != nil {
+		return nil, err
 	}
 
 	return objectFile, nil

From 8237ac6632ecceec3f65de247360e1cffd4b2c55 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Wed, 12 Jun 2024 11:08:09 +0200
Subject: [PATCH 27/28] Added integration test

---
 .../compile_4/lib_discovery_caching_test.go   | 110 ++++++++++++++++++
 .../SketchA/SketchA.ino                       |   3 +
 .../libraries/LibA/LibA.h                     |   6 +
 .../libraries/LibA/file1.cpp                  |   5 +
 .../libraries/LibB/LibB.h                     |   0
 .../libraries/LibC/LibB.h                     |   2 +
 .../libraries/LibC/LibC.h                     |   0
 7 files changed, 126 insertions(+)
 create mode 100644 internal/integrationtest/compile_4/lib_discovery_caching_test.go
 create mode 100644 internal/integrationtest/compile_4/testdata/libraries_discovery_caching/SketchA/SketchA.ino
 create mode 100644 internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/LibA.h
 create mode 100644 internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/file1.cpp
 create mode 100644 internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibB/LibB.h
 create mode 100644 internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibC/LibB.h
 create mode 100644 internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibC/LibC.h

diff --git a/internal/integrationtest/compile_4/lib_discovery_caching_test.go b/internal/integrationtest/compile_4/lib_discovery_caching_test.go
new file mode 100644
index 00000000000..9d8bd4b5277
--- /dev/null
+++ b/internal/integrationtest/compile_4/lib_discovery_caching_test.go
@@ -0,0 +1,110 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package compile_test
+
+import (
+	"testing"
+
+	"github.com/arduino/arduino-cli/internal/integrationtest"
+	"github.com/arduino/go-paths-helper"
+	"github.com/stretchr/testify/require"
+	"go.bug.st/testifyjson/requirejson"
+)
+
+func TestLibDiscoveryCache(t *testing.T) {
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	t.Cleanup(env.CleanUp)
+
+	// Install Arduino AVR Boards
+	_, _, err := cli.Run("core", "install", "arduino:avr@1.8.6")
+	require.NoError(t, err)
+
+	// Copy the testdata sketchbook
+	testdata, err := paths.New("testdata", "libraries_discovery_caching").Abs()
+	require.NoError(t, err)
+	sketchbook := cli.SketchbookDir()
+	require.NoError(t, sketchbook.RemoveAll())
+	require.NoError(t, testdata.CopyDirTo(cli.SketchbookDir()))
+
+	buildpath, err := paths.MkTempDir("", "tmpbuildpath")
+	require.NoError(t, err)
+	t.Cleanup(func() { buildpath.RemoveAll() })
+
+	{
+		sketchA := sketchbook.Join("SketchA")
+		{
+			outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String())
+			require.NoError(t, err)
+			j := requirejson.Parse(t, outjson)
+			j.MustContain(`{"builder_result":{
+		"used_libraries": [
+			{ "name": "LibA" },
+        	{ "name": "LibB" }
+    	],
+	}}`)
+		}
+
+		// Update SketchA
+		require.NoError(t, sketchA.Join("SketchA.ino").WriteFile([]byte(`
+#include <LibC.h>
+#include <LibA.h>
+void setup() {}
+void loop() {libAFunction();}
+`)))
+
+		{
+			// This compile should FAIL!
+			outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String())
+			require.Error(t, err)
+			j := requirejson.Parse(t, outjson)
+			j.MustContain(`{
+"builder_result":{
+	"used_libraries": [
+		{ "name": "LibC" },
+		{ "name": "LibA" }
+	],
+    "diagnostics": [
+		{
+			"severity": "ERROR",
+			"message": "'libAFunction' was not declared in this scope\n void loop() {libAFunction();}\n              ^~~~~~~~~~~~"
+		}
+	]
+}}`)
+			j.Query(".compiler_out").MustContain(`"The list of included libraries has been changed... rebuilding all libraries."`)
+		}
+
+		{
+			// This compile should FAIL!
+			outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String())
+			require.Error(t, err)
+			j := requirejson.Parse(t, outjson)
+			j.MustContain(`{
+"builder_result":{
+	"used_libraries": [
+		{ "name": "LibC" },
+		{ "name": "LibA" }
+	],
+    "diagnostics": [
+		{
+			"severity": "ERROR",
+			"message": "'libAFunction' was not declared in this scope\n void loop() {libAFunction();}\n              ^~~~~~~~~~~~"
+		}
+	]
+}}`)
+			j.Query(".compiler_out").MustNotContain(`"The list of included libraries has changed... rebuilding all libraries."`)
+		}
+	}
+}
diff --git a/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/SketchA/SketchA.ino b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/SketchA/SketchA.ino
new file mode 100644
index 00000000000..b95755d1f82
--- /dev/null
+++ b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/SketchA/SketchA.ino
@@ -0,0 +1,3 @@
+#include <LibA.h>
+void setup() {}
+void loop() {libAFunction();}
diff --git a/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/LibA.h b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/LibA.h
new file mode 100644
index 00000000000..5cc5459e363
--- /dev/null
+++ b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/LibA.h
@@ -0,0 +1,6 @@
+
+#include <LibB.h>
+
+#ifndef CHECK
+void libAFunction();
+#endif
diff --git a/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/file1.cpp b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/file1.cpp
new file mode 100644
index 00000000000..08474b5de80
--- /dev/null
+++ b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/file1.cpp
@@ -0,0 +1,5 @@
+#include <LibB.h>
+
+#ifndef CHECK
+void libAFunction() {}
+#endif
diff --git a/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibB/LibB.h b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibB/LibB.h
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibC/LibB.h b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibC/LibB.h
new file mode 100644
index 00000000000..4dbb6c43193
--- /dev/null
+++ b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibC/LibB.h
@@ -0,0 +1,2 @@
+
+#define CHECK
diff --git a/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibC/LibC.h b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibC/LibC.h
new file mode 100644
index 00000000000..e69de29bb2d

From 87765a41146eb023fcb0151234b91d91695db5ad Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Thu, 13 Jun 2024 22:49:54 +0200
Subject: [PATCH 28/28] fix: libraries are recompiled if the list of include
 paths changes

---
 internal/arduino/builder/builder.go           |  8 ++++++
 .../builder/internal/detector/detector.go     | 28 ++++++++++++++-----
 2 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/internal/arduino/builder/builder.go b/internal/arduino/builder/builder.go
index 4116b9a4739..b7e415b0865 100644
--- a/internal/arduino/builder/builder.go
+++ b/internal/arduino/builder/builder.go
@@ -326,6 +326,14 @@ func (b *Builder) preprocess() error {
 	if err != nil {
 		return err
 	}
+	if b.libsDetector.IncludeFoldersChanged() && b.librariesBuildPath.Exist() {
+		if b.logger.Verbose() {
+			b.logger.Info(i18n.Tr("The list of included libraries has been changed... rebuilding all libraries."))
+		}
+		if err := b.librariesBuildPath.RemoveAll(); err != nil {
+			return err
+		}
+	}
 	b.Progress.CompleteStep()
 
 	b.warnAboutArchIncompatibleLibraries(b.libsDetector.ImportedLibraries())
diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go
index 17a70ebf359..847c67e9cd2 100644
--- a/internal/arduino/builder/internal/detector/detector.go
+++ b/internal/arduino/builder/internal/detector/detector.go
@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"os/exec"
 	"regexp"
+	"slices"
 	"strings"
 	"time"
 
@@ -59,6 +60,7 @@ type SketchLibrariesDetector struct {
 	logger                        *logger.BuilderLogger
 	diagnosticStore               *diagnostics.Store
 	preRunner                     *runner.Runner
+	detectedChangeInLibraries     bool
 }
 
 // NewSketchLibrariesDetector todo
@@ -174,6 +176,12 @@ func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList {
 	return l.includeFolders
 }
 
+// IncludeFoldersChanged returns true if the include folders list changed
+// from the previous compile.
+func (l *SketchLibrariesDetector) IncludeFoldersChanged() bool {
+	return l.detectedChangeInLibraries
+}
+
 // addIncludeFolder add the given folder to the include path.
 func (l *SketchLibrariesDetector) addIncludeFolder(folder *paths.Path) {
 	l.includeFolders = append(l.includeFolders, folder)
@@ -219,17 +227,21 @@ func (l *SketchLibrariesDetector) findIncludes(
 	platformArch string,
 	jobs int,
 ) error {
-	librariesResolutionCache := buildPath.Join("libraries.cache")
-	if l.useCachedLibrariesResolution && librariesResolutionCache.Exist() {
-		d, err := librariesResolutionCache.ReadFile()
+	librariesResolutionCachePath := buildPath.Join("libraries.cache")
+	var cachedIncludeFolders paths.PathList
+	if librariesResolutionCachePath.Exist() {
+		d, err := librariesResolutionCachePath.ReadFile()
 		if err != nil {
 			return err
 		}
-		if err := json.Unmarshal(d, &l.includeFolders); err != nil {
+		if err := json.Unmarshal(d, &cachedIncludeFolders); err != nil {
 			return err
 		}
+	}
+	if l.useCachedLibrariesResolution && librariesResolutionCachePath.Exist() {
+		l.includeFolders = cachedIncludeFolders
 		if l.logger.Verbose() {
-			l.logger.Info("Using cached library discovery: " + librariesResolutionCache.String())
+			l.logger.Info("Using cached library discovery: " + librariesResolutionCachePath.String())
 		}
 		return nil
 	}
@@ -301,10 +313,12 @@ func (l *SketchLibrariesDetector) findIncludes(
 
 	if d, err := json.Marshal(l.includeFolders); err != nil {
 		return err
-	} else if err := librariesResolutionCache.WriteFile(d); err != nil {
+	} else if err := librariesResolutionCachePath.WriteFile(d); err != nil {
 		return err
 	}
-
+	l.detectedChangeInLibraries = !slices.Equal(
+		cachedIncludeFolders.AsStrings(),
+		l.includeFolders.AsStrings())
 	return nil
 }