diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index 6ddfbab48ff..32aedb139e9 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -96,7 +96,11 @@ import ( "encoding/json" "fmt" "os/exec" + "runtime" "time" + "sync" + "strings" + "sync/atomic" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/arduino/libraries" @@ -107,6 +111,9 @@ import ( "github.com/pkg/errors" ) +var libMux sync.Mutex +var fileMux sync.Mutex + type ContainerFindIncludes struct{} func (s *ContainerFindIncludes) Run(ctx *types.Context) error { @@ -125,9 +132,11 @@ func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error { cachePath := ctx.BuildPath.Join("includes.cache") cache := readCache(cachePath) - appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.core.path")) + // The entry with no_resolve prefix will be ignored in the preload phase + // in case they are already added into context at here + appendIncludeFolder(ctx, cache, nil, "no_resolve_1", ctx.BuildProperties.GetPath("build.core.path")) if ctx.BuildProperties.Get("build.variant.path") != "" { - appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.variant.path")) + appendIncludeFolder(ctx, cache, nil, "no_resolve_2", ctx.BuildProperties.GetPath("build.variant.path")) } sketch := ctx.Sketch @@ -144,16 +153,69 @@ func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error { queueSourceFilesFromFolder(ctx, sourceFilePaths, sketch, srcSubfolderPath, true /* recurse */) } - for !sourceFilePaths.Empty() { - err := findIncludesUntilDone(ctx, cache, sourceFilePaths.Pop()) + preloadCachedFolder(ctx, cache) + + // The first source file is the main .ino.cpp + // handle it first to setup environment for other files + findIncludesUntilDone(ctx, cache, sourceFilePaths.Pop()) + + var errorsList []error + var errorsMux sync.Mutex + + queue := make(chan types.SourceFile) + job := func(source types.SourceFile) { + // Find all includes + err := findIncludesUntilDone(ctx, cache, source) if err != nil { - cachePath.Remove() - return errors.WithStack(err) + errorsMux.Lock() + errorsList = append(errorsList, err) + errorsMux.Unlock() + } + } + + // Spawn jobs runners to make the scan faster + var wg sync.WaitGroup + jobs := ctx.Jobs + if jobs == 0 { + jobs = runtime.NumCPU() + } + var unhandled int32 = 0 + for i := 0; i < jobs; i++ { + wg.Add(1) + go func() { + for source := range queue { + job(source) + atomic.AddInt32(&unhandled, -1) + } + wg.Done() + }() + } + + // Loop until all files are handled + for (!sourceFilePaths.Empty() || unhandled != 0 ) { + errorsMux.Lock() + gotError := len(errorsList) > 0 + errorsMux.Unlock() + if gotError { + break } + if(!sourceFilePaths.Empty()){ + fileMux.Lock() + queue <- sourceFilePaths.Pop() + fileMux.Unlock() + atomic.AddInt32(&unhandled, 1) + } + } + close(queue) + wg.Wait() + + if len(errorsList) > 0 { + // output the first error + cachePath.Remove() + return errors.WithStack(errorsList[0]) } // Finalize the cache - cache.ExpectEnd() err = writeCache(cache, cachePath) if err != nil { return errors.WithStack(err) @@ -173,8 +235,19 @@ func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error { // and should be the empty string for the default include folders, like // the core or variant. func appendIncludeFolder(ctx *types.Context, cache *includeCache, sourceFilePath *paths.Path, include string, folder *paths.Path) { + libMux.Lock() + ctx.IncludeFolders = append(ctx.IncludeFolders, folder) + libMux.Unlock() + entry := &includeCacheEntry{Sourcefile: sourceFilePath, Include: include, Includepath: folder} + cache.entries.Store(include, entry) + cache.valid = false +} + +// Append the given folder to the include path without touch the cache, because it is already in cache +func appendIncludeFolderWoCache(ctx *types.Context, include string, folder *paths.Path) { + libMux.Lock() ctx.IncludeFolders = append(ctx.IncludeFolders, folder) - cache.ExpectEntry(sourceFilePath, include, folder) + libMux.Unlock() } func runCommand(ctx *types.Context, command types.Command) error { @@ -204,56 +277,7 @@ func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool { type includeCache struct { // Are the cache contents valid so far? valid bool - // Index into entries of the next entry to be processed. Unused - // when the cache is invalid. - next int - entries []*includeCacheEntry -} - -// Return the next cache entry. Should only be called when the cache is -// valid and a next entry is available (the latter can be checked with -// ExpectFile). Does not advance the cache. -func (cache *includeCache) Next() *includeCacheEntry { - return cache.entries[cache.next] -} - -// Check that the next cache entry is about the given file. If it is -// not, or no entry is available, the cache is invalidated. Does not -// advance the cache. -func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { - if cache.valid && (cache.next >= len(cache.entries) || !cache.Next().Sourcefile.EqualsTo(sourcefile)) { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } -} - -// Check that the next entry matches the given values. If so, advance -// the cache. If not, the cache is invalidated. If the cache is -// invalidated, or was already invalid, an entry with the given values -// is appended. -func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, librarypath *paths.Path) { - entry := &includeCacheEntry{Sourcefile: sourcefile, Include: include, Includepath: librarypath} - if cache.valid { - if cache.next < len(cache.entries) && cache.Next().Equals(entry) { - cache.next++ - } else { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } - } - - if !cache.valid { - cache.entries = append(cache.entries, entry) - } -} - -// Check that the cache is completely consumed. If not, the cache is -// invalidated. -func (cache *includeCache) ExpectEnd() { - if cache.valid && cache.next < len(cache.entries) { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } + entries sync.Map } // Read the cache from the given file @@ -264,10 +288,18 @@ func readCache(path *paths.Path) *includeCache { return &includeCache{} } result := &includeCache{} - err = json.Unmarshal(bytes, &result.entries) + var entries []*includeCacheEntry + err = json.Unmarshal(bytes, &entries) if err != nil { // Return an empty, invalid cache return &includeCache{} + } else { + for _, entry := range entries { + if entry.Include != "" || entry.Includepath != nil { + // Put the entry into cache + result.entries.Store(entry.Include, entry) + } + } } result.valid = true return result @@ -283,7 +315,14 @@ func writeCache(cache *includeCache, path *paths.Path) error { if cache.valid { path.Chtimes(time.Now(), time.Now()) } else { - bytes, err := json.MarshalIndent(cache.entries, "", " ") + var entries []*includeCacheEntry + cache.entries.Range(func(k, v interface{}) bool { + if entry, ok := v.(*includeCacheEntry); ok { + entries = append(entries, entry) + } + return true + }) + bytes, err := json.MarshalIndent(entries, "", " ") if err != nil { return errors.WithStack(err) } @@ -295,6 +334,46 @@ func writeCache(cache *includeCache, path *paths.Path) error { return nil } +// Preload the cached include files and libraries +func preloadCachedFolder(ctx *types.Context, cache *includeCache) { + var entryToRemove []string + cache.entries.Range(func(include, entry interface{}) bool { + + header, ok := include.(string) + if(ok) { + // Ignore the pre-added folder + if(!strings.HasPrefix(header, "no_resolve")) { + library, imported := ResolveLibrary(ctx, header) + if library == nil { + if !imported { + // Cannot find the library and it is not imported, is it gone? Remove it later + entryToRemove = append(entryToRemove, header) + cache.valid = false + } + } else { + + // Add this library to the list of libraries, the + // include path and queue its source files for further + // include scanning + ctx.ImportedLibraries = append(ctx.ImportedLibraries, library) + // Since it is already in cache, append the include folder only + appendIncludeFolderWoCache(ctx, header, library.SourceDir) + sourceDirs := library.SourceDirs() + for _, sourceDir := range sourceDirs { + queueSourceFilesFromFolder(ctx, ctx.CollectedSourceFiles, library, sourceDir.Dir, sourceDir.Recurse) + } + } + } + } + return true + }) + // Remove the invalid entry + for entry := range entryToRemove { + cache.entries.Delete(entry) + } +} + +// For the uncached/updated source file, find the include files func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile types.SourceFile) error { sourcePath := sourceFile.SourcePath(ctx) targetFilePath := paths.NullPath() @@ -321,7 +400,6 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t first := true for { var include string - cache.ExpectFile(sourcePath) includes := ctx.IncludeFolders if library, ok := sourceFile.Origin.(*libraries.Library); ok && library.UtilityDir != nil { @@ -342,11 +420,11 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t var preproc_err error var preproc_stderr []byte - if unchanged && cache.valid { - include = cache.Next().Include + if unchanged { if first && ctx.Verbose { ctx.Info(tr("Using cached library dependencies for file: %[1]s", sourcePath)) } + return nil } else { preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) // Unwrap error and see if it is an ExitError. @@ -369,34 +447,42 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t if include == "" { // No missing includes found, we're done - cache.ExpectEntry(sourcePath, "", nil) return nil } - library := ResolveLibrary(ctx, include) + libMux.Lock() + library, imported := ResolveLibrary(ctx, include) if library == nil { - // Library could not be resolved, show error - // err := runCommand(ctx, &GCCPreprocRunner{SourceFilePath: sourcePath, TargetFileName: paths.New(constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E), Includes: includes}) - // return errors.WithStack(err) - if preproc_err == nil || preproc_stderr == nil { - // Filename came from cache, so run preprocessor to obtain error to show - preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) - if preproc_err == nil { - // If there is a missing #include in the cache, but running - // gcc does not reproduce that, there is something wrong. - // Returning an error here will cause the cache to be - // deleted, so hopefully the next compilation will succeed. - return errors.New(tr("Internal error in cache")) + if imported { + // Added by others + libMux.Unlock() + continue + } else { + defer libMux.Unlock() + // Library could not be resolved, show error + // err := runCommand(ctx, &GCCPreprocRunner{SourceFilePath: sourcePath, TargetFileName: paths.New(constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E), Includes: includes}) + // return errors.WithStack(err) + if preproc_err == nil || preproc_stderr == nil { + // Filename came from cache, so run preprocessor to obtain error to show + preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) + if preproc_err == nil { + // If there is a missing #include in the cache, but running + // gcc does not reproduce that, there is something wrong. + // Returning an error here will cause the cache to be + // deleted, so hopefully the next compilation will succeed. + return errors.New(tr("Internal error in cache")) + } } + ctx.Stderr.Write(preproc_stderr) + return errors.WithStack(preproc_err) } - ctx.Stderr.Write(preproc_stderr) - return errors.WithStack(preproc_err) } // Add this library to the list of libraries, the // include path and queue its source files for further // include scanning ctx.ImportedLibraries = append(ctx.ImportedLibraries, library) + libMux.Unlock() appendIncludeFolder(ctx, cache, sourcePath, include, library.SourceDir) sourceDirs := library.SourceDirs() for _, sourceDir := range sourceDirs { @@ -421,7 +507,10 @@ func queueSourceFilesFromFolder(ctx *types.Context, queue *types.UniqueSourceFil if err != nil { return errors.WithStack(err) } + + fileMux.Lock() queue.Push(sourceFile) + fileMux.Unlock() } return nil diff --git a/legacy/builder/resolve_library.go b/legacy/builder/resolve_library.go index 3cb3c07aee3..16108a85c6e 100644 --- a/legacy/builder/resolve_library.go +++ b/legacy/builder/resolve_library.go @@ -22,7 +22,7 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/types" ) -func ResolveLibrary(ctx *types.Context, header string) *libraries.Library { +func ResolveLibrary(ctx *types.Context, header string) (*libraries.Library, bool) { resolver := ctx.LibrariesResolver importedLibraries := ctx.ImportedLibraries @@ -35,12 +35,12 @@ func ResolveLibrary(ctx *types.Context, header string) *libraries.Library { } if candidates == nil || len(candidates) == 0 { - return nil + return nil, false } for _, candidate := range candidates { if importedLibraries.Contains(candidate) { - return nil + return nil, true } } @@ -63,7 +63,7 @@ func ResolveLibrary(ctx *types.Context, header string) *libraries.Library { NotUsedLibraries: filterOutLibraryFrom(candidates, selected), } - return selected + return selected, false } func filterOutLibraryFrom(libs libraries.List, libraryToRemove *libraries.Library) libraries.List {