From 2aa47f27405d478dafb585477c0c89c2a1641219 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 8 Apr 2026 14:52:56 -0700 Subject: [PATCH 1/2] Limit loader/emitter to GOMAXPROCS --- internal/compiler/fileloader.go | 4 ++-- internal/compiler/program.go | 2 +- internal/core/workgroup.go | 37 +++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index 8bda8dfedf1..0f5e0319b9d 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -138,7 +138,7 @@ func processAllProgramFiles( CurrentDirectory: opts.Host.GetCurrentDirectory(), }, filesParser: &filesParser{ - wg: core.NewWorkGroup(singleThreaded), + wg: core.NewThrottledWorkGroup(singleThreaded), maxDepth: maxNodeModuleJsDepth, }, rootTasks: make([]*parseTask, 0, len(rootFiles)+len(compilerOptions.Lib)), @@ -268,7 +268,7 @@ func (p *fileLoader) addProjectReferenceTasks(singleThreaded bool) { parser := &projectReferenceParser{ loader: p, - wg: core.NewWorkGroup(singleThreaded), + wg: core.NewThrottledWorkGroup(singleThreaded), } rootTasks := createProjectReferenceParseTasks(projectReferences) parser.parse(rootTasks) diff --git a/internal/compiler/program.go b/internal/compiler/program.go index 2f3aa3024d4..a19f1960b8b 100644 --- a/internal/compiler/program.go +++ b/internal/compiler/program.go @@ -1580,7 +1580,7 @@ func (p *Program) Emit(ctx context.Context, options EmitOptions) *EmitResult { return printer.NewTextWriter(newLine, 0) }, } - wg := core.NewWorkGroup(p.SingleThreaded()) + wg := core.NewThrottledWorkGroup(p.SingleThreaded()) var emitters []*emitter sourceFiles := p.getSourceFilesToEmit(options.TargetSourceFile, options.EmitOnly == EmitOnlyForcedDts) diff --git a/internal/core/workgroup.go b/internal/core/workgroup.go index 73618a1a19f..0f7bb9f02a3 100644 --- a/internal/core/workgroup.go +++ b/internal/core/workgroup.go @@ -2,6 +2,7 @@ package core import ( "context" + "runtime" "sync" "sync/atomic" @@ -24,6 +25,17 @@ func NewWorkGroup(singleThreaded bool) WorkGroup { return ¶llelWorkGroup{} } +// NewThrottledWorkGroup creates a WorkGroup that limits concurrent execution to GOMAXPROCS. +// If singleThreaded is true, all work runs sequentially in RunAndWait. +func NewThrottledWorkGroup(singleThreaded bool) WorkGroup { + if singleThreaded { + return &singleThreadedWorkGroup{} + } + return &throttledWorkGroup{ + sem: make(chan struct{}, runtime.GOMAXPROCS(0)), + } +} + type parallelWorkGroup struct { done atomic.Bool wg sync.WaitGroup @@ -88,6 +100,31 @@ func (w *singleThreadedWorkGroup) pop() func() { return fn } +type throttledWorkGroup struct { + done atomic.Bool + wg sync.WaitGroup + sem chan struct{} +} + +var _ WorkGroup = (*throttledWorkGroup)(nil) + +func (w *throttledWorkGroup) Queue(fn func()) { + if w.done.Load() { + panic("Queue called after RunAndWait returned") + } + + w.wg.Go(func() { + w.sem <- struct{}{} + defer func() { <-w.sem }() + fn() + }) +} + +func (w *throttledWorkGroup) RunAndWait() { + defer w.done.Store(true) + w.wg.Wait() +} + // ThrottleGroup is like errgroup.Group but with global concurrency limiting via a semaphore. type ThrottleGroup struct { semaphore chan struct{} From cab6754494618301e8c39f13b5a006cd66eaaaf0 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Fri, 10 Apr 2026 19:09:13 -0700 Subject: [PATCH 2/2] wild idea --- internal/compiler/fileloader.go | 8 +++++--- internal/compiler/filesparser.go | 15 +++++++++++++++ internal/core/workgroup.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index 0f5e0319b9d..db373bdfa5c 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -138,8 +138,10 @@ func processAllProgramFiles( CurrentDirectory: opts.Host.GetCurrentDirectory(), }, filesParser: &filesParser{ - wg: core.NewThrottledWorkGroup(singleThreaded), - maxDepth: maxNodeModuleJsDepth, + wg: core.NewWorkGroup(singleThreaded), + resolveSem: core.NewSemaphore(singleThreaded), + parseSem: core.NewSemaphore(singleThreaded), + maxDepth: maxNodeModuleJsDepth, }, rootTasks: make([]*parseTask, 0, len(rootFiles)+len(compilerOptions.Lib)), supportedExtensions: supportedExtensions, @@ -268,7 +270,7 @@ func (p *fileLoader) addProjectReferenceTasks(singleThreaded bool) { parser := &projectReferenceParser{ loader: p, - wg: core.NewThrottledWorkGroup(singleThreaded), + wg: core.NewWorkGroup(singleThreaded), } rootTasks := createProjectReferenceParseTasks(projectReferences) parser.parse(rootTasks) diff --git a/internal/compiler/filesparser.go b/internal/compiler/filesparser.go index ccf60204b78..d837bfe2847 100644 --- a/internal/compiler/filesparser.go +++ b/internal/compiler/filesparser.go @@ -102,10 +102,16 @@ func (t *parseTask) load(loader *fileLoader) { // to avoid adding spurious lookups to file watcher tracking. t.metadata = ast.SourceFileMetaData{ImpliedNodeFormat: core.ResolutionModeCommonJS} } else { + // Resolution: load metadata (package.json lookups involve stat calls). + loader.filesParser.resolveSem.Acquire() t.metadata = loader.loadSourceFileMetaData(t.normalizedFilePath) + loader.filesParser.resolveSem.Release() } + // Parsing: parse the source file (CPU-heavy). + loader.filesParser.parseSem.Acquire() file := loader.parseSourceFile(t) + loader.filesParser.parseSem.Release() if file == nil { return } @@ -113,6 +119,9 @@ func (t *parseTask) load(loader *fileLoader) { t.file = file t.subTasks = make([]*parseTask, 0, len(file.ReferencedFiles)+len(file.Imports())+len(file.ModuleAugmentations)) + // Resolution: resolve references, type directives, and imports (stat-heavy). + loader.filesParser.resolveSem.Acquire() + compilerOptions := loader.opts.Config.CompilerOptions() if !compilerOptions.NoResolve.IsTrue() { for index, ref := range file.ReferencedFiles { @@ -152,6 +161,8 @@ func (t *parseTask) load(loader *fileLoader) { } loader.resolveImportsAndModuleAugmentations(t) + + loader.filesParser.resolveSem.Release() } func (t *parseTask) redirect(loader *fileLoader, fileName string) { @@ -165,6 +176,8 @@ func (t *parseTask) redirect(loader *fileLoader, fileName string) { } func (t *parseTask) loadAutomaticTypeDirectives(loader *fileLoader) { + loader.filesParser.resolveSem.Acquire() + defer loader.filesParser.resolveSem.Release() toParseTypeRefs, typeResolutionsInFile, typeResolutionsTrace, pDiagnostics := loader.resolveAutomaticTypeDirectives(t.normalizedFilePath) t.typeResolutionsInFile = typeResolutionsInFile t.typeResolutionsTrace = typeResolutionsTrace @@ -197,6 +210,8 @@ func (t *parseTask) addSubTask(ref resolvedRef, libFile *LibFile) { type filesParser struct { wg core.WorkGroup + resolveSem *core.Semaphore + parseSem *core.Semaphore taskDataByPath collections.SyncMap[tspath.Path, *parseTaskData] maxDepth int } diff --git a/internal/core/workgroup.go b/internal/core/workgroup.go index 0f7bb9f02a3..8f96dd37249 100644 --- a/internal/core/workgroup.go +++ b/internal/core/workgroup.go @@ -36,6 +36,38 @@ func NewThrottledWorkGroup(singleThreaded bool) WorkGroup { } } +// Semaphore limits concurrent access to a resource. +// A nil *Semaphore is valid and performs no limiting; +// callers are expected to arrange sequential execution themselves +// when no semaphore is provided. +type Semaphore struct { + ch chan struct{} +} + +// NewSemaphore creates a Semaphore sized to GOMAXPROCS. +// Returns nil when singleThreaded is true, as the caller's WorkGroup +// already ensures sequential execution. +func NewSemaphore(singleThreaded bool) *Semaphore { + if singleThreaded { + return nil + } + return &Semaphore{ch: make(chan struct{}, runtime.GOMAXPROCS(0))} +} + +// Acquire blocks until a slot is available. +func (s *Semaphore) Acquire() { + if s != nil { + s.ch <- struct{}{} + } +} + +// Release frees a slot. +func (s *Semaphore) Release() { + if s != nil { + <-s.ch + } +} + type parallelWorkGroup struct { done atomic.Bool wg sync.WaitGroup