From 14c434a6a4843ef95b1a446293f3fc10ec4c2f76 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Sun, 25 Apr 2021 08:56:07 -0700 Subject: [PATCH] Cleanup --- cache.go | 2 +- context.go | 43 ++++++++++++++++++++++++++++--------------- goldsmith.go | 20 +++++++++----------- loader.go | 8 ++++---- saver.go | 14 +++++++------- 5 files changed, 49 insertions(+), 38 deletions(-) diff --git a/cache.go b/cache.go index 8004f43..71b7acc 100644 --- a/cache.go +++ b/cache.go @@ -69,7 +69,7 @@ func (cache *cache) storeFile(context *Context, outputFile *File, inputFiles []* func (cache *cache) buildCachePath(context *Context, outputPath string, inputFiles []*File) (string, error) { uintBuff := make([]byte, 4) - binary.LittleEndian.PutUint32(uintBuff, context.hash) + binary.LittleEndian.PutUint32(uintBuff, context.chainHash) hasher := crc32.NewIEEE() hasher.Write(uintBuff) diff --git a/context.go b/context.go index 0d9e3b4..167ae11 100644 --- a/context.go +++ b/context.go @@ -14,14 +14,16 @@ import ( type Context struct { goldsmith *Goldsmith - plugin Plugin - hash uint32 + plugin Plugin + chainHash uint32 - filtersExternal filterStack - filtersInternal filterStack + filtersExt filterStack + filtersInt filterStack - inputFiles chan *File - outputFiles chan *File + threads int + + filesIn chan *File + filesOut chan *File } // CreateFileFrom data creates a new file instance from the provided data buffer. @@ -58,7 +60,7 @@ func (*Context) CreateFileFromAsset(sourcePath, dataPath string) (*File, error) // DispatchFile causes the file to get passed to the next link in the chain. func (context *Context) DispatchFile(file *File) { - context.outputFiles <- file + context.filesOut <- file } // DispatchAndCacheFile caches the file data (excluding the metadata), taking @@ -69,7 +71,7 @@ func (context *Context) DispatchAndCacheFile(outputFile *File, inputFiles ...*Fi context.goldsmith.fileCache.storeFile(context, outputFile, inputFiles) } - context.outputFiles <- outputFile + context.filesOut <- outputFile } // RetrieveCachedFile looks up file data (excluding the metadata), given an @@ -86,12 +88,18 @@ func (context *Context) RetrieveCachedFile(outputPath string, inputFiles ...*Fil // Specify internal filter(s) that exclude files from being processed. func (context *Context) Filter(filters ...Filter) *Context { - context.filtersInternal = filters + context.filtersInt = filters + return context +} + +// Specify the maximum number of threads used for processing. +func (context *Context) Threads(threads int) *Context { + context.threads = threads return context } func (context *Context) step() { - defer close(context.outputFiles) + defer close(context.filesOut) if initializer, ok := context.plugin.(Initializer); ok { if err := initializer.Initialize(context); err != nil { @@ -100,16 +108,21 @@ func (context *Context) step() { } } - if context.inputFiles != nil { + if context.filesIn != nil { processor, _ := context.plugin.(Processor) + threads := context.threads + if threads < 1 { + threads = runtime.NumCPU() + } + var wg sync.WaitGroup - for i := 0; i < runtime.NumCPU(); i++ { + for i := 0; i < threads; i++ { wg.Add(1) go func() { defer wg.Done() - for inputFile := range context.inputFiles { - if processor != nil && context.filtersInternal.accept(inputFile) && context.filtersExternal.accept(inputFile) { + for inputFile := range context.filesIn { + if processor != nil && context.filtersInt.accept(inputFile) && context.filtersExt.accept(inputFile) { if _, err := inputFile.Seek(0, os.SEEK_SET); err != nil { context.goldsmith.fault("core", inputFile, err) } @@ -117,7 +130,7 @@ func (context *Context) step() { context.goldsmith.fault(context.plugin.Name(), inputFile, err) } } else { - context.outputFiles <- inputFile + context.filesOut <- inputFile } } }() diff --git a/goldsmith.go b/goldsmith.go index e8edd15..ac19d33 100644 --- a/goldsmith.go +++ b/goldsmith.go @@ -17,9 +17,8 @@ type Goldsmith struct { contextHasher hash.Hash32 fileCache *cache - - filters filterStack - clean bool + filters filterStack + clean bool errors []error mutex sync.Mutex @@ -53,16 +52,16 @@ func (goldsmith *Goldsmith) Chain(plugin Plugin) *Goldsmith { goldsmith.contextHasher.Write([]byte(plugin.Name())) context := &Context{ - goldsmith: goldsmith, - plugin: plugin, - hash: goldsmith.contextHasher.Sum32(), - outputFiles: make(chan *File), + goldsmith: goldsmith, + plugin: plugin, + chainHash: goldsmith.contextHasher.Sum32(), + filesOut: make(chan *File), } - context.filtersExternal = append(context.filtersExternal, goldsmith.filters...) + context.filtersExt = append(context.filtersExt, goldsmith.filters...) if len(goldsmith.contexts) > 0 { - context.inputFiles = goldsmith.contexts[len(goldsmith.contexts)-1].outputFiles + context.filesIn = goldsmith.contexts[len(goldsmith.contexts)-1].filesOut } goldsmith.contexts = append(goldsmith.contexts, context) @@ -86,8 +85,7 @@ func (goldsmith *Goldsmith) End(targetDir string) []error { goldsmith.targetDir = targetDir goldsmith.Chain(&saver{ - clean: goldsmith.clean, - tokens: make(map[string]bool), + clean: goldsmith.clean, }) for _, context := range goldsmith.contexts { diff --git a/loader.go b/loader.go index d469237..fba2cd9 100644 --- a/loader.go +++ b/loader.go @@ -8,16 +8,16 @@ func (*loader) Name() string { return "loader" } -func (*loader) Initialize(ctx *Context) error { +func (*loader) Initialize(context *Context) error { infos := make(chan fileInfo) - go scanDir(ctx.goldsmith.sourceDir, infos) + go scanDir(context.goldsmith.sourceDir, infos) for info := range infos { if info.IsDir() { continue } - relPath, _ := filepath.Rel(ctx.goldsmith.sourceDir, info.path) + relPath, _ := filepath.Rel(context.goldsmith.sourceDir, info.path) file := &File{ sourcePath: relPath, @@ -27,7 +27,7 @@ func (*loader) Initialize(ctx *Context) error { dataPath: info.path, } - ctx.DispatchFile(file) + context.DispatchFile(file) } return nil diff --git a/saver.go b/saver.go index 2d7d4ac..0545f49 100644 --- a/saver.go +++ b/saver.go @@ -3,24 +3,24 @@ package goldsmith import ( "os" "path/filepath" - "sync" ) type saver struct { - clean bool - + clean bool tokens map[string]bool - mutex sync.Mutex } func (*saver) Name() string { return "saver" } -func (saver *saver) Process(context *Context, file *File) error { - saver.mutex.Lock() - defer saver.mutex.Unlock() +func (saver *saver) Initialize(context *Context) error { + saver.tokens = make(map[string]bool) + context.Threads(1) + return nil +} +func (saver *saver) Process(context *Context, file *File) error { for token := cleanPath(file.sourcePath); token != "."; token = filepath.Dir(token) { saver.tokens[token] = true }