From ec13a459ec6ceb896600637f0e82be006cbbb338 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Sat, 1 May 2021 15:12:50 -0700 Subject: [PATCH] New export path --- cache.go | 2 +- context.go | 55 ++++++++++++++++++++++++++------------ goldsmith.go | 74 ++++++++-------------------------------------------- loader.go | 12 ++++----- saver.go | 49 ++++++++++++++++++++++++++++++++++ 5 files changed, 104 insertions(+), 88 deletions(-) create mode 100644 saver.go diff --git a/cache.go b/cache.go index 8004f43..71b7acc 100644 --- a/cache.go +++ b/cache.go @@ -69,7 +69,7 @@ func (cache *cache) storeFile(context *Context, outputFile *File, inputFiles []* func (cache *cache) buildCachePath(context *Context, outputPath string, inputFiles []*File) (string, error) { uintBuff := make([]byte, 4) - binary.LittleEndian.PutUint32(uintBuff, context.hash) + binary.LittleEndian.PutUint32(uintBuff, context.chainHash) hasher := crc32.NewIEEE() hasher.Write(uintBuff) diff --git a/context.go b/context.go index 3d260ea..ba7ceb5 100644 --- a/context.go +++ b/context.go @@ -14,14 +14,16 @@ import ( type Context struct { goldsmith *Goldsmith - plugin Plugin - hash uint32 + plugin Plugin + chainHash uint32 - filtersExternal filterStack - filtersInternal filterStack + filtersExt filterStack + filtersInt filterStack - inputFiles chan *File - outputFiles chan *File + threads int + + filesIn chan *File + filesOut chan *File } // CreateFileFrom data creates a new file instance from the provided data buffer. @@ -58,32 +60,46 @@ func (*Context) CreateFileFromAsset(sourcePath, dataPath string) (*File, error) // DispatchFile causes the file to get passed to the next link in the chain. func (context *Context) DispatchFile(file *File) { - context.outputFiles <- file + context.filesOut <- file } // DispatchAndCacheFile caches the file data (excluding the metadata), taking // dependencies on any input files that are needed to generate it, and then // passes it to the next link in the chain. func (context *Context) DispatchAndCacheFile(outputFile *File, inputFiles ...*File) { - context.goldsmith.storeFile(context, outputFile, inputFiles) - context.outputFiles <- outputFile + if context.goldsmith.cache != nil { + context.goldsmith.cache.storeFile(context, outputFile, inputFiles) + } + + context.filesOut <- outputFile } // RetrieveCachedFile looks up file data (excluding the metadata), given an // output path and any input files that are needed to generate it. The function // will return nil if the desired file is not found in the cache. func (context *Context) RetrieveCachedFile(outputPath string, inputFiles ...*File) *File { - return context.goldsmith.retrieveFile(context, outputPath, inputFiles) + var outputFile *File + if context.goldsmith.cache != nil { + outputFile, _ = context.goldsmith.cache.retrieveFile(context, outputPath, inputFiles) + } + + return outputFile } // Specify internal filter(s) that exclude files from being processed. func (context *Context) Filter(filters ...Filter) *Context { - context.filtersInternal = filters + context.filtersInt = filters + return context +} + +// Specify the maximum number of threads used for processing. +func (context *Context) Threads(threads int) *Context { + context.threads = threads return context } func (context *Context) step() { - defer close(context.outputFiles) + defer close(context.filesOut) if initializer, ok := context.plugin.(Initializer); ok { if err := initializer.Initialize(context); err != nil { @@ -92,16 +108,21 @@ func (context *Context) step() { } } - if context.inputFiles != nil { + if context.filesIn != nil { processor, _ := context.plugin.(Processor) + threads := context.threads + if threads < 1 { + threads = runtime.NumCPU() + } + var wg sync.WaitGroup - for i := 0; i < runtime.NumCPU(); i++ { + for i := 0; i < threads; i++ { wg.Add(1) go func() { defer wg.Done() - for inputFile := range context.inputFiles { - if processor != nil && context.filtersInternal.accept(inputFile) && context.filtersExternal.accept(inputFile) { + for inputFile := range context.filesIn { + if processor != nil && context.filtersInt.accept(inputFile) && context.filtersExt.accept(inputFile) { if _, err := inputFile.Seek(0, os.SEEK_SET); err != nil { context.goldsmith.fault("core", inputFile, err) } @@ -109,7 +130,7 @@ func (context *Context) step() { context.goldsmith.fault(context.plugin.Name(), inputFile, err) } } else { - context.outputFiles <- inputFile + context.filesOut <- inputFile } } }() diff --git a/goldsmith.go b/goldsmith.go index bad1dca..f6c8f51 100644 --- a/goldsmith.go +++ b/goldsmith.go @@ -5,8 +5,6 @@ import ( "fmt" "hash" "hash/crc32" - "os" - "path/filepath" "sync" ) @@ -18,9 +16,7 @@ type Goldsmith struct { contexts []*Context contextHasher hash.Hash32 - fileRefs map[string]bool - fileCache *cache - + cache *cache filters filterStack clean bool @@ -33,16 +29,15 @@ func Begin(sourceDir string) *Goldsmith { goldsmith := &Goldsmith{ sourceDir: sourceDir, contextHasher: crc32.NewIEEE(), - fileRefs: make(map[string]bool), } - goldsmith.Chain(new(loader)) + goldsmith.Chain(&loader{}) return goldsmith } // Cache enables caching in cacheDir for the remainder of the chain. func (goldsmith *Goldsmith) Cache(cacheDir string) *Goldsmith { - goldsmith.fileCache = &cache{cacheDir} + goldsmith.cache = &cache{cacheDir} return goldsmith } @@ -57,16 +52,16 @@ func (goldsmith *Goldsmith) Chain(plugin Plugin) *Goldsmith { goldsmith.contextHasher.Write([]byte(plugin.Name())) context := &Context{ - goldsmith: goldsmith, - plugin: plugin, - hash: goldsmith.contextHasher.Sum32(), - outputFiles: make(chan *File), + goldsmith: goldsmith, + plugin: plugin, + chainHash: goldsmith.contextHasher.Sum32(), + filesOut: make(chan *File), } - context.filtersExternal = append(context.filtersExternal, goldsmith.filters...) + context.filtersExt = append(context.filtersExt, goldsmith.filters...) if len(goldsmith.contexts) > 0 { - context.inputFiles = goldsmith.contexts[len(goldsmith.contexts)-1].outputFiles + context.filesIn = goldsmith.contexts[len(goldsmith.contexts)-1].filesOut } goldsmith.contexts = append(goldsmith.contexts, context) @@ -89,66 +84,19 @@ func (goldsmith *Goldsmith) FilterPop() *Goldsmith { func (goldsmith *Goldsmith) End(targetDir string) []error { goldsmith.targetDir = targetDir + goldsmith.Chain(&saver{clean: goldsmith.clean}) for _, context := range goldsmith.contexts { go context.step() } context := goldsmith.contexts[len(goldsmith.contexts)-1] - for file := range context.outputFiles { - if goldsmith.filters.accept(file) { - goldsmith.exportFile(file) - } - } + for range context.filesOut { - if goldsmith.clean { - goldsmith.removeUnreferencedFiles() } return goldsmith.errors } -func (goldsmith *Goldsmith) retrieveFile(context *Context, outputPath string, inputFiles []*File) *File { - if goldsmith.fileCache != nil { - outputFile, _ := goldsmith.fileCache.retrieveFile(context, outputPath, inputFiles) - return outputFile - } - - return nil -} - -func (goldsmith *Goldsmith) storeFile(context *Context, outputFile *File, inputFiles []*File) { - if goldsmith.fileCache != nil { - goldsmith.fileCache.storeFile(context, outputFile, inputFiles) - } - -} - -func (goldsmith *Goldsmith) removeUnreferencedFiles() { - infos := make(chan fileInfo) - go scanDir(goldsmith.targetDir, infos) - - for info := range infos { - if info.path != goldsmith.targetDir { - relPath, _ := filepath.Rel(goldsmith.targetDir, info.path) - if contained, _ := goldsmith.fileRefs[relPath]; !contained { - os.RemoveAll(info.path) - } - } - } -} - -func (goldsmith *Goldsmith) exportFile(file *File) error { - if err := file.export(goldsmith.targetDir); err != nil { - return err - } - - for pathSeg := cleanPath(file.sourcePath); pathSeg != "."; pathSeg = filepath.Dir(pathSeg) { - goldsmith.fileRefs[pathSeg] = true - } - - return nil -} - func (goldsmith *Goldsmith) fault(name string, file *File, err error) { goldsmith.mutex.Lock() defer goldsmith.mutex.Unlock() diff --git a/loader.go b/loader.go index 478a399..fba2cd9 100644 --- a/loader.go +++ b/loader.go @@ -2,24 +2,22 @@ package goldsmith import "path/filepath" -type loader struct { - Initializer -} +type loader struct{} func (*loader) Name() string { return "loader" } -func (*loader) Initialize(ctx *Context) error { +func (*loader) Initialize(context *Context) error { infos := make(chan fileInfo) - go scanDir(ctx.goldsmith.sourceDir, infos) + go scanDir(context.goldsmith.sourceDir, infos) for info := range infos { if info.IsDir() { continue } - relPath, _ := filepath.Rel(ctx.goldsmith.sourceDir, info.path) + relPath, _ := filepath.Rel(context.goldsmith.sourceDir, info.path) file := &File{ sourcePath: relPath, @@ -29,7 +27,7 @@ func (*loader) Initialize(ctx *Context) error { dataPath: info.path, } - ctx.DispatchFile(file) + context.DispatchFile(file) } return nil diff --git a/saver.go b/saver.go new file mode 100644 index 0000000..0545f49 --- /dev/null +++ b/saver.go @@ -0,0 +1,49 @@ +package goldsmith + +import ( + "os" + "path/filepath" +) + +type saver struct { + clean bool + tokens map[string]bool +} + +func (*saver) Name() string { + return "saver" +} + +func (saver *saver) Initialize(context *Context) error { + saver.tokens = make(map[string]bool) + context.Threads(1) + return nil +} + +func (saver *saver) Process(context *Context, file *File) error { + for token := cleanPath(file.sourcePath); token != "."; token = filepath.Dir(token) { + saver.tokens[token] = true + } + + return file.export(context.goldsmith.targetDir) +} + +func (saver *saver) Finalize(context *Context) error { + if !saver.clean { + return nil + } + + infos := make(chan fileInfo) + go scanDir(context.goldsmith.targetDir, infos) + + for info := range infos { + if info.path != context.goldsmith.targetDir { + relPath, _ := filepath.Rel(context.goldsmith.targetDir, info.path) + if contained, _ := saver.tokens[relPath]; !contained { + os.RemoveAll(info.path) + } + } + } + + return nil +}