Revert "Simpler exporting"

This reverts commit 9cde4d9676.
This commit is contained in:
Alex Yatskov 2021-04-25 09:48:52 -07:00
parent 14c9c2065f
commit 8cddfc6e59
5 changed files with 91 additions and 115 deletions

View File

@ -69,7 +69,7 @@ func (cache *cache) storeFile(context *Context, outputFile *File, inputFiles []*
func (cache *cache) buildCachePath(context *Context, outputPath string, inputFiles []*File) (string, error) { func (cache *cache) buildCachePath(context *Context, outputPath string, inputFiles []*File) (string, error) {
uintBuff := make([]byte, 4) uintBuff := make([]byte, 4)
binary.LittleEndian.PutUint32(uintBuff, context.chainHash) binary.LittleEndian.PutUint32(uintBuff, context.hash)
hasher := crc32.NewIEEE() hasher := crc32.NewIEEE()
hasher.Write(uintBuff) hasher.Write(uintBuff)

View File

@ -14,16 +14,14 @@ import (
type Context struct { type Context struct {
goldsmith *Goldsmith goldsmith *Goldsmith
plugin Plugin plugin Plugin
chainHash uint32 hash uint32
filtersExt filterStack filtersExternal filterStack
filtersInt filterStack filtersInternal filterStack
threads int inputFiles chan *File
outputFiles chan *File
filesIn chan *File
filesOut chan *File
} }
// CreateFileFrom data creates a new file instance from the provided data buffer. // CreateFileFrom data creates a new file instance from the provided data buffer.
@ -60,46 +58,32 @@ func (*Context) CreateFileFromAsset(sourcePath, dataPath string) (*File, error)
// DispatchFile causes the file to get passed to the next link in the chain. // DispatchFile causes the file to get passed to the next link in the chain.
func (context *Context) DispatchFile(file *File) { func (context *Context) DispatchFile(file *File) {
context.filesOut <- file context.outputFiles <- file
} }
// DispatchAndCacheFile caches the file data (excluding the metadata), taking // DispatchAndCacheFile caches the file data (excluding the metadata), taking
// dependencies on any input files that are needed to generate it, and then // dependencies on any input files that are needed to generate it, and then
// passes it to the next link in the chain. // passes it to the next link in the chain.
func (context *Context) DispatchAndCacheFile(outputFile *File, inputFiles ...*File) { func (context *Context) DispatchAndCacheFile(outputFile *File, inputFiles ...*File) {
if context.goldsmith.cache != nil { context.goldsmith.storeFile(context, outputFile, inputFiles)
context.goldsmith.cache.storeFile(context, outputFile, inputFiles) context.outputFiles <- outputFile
}
context.filesOut <- outputFile
} }
// RetrieveCachedFile looks up file data (excluding the metadata), given an // RetrieveCachedFile looks up file data (excluding the metadata), given an
// output path and any input files that are needed to generate it. The function // output path and any input files that are needed to generate it. The function
// will return nil if the desired file is not found in the cache. // will return nil if the desired file is not found in the cache.
func (context *Context) RetrieveCachedFile(outputPath string, inputFiles ...*File) *File { func (context *Context) RetrieveCachedFile(outputPath string, inputFiles ...*File) *File {
var outputFile *File return context.goldsmith.retrieveFile(context, outputPath, inputFiles)
if context.goldsmith.cache != nil {
outputFile, _ = context.goldsmith.cache.retrieveFile(context, outputPath, inputFiles)
}
return outputFile
} }
// Specify internal filter(s) that exclude files from being processed. // Specify internal filter(s) that exclude files from being processed.
func (context *Context) Filter(filters ...Filter) *Context { func (context *Context) Filter(filters ...Filter) *Context {
context.filtersInt = filters context.filtersInternal = filters
return context
}
// Specify the maximum number of threads used for processing.
func (context *Context) Threads(threads int) *Context {
context.threads = threads
return context return context
} }
func (context *Context) step() { func (context *Context) step() {
defer close(context.filesOut) defer close(context.outputFiles)
if initializer, ok := context.plugin.(Initializer); ok { if initializer, ok := context.plugin.(Initializer); ok {
if err := initializer.Initialize(context); err != nil { if err := initializer.Initialize(context); err != nil {
@ -108,21 +92,16 @@ func (context *Context) step() {
} }
} }
if context.filesIn != nil { if context.inputFiles != nil {
processor, _ := context.plugin.(Processor) processor, _ := context.plugin.(Processor)
threads := context.threads
if threads < 1 {
threads = runtime.NumCPU()
}
var wg sync.WaitGroup var wg sync.WaitGroup
for i := 0; i < threads; i++ { for i := 0; i < runtime.NumCPU(); i++ {
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
for inputFile := range context.filesIn { for inputFile := range context.inputFiles {
if processor != nil && context.filtersInt.accept(inputFile) && context.filtersExt.accept(inputFile) { if processor != nil && context.filtersInternal.accept(inputFile) && context.filtersExternal.accept(inputFile) {
if _, err := inputFile.Seek(0, os.SEEK_SET); err != nil { if _, err := inputFile.Seek(0, os.SEEK_SET); err != nil {
context.goldsmith.fault("core", inputFile, err) context.goldsmith.fault("core", inputFile, err)
} }
@ -130,7 +109,7 @@ func (context *Context) step() {
context.goldsmith.fault(context.plugin.Name(), inputFile, err) context.goldsmith.fault(context.plugin.Name(), inputFile, err)
} }
} else { } else {
context.filesOut <- inputFile context.outputFiles <- inputFile
} }
} }
}() }()

View File

@ -5,6 +5,8 @@ import (
"fmt" "fmt"
"hash" "hash"
"hash/crc32" "hash/crc32"
"os"
"path/filepath"
"sync" "sync"
) )
@ -16,7 +18,9 @@ type Goldsmith struct {
contexts []*Context contexts []*Context
contextHasher hash.Hash32 contextHasher hash.Hash32
cache *cache fileRefs map[string]bool
fileCache *cache
filters filterStack filters filterStack
clean bool clean bool
@ -29,15 +33,16 @@ func Begin(sourceDir string) *Goldsmith {
goldsmith := &Goldsmith{ goldsmith := &Goldsmith{
sourceDir: sourceDir, sourceDir: sourceDir,
contextHasher: crc32.NewIEEE(), contextHasher: crc32.NewIEEE(),
fileRefs: make(map[string]bool),
} }
goldsmith.Chain(&loader{}) goldsmith.Chain(new(loader))
return goldsmith return goldsmith
} }
// Cache enables caching in cacheDir for the remainder of the chain. // Cache enables caching in cacheDir for the remainder of the chain.
func (goldsmith *Goldsmith) Cache(cacheDir string) *Goldsmith { func (goldsmith *Goldsmith) Cache(cacheDir string) *Goldsmith {
goldsmith.cache = &cache{cacheDir} goldsmith.fileCache = &cache{cacheDir}
return goldsmith return goldsmith
} }
@ -52,16 +57,16 @@ func (goldsmith *Goldsmith) Chain(plugin Plugin) *Goldsmith {
goldsmith.contextHasher.Write([]byte(plugin.Name())) goldsmith.contextHasher.Write([]byte(plugin.Name()))
context := &Context{ context := &Context{
goldsmith: goldsmith, goldsmith: goldsmith,
plugin: plugin, plugin: plugin,
chainHash: goldsmith.contextHasher.Sum32(), hash: goldsmith.contextHasher.Sum32(),
filesOut: make(chan *File), outputFiles: make(chan *File),
} }
context.filtersExt = append(context.filtersExt, goldsmith.filters...) context.filtersExternal = append(context.filtersExternal, goldsmith.filters...)
if len(goldsmith.contexts) > 0 { if len(goldsmith.contexts) > 0 {
context.filesIn = goldsmith.contexts[len(goldsmith.contexts)-1].filesOut context.inputFiles = goldsmith.contexts[len(goldsmith.contexts)-1].outputFiles
} }
goldsmith.contexts = append(goldsmith.contexts, context) goldsmith.contexts = append(goldsmith.contexts, context)
@ -84,23 +89,66 @@ func (goldsmith *Goldsmith) FilterPop() *Goldsmith {
func (goldsmith *Goldsmith) End(targetDir string) []error { func (goldsmith *Goldsmith) End(targetDir string) []error {
goldsmith.targetDir = targetDir goldsmith.targetDir = targetDir
var wg sync.WaitGroup
goldsmith.Chain(&saver{
clean: goldsmith.clean,
wg: &wg,
})
wg.Add(1)
for _, context := range goldsmith.contexts { for _, context := range goldsmith.contexts {
go context.step() go context.step()
} }
wg.Wait() context := goldsmith.contexts[len(goldsmith.contexts)-1]
for file := range context.outputFiles {
if goldsmith.filters.accept(file) {
goldsmith.exportFile(file)
}
}
if goldsmith.clean {
goldsmith.removeUnreferencedFiles()
}
return goldsmith.errors return goldsmith.errors
} }
func (goldsmith *Goldsmith) retrieveFile(context *Context, outputPath string, inputFiles []*File) *File {
if goldsmith.fileCache != nil {
outputFile, _ := goldsmith.fileCache.retrieveFile(context, outputPath, inputFiles)
return outputFile
}
return nil
}
func (goldsmith *Goldsmith) storeFile(context *Context, outputFile *File, inputFiles []*File) {
if goldsmith.fileCache != nil {
goldsmith.fileCache.storeFile(context, outputFile, inputFiles)
}
}
func (goldsmith *Goldsmith) removeUnreferencedFiles() {
infos := make(chan fileInfo)
go scanDir(goldsmith.targetDir, infos)
for info := range infos {
if info.path != goldsmith.targetDir {
relPath, _ := filepath.Rel(goldsmith.targetDir, info.path)
if contained, _ := goldsmith.fileRefs[relPath]; !contained {
os.RemoveAll(info.path)
}
}
}
}
func (goldsmith *Goldsmith) exportFile(file *File) error {
if err := file.export(goldsmith.targetDir); err != nil {
return err
}
for pathSeg := cleanPath(file.sourcePath); pathSeg != "."; pathSeg = filepath.Dir(pathSeg) {
goldsmith.fileRefs[pathSeg] = true
}
return nil
}
func (goldsmith *Goldsmith) fault(name string, file *File, err error) { func (goldsmith *Goldsmith) fault(name string, file *File, err error) {
goldsmith.mutex.Lock() goldsmith.mutex.Lock()
defer goldsmith.mutex.Unlock() defer goldsmith.mutex.Unlock()

View File

@ -2,22 +2,24 @@ package goldsmith
import "path/filepath" import "path/filepath"
type loader struct{} type loader struct {
Initializer
}
func (*loader) Name() string { func (*loader) Name() string {
return "loader" return "loader"
} }
func (*loader) Initialize(context *Context) error { func (*loader) Initialize(ctx *Context) error {
infos := make(chan fileInfo) infos := make(chan fileInfo)
go scanDir(context.goldsmith.sourceDir, infos) go scanDir(ctx.goldsmith.sourceDir, infos)
for info := range infos { for info := range infos {
if info.IsDir() { if info.IsDir() {
continue continue
} }
relPath, _ := filepath.Rel(context.goldsmith.sourceDir, info.path) relPath, _ := filepath.Rel(ctx.goldsmith.sourceDir, info.path)
file := &File{ file := &File{
sourcePath: relPath, sourcePath: relPath,
@ -27,7 +29,7 @@ func (*loader) Initialize(context *Context) error {
dataPath: info.path, dataPath: info.path,
} }
context.DispatchFile(file) ctx.DispatchFile(file)
} }
return nil return nil

View File

@ -1,53 +0,0 @@
package goldsmith
import (
"os"
"path/filepath"
"sync"
)
type saver struct {
clean bool
tokens map[string]bool
wg *sync.WaitGroup
}
func (*saver) Name() string {
return "saver"
}
func (saver *saver) Initialize(context *Context) error {
saver.tokens = make(map[string]bool)
context.Threads(1)
return nil
}
func (saver *saver) Process(context *Context, file *File) error {
for token := cleanPath(file.sourcePath); token != "."; token = filepath.Dir(token) {
saver.tokens[token] = true
}
return file.export(context.goldsmith.targetDir)
}
func (saver *saver) Finalize(context *Context) error {
defer saver.wg.Done()
if !saver.clean {
return nil
}
infos := make(chan fileInfo)
go scanDir(context.goldsmith.targetDir, infos)
for info := range infos {
if info.path != context.goldsmith.targetDir {
relPath, _ := filepath.Rel(context.goldsmith.targetDir, info.path)
if contained, _ := saver.tokens[relPath]; !contained {
os.RemoveAll(info.path)
}
}
}
return nil
}