Revert "Simpler exporting"

This reverts commit 9cde4d9676.

cache.go

@@ -69,7 +69,7 @@ func (cache *cache) storeFile(context *Context, outputFile *File, inputFiles []*
 
 func (cache *cache) buildCachePath(context *Context, outputPath string, inputFiles []*File) (string, error) {
 	uintBuff := make([]byte, 4)
-	binary.LittleEndian.PutUint32(uintBuff, context.chainHash)
+	binary.LittleEndian.PutUint32(uintBuff, context.hash)
 
 	hasher := crc32.NewIEEE()
 	hasher.Write(uintBuff)

context.go

@@ -14,16 +14,14 @@ import (
 type Context struct {
 	goldsmith *Goldsmith
 
-	plugin    Plugin
+	plugin Plugin
-	chainHash uint32
+	hash   uint32
 
-	filtersExt filterStack
+	filtersExternal filterStack
-	filtersInt filterStack
+	filtersInternal filterStack
 
-	threads int
+	inputFiles  chan *File
-
+	outputFiles chan *File
-	filesIn  chan *File
-	filesOut chan *File
 }
 
 // CreateFileFrom data creates a new file instance from the provided data buffer.
@@ -60,46 +58,32 @@ func (*Context) CreateFileFromAsset(sourcePath, dataPath string) (*File, error)
 
 // DispatchFile causes the file to get passed to the next link in the chain.
 func (context *Context) DispatchFile(file *File) {
-	context.filesOut <- file
+	context.outputFiles <- file
 }
 
 // DispatchAndCacheFile caches the file data (excluding the metadata), taking
 // dependencies on any input files that are needed to generate it, and then
 // passes it to the next link in the chain.
 func (context *Context) DispatchAndCacheFile(outputFile *File, inputFiles ...*File) {
-	if context.goldsmith.cache != nil {
+	context.goldsmith.storeFile(context, outputFile, inputFiles)
-		context.goldsmith.cache.storeFile(context, outputFile, inputFiles)
+	context.outputFiles <- outputFile
-	}
-
-	context.filesOut <- outputFile
 }
 
 // RetrieveCachedFile looks up file data (excluding the metadata), given an
 // output path and any input files that are needed to generate it. The function
 // will return nil if the desired file is not found in the cache.
 func (context *Context) RetrieveCachedFile(outputPath string, inputFiles ...*File) *File {
-	var outputFile *File
+	return context.goldsmith.retrieveFile(context, outputPath, inputFiles)
-	if context.goldsmith.cache != nil {
-		outputFile, _ = context.goldsmith.cache.retrieveFile(context, outputPath, inputFiles)
-	}
-
-	return outputFile
 }
 
 // Specify internal filter(s) that exclude files from being processed.
 func (context *Context) Filter(filters ...Filter) *Context {
-	context.filtersInt = filters
+	context.filtersInternal = filters
-	return context
-}
-
-// Specify the maximum number of threads used for processing.
-func (context *Context) Threads(threads int) *Context {
-	context.threads = threads
 	return context
 }
 
 func (context *Context) step() {
-	defer close(context.filesOut)
+	defer close(context.outputFiles)
 
 	if initializer, ok := context.plugin.(Initializer); ok {
 		if err := initializer.Initialize(context); err != nil {
@@ -108,21 +92,16 @@ func (context *Context) step() {
 		}
 	}
 
-	if context.filesIn != nil {
+	if context.inputFiles != nil {
 		processor, _ := context.plugin.(Processor)
 
-		threads := context.threads
-		if threads < 1 {
-			threads = runtime.NumCPU()
-		}
-
 		var wg sync.WaitGroup
-		for i := 0; i < threads; i++ {
+		for i := 0; i < runtime.NumCPU(); i++ {
 			wg.Add(1)
 			go func() {
 				defer wg.Done()
-				for inputFile := range context.filesIn {
+				for inputFile := range context.inputFiles {
-					if processor != nil && context.filtersInt.accept(inputFile) && context.filtersExt.accept(inputFile) {
+					if processor != nil && context.filtersInternal.accept(inputFile) && context.filtersExternal.accept(inputFile) {
 						if _, err := inputFile.Seek(0, os.SEEK_SET); err != nil {
 							context.goldsmith.fault("core", inputFile, err)
 						}
@@ -130,7 +109,7 @@ func (context *Context) step() {
 							context.goldsmith.fault(context.plugin.Name(), inputFile, err)
 						}
 					} else {
-						context.filesOut <- inputFile
+						context.outputFiles <- inputFile
 					}
 				}
 			}()

goldsmith.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"hash"
 	"hash/crc32"
+	"os"
+	"path/filepath"
 	"sync"
 )
 
@@ -16,7 +18,9 @@ type Goldsmith struct {
 	contexts      []*Context
 	contextHasher hash.Hash32
 
-	cache   *cache
+	fileRefs  map[string]bool
+	fileCache *cache
+
 	filters filterStack
 	clean   bool
 
@@ -29,15 +33,16 @@ func Begin(sourceDir string) *Goldsmith {
 	goldsmith := &Goldsmith{
 		sourceDir:     sourceDir,
 		contextHasher: crc32.NewIEEE(),
+		fileRefs:      make(map[string]bool),
 	}
 
-	goldsmith.Chain(&loader{})
+	goldsmith.Chain(new(loader))
 	return goldsmith
 }
 
 // Cache enables caching in cacheDir for the remainder of the chain.
 func (goldsmith *Goldsmith) Cache(cacheDir string) *Goldsmith {
-	goldsmith.cache = &cache{cacheDir}
+	goldsmith.fileCache = &cache{cacheDir}
 	return goldsmith
 }
 
@@ -52,16 +57,16 @@ func (goldsmith *Goldsmith) Chain(plugin Plugin) *Goldsmith {
 	goldsmith.contextHasher.Write([]byte(plugin.Name()))
 
 	context := &Context{
-		goldsmith: goldsmith,
+		goldsmith:   goldsmith,
-		plugin:    plugin,
+		plugin:      plugin,
-		chainHash: goldsmith.contextHasher.Sum32(),
+		hash:        goldsmith.contextHasher.Sum32(),
-		filesOut:  make(chan *File),
+		outputFiles: make(chan *File),
 	}
 
-	context.filtersExt = append(context.filtersExt, goldsmith.filters...)
+	context.filtersExternal = append(context.filtersExternal, goldsmith.filters...)
 
 	if len(goldsmith.contexts) > 0 {
-		context.filesIn = goldsmith.contexts[len(goldsmith.contexts)-1].filesOut
+		context.inputFiles = goldsmith.contexts[len(goldsmith.contexts)-1].outputFiles
 	}
 
 	goldsmith.contexts = append(goldsmith.contexts, context)
@@ -84,23 +89,66 @@ func (goldsmith *Goldsmith) FilterPop() *Goldsmith {
 func (goldsmith *Goldsmith) End(targetDir string) []error {
 	goldsmith.targetDir = targetDir
 
-	var wg sync.WaitGroup
-	goldsmith.Chain(&saver{
-		clean: goldsmith.clean,
-		wg:    &wg,
-	})
-
-	wg.Add(1)
-
 	for _, context := range goldsmith.contexts {
 		go context.step()
 	}
 
-	wg.Wait()
+	context := goldsmith.contexts[len(goldsmith.contexts)-1]
+	for file := range context.outputFiles {
+		if goldsmith.filters.accept(file) {
+			goldsmith.exportFile(file)
+		}
+	}
+
+	if goldsmith.clean {
+		goldsmith.removeUnreferencedFiles()
+	}
 
 	return goldsmith.errors
 }
 
+func (goldsmith *Goldsmith) retrieveFile(context *Context, outputPath string, inputFiles []*File) *File {
+	if goldsmith.fileCache != nil {
+		outputFile, _ := goldsmith.fileCache.retrieveFile(context, outputPath, inputFiles)
+		return outputFile
+	}
+
+	return nil
+}
+
+func (goldsmith *Goldsmith) storeFile(context *Context, outputFile *File, inputFiles []*File) {
+	if goldsmith.fileCache != nil {
+		goldsmith.fileCache.storeFile(context, outputFile, inputFiles)
+	}
+
+}
+
+func (goldsmith *Goldsmith) removeUnreferencedFiles() {
+	infos := make(chan fileInfo)
+	go scanDir(goldsmith.targetDir, infos)
+
+	for info := range infos {
+		if info.path != goldsmith.targetDir {
+			relPath, _ := filepath.Rel(goldsmith.targetDir, info.path)
+			if contained, _ := goldsmith.fileRefs[relPath]; !contained {
+				os.RemoveAll(info.path)
+			}
+		}
+	}
+}
+
+func (goldsmith *Goldsmith) exportFile(file *File) error {
+	if err := file.export(goldsmith.targetDir); err != nil {
+		return err
+	}
+
+	for pathSeg := cleanPath(file.sourcePath); pathSeg != "."; pathSeg = filepath.Dir(pathSeg) {
+		goldsmith.fileRefs[pathSeg] = true
+	}
+
+	return nil
+}
+
 func (goldsmith *Goldsmith) fault(name string, file *File, err error) {
 	goldsmith.mutex.Lock()
 	defer goldsmith.mutex.Unlock()

loader.go

@@ -2,22 +2,24 @@ package goldsmith
 
 import "path/filepath"
 
-type loader struct{}
+type loader struct {
+	Initializer
+}
 
 func (*loader) Name() string {
 	return "loader"
 }
 
-func (*loader) Initialize(context *Context) error {
+func (*loader) Initialize(ctx *Context) error {
 	infos := make(chan fileInfo)
-	go scanDir(context.goldsmith.sourceDir, infos)
+	go scanDir(ctx.goldsmith.sourceDir, infos)
 
 	for info := range infos {
 		if info.IsDir() {
 			continue
 		}
 
-		relPath, _ := filepath.Rel(context.goldsmith.sourceDir, info.path)
+		relPath, _ := filepath.Rel(ctx.goldsmith.sourceDir, info.path)
 
 		file := &File{
 			sourcePath: relPath,
@@ -27,7 +29,7 @@ func (*loader) Initialize(context *Context) error {
 			dataPath:   info.path,
 		}
 
-		context.DispatchFile(file)
+		ctx.DispatchFile(file)
 	}
 
 	return nil

saver.go

@@ -1,53 +0,0 @@
-package goldsmith
-
-import (
-	"os"
-	"path/filepath"
-	"sync"
-)
-
-type saver struct {
-	clean  bool
-	tokens map[string]bool
-	wg     *sync.WaitGroup
-}
-
-func (*saver) Name() string {
-	return "saver"
-}
-
-func (saver *saver) Initialize(context *Context) error {
-	saver.tokens = make(map[string]bool)
-	context.Threads(1)
-	return nil
-}
-
-func (saver *saver) Process(context *Context, file *File) error {
-	for token := cleanPath(file.sourcePath); token != "."; token = filepath.Dir(token) {
-		saver.tokens[token] = true
-	}
-
-	return file.export(context.goldsmith.targetDir)
-}
-
-func (saver *saver) Finalize(context *Context) error {
-	defer saver.wg.Done()
-
-	if !saver.clean {
-		return nil
-	}
-
-	infos := make(chan fileInfo)
-	go scanDir(context.goldsmith.targetDir, infos)
-
-	for info := range infos {
-		if info.path != context.goldsmith.targetDir {
-			relPath, _ := filepath.Rel(context.goldsmith.targetDir, info.path)
-			if contained, _ := saver.tokens[relPath]; !contained {
-				os.RemoveAll(info.path)
-			}
-		}
-	}
-
-	return nil
-}