This commit is contained in:
Alex Yatskov 2018-12-09 17:00:46 -08:00
parent c8e3e98432
commit 966201a95f
6 changed files with 149 additions and 149 deletions

View File

@ -9,12 +9,12 @@ import (
"sort"
)
type fileCache struct {
type cache struct {
baseDir string
}
func (c *fileCache) retrieveFile(context *Context, outputPath string, inputFiles []*File) (*File, error) {
cachePath, err := c.buildCachePath(context, outputPath, inputFiles)
func (cache *cache) retrieveFile(context *Context, outputPath string, inputFiles []*File) (*File, error) {
cachePath, err := cache.buildCachePath(context, outputPath, inputFiles)
if err != nil {
return nil, err
}
@ -31,13 +31,13 @@ func (c *fileCache) retrieveFile(context *Context, outputPath string, inputFiles
return outputFile, nil
}
func (c *fileCache) storeFile(context *Context, outputFile *File, inputFiles []*File) error {
cachePath, err := c.buildCachePath(context, outputFile.Path(), inputFiles)
func (cache *cache) storeFile(context *Context, outputFile *File, inputFiles []*File) error {
cachePath, err := cache.buildCachePath(context, outputFile.Path(), inputFiles)
if err != nil {
return err
}
if err := os.MkdirAll(c.baseDir, 0755); err != nil {
if err := os.MkdirAll(cache.baseDir, 0755); err != nil {
return err
}
@ -67,7 +67,7 @@ func (c *fileCache) storeFile(context *Context, outputFile *File, inputFiles []*
return nil
}
func (c *fileCache) buildCachePath(context *Context, outputPath string, inputFiles []*File) (string, error) {
func (cache *cache) buildCachePath(context *Context, outputPath string, inputFiles []*File) (string, error) {
uintBuff := make([]byte, 4)
binary.LittleEndian.PutUint32(uintBuff, context.hash)
@ -88,7 +88,7 @@ func (c *fileCache) buildCachePath(context *Context, outputPath string, inputFil
hasher.Write([]byte(inputFile.Path()))
}
cachePath := filepath.Join(c.baseDir, fmt.Sprintf(
cachePath := filepath.Join(cache.baseDir, fmt.Sprintf(
"gs_%.8x%s",
hasher.Sum32(),
filepath.Ext(outputPath),

View File

@ -50,52 +50,52 @@ func (*Context) CreateFileFromAsset(sourcePath, dataPath string) (*File, error)
return file, nil
}
func (ctx *Context) DispatchFile(file *File) {
ctx.outputFiles <- file
func (context *Context) DispatchFile(file *File) {
context.outputFiles <- file
}
func (ctx *Context) DispatchAndCacheFile(outputFile *File, inputFiles ...*File) {
ctx.goldsmith.storeFile(ctx, outputFile, inputFiles)
ctx.outputFiles <- outputFile
func (context *Context) DispatchAndCacheFile(outputFile *File, inputFiles ...*File) {
context.goldsmith.storeFile(context, outputFile, inputFiles)
context.outputFiles <- outputFile
}
func (ctx *Context) RetrieveCachedFile(outputPath string, inputFiles ...*File) *File {
return ctx.goldsmith.retrieveFile(ctx, outputPath, inputFiles)
func (context *Context) RetrieveCachedFile(outputPath string, inputFiles ...*File) *File {
return context.goldsmith.retrieveFile(context, outputPath, inputFiles)
}
func (ctx *Context) step() {
defer close(ctx.outputFiles)
func (context *Context) step() {
defer close(context.outputFiles)
var err error
var filter Filter
if initializer, ok := ctx.plugin.(Initializer); ok {
filter, err = initializer.Initialize(ctx)
if initializer, ok := context.plugin.(Initializer); ok {
filter, err = initializer.Initialize(context)
if err != nil {
ctx.goldsmith.fault(ctx.plugin.Name(), nil, err)
context.goldsmith.fault(context.plugin.Name(), nil, err)
return
}
}
if ctx.inputFiles != nil {
processor, _ := ctx.plugin.(Processor)
if context.inputFiles != nil {
processor, _ := context.plugin.(Processor)
var wg sync.WaitGroup
for i := 0; i < runtime.NumCPU(); i++ {
wg.Add(1)
go func() {
defer wg.Done()
for inputFile := range ctx.inputFiles {
for inputFile := range context.inputFiles {
accept := processor != nil
var fileFilters []Filter
fileFilters = append(fileFilters, ctx.fileFilters...)
fileFilters = append(fileFilters, context.fileFilters...)
if filter != nil {
fileFilters = append(fileFilters, filter)
}
for _, fileFilter := range fileFilters {
if accept, err = fileFilter.Accept(ctx, inputFile); err != nil {
ctx.goldsmith.fault(fileFilter.Name(), inputFile, err)
if accept, err = fileFilter.Accept(context, inputFile); err != nil {
context.goldsmith.fault(fileFilter.Name(), inputFile, err)
return
}
if !accept {
@ -105,13 +105,13 @@ func (ctx *Context) step() {
if accept {
if _, err := inputFile.Seek(0, os.SEEK_SET); err != nil {
ctx.goldsmith.fault("core", inputFile, err)
context.goldsmith.fault("core", inputFile, err)
}
if err := processor.Process(ctx, inputFile); err != nil {
ctx.goldsmith.fault(ctx.plugin.Name(), inputFile, err)
if err := processor.Process(context, inputFile); err != nil {
context.goldsmith.fault(context.plugin.Name(), inputFile, err)
}
} else {
ctx.outputFiles <- inputFile
context.outputFiles <- inputFile
}
}
}()
@ -119,9 +119,9 @@ func (ctx *Context) step() {
wg.Wait()
}
if finalizer, ok := ctx.plugin.(Finalizer); ok {
if err := finalizer.Finalize(ctx); err != nil {
ctx.goldsmith.fault(ctx.plugin.Name(), nil, err)
if finalizer, ok := context.plugin.(Finalizer); ok {
if err := finalizer.Finalize(context); err != nil {
context.goldsmith.fault(context.plugin.Name(), nil, err)
}
}
}

View File

@ -8,11 +8,11 @@ type Error struct {
Err error
}
func (e Error) Error() string {
func (err Error) Error() string {
var path string
if len(e.Path) > 0 {
path = "@" + e.Path
if len(err.Path) > 0 {
path = "@" + err.Path
}
return fmt.Sprintf("[%s%s]: %s", e.Name, path, e.Err.Error())
return fmt.Sprintf("[%s%s]: %s", err.Name, path, err.Err.Error())
}

116
file.go
View File

@ -26,75 +26,61 @@ type File struct {
modTime time.Time
}
func (f *File) Path() string {
return f.sourcePath
func (file *File) Path() string {
return file.sourcePath
}
func (f *File) Name() string {
return path.Base(f.sourcePath)
func (file *File) Name() string {
return path.Base(file.sourcePath)
}
func (f *File) Dir() string {
return path.Dir(f.sourcePath)
func (file *File) Dir() string {
return path.Dir(file.sourcePath)
}
func (f *File) Ext() string {
return path.Ext(f.sourcePath)
func (file *File) Ext() string {
return path.Ext(file.sourcePath)
}
func (f *File) Size() int64 {
return f.size
func (file *File) Size() int64 {
return file.size
}
func (f *File) ModTime() time.Time {
return f.modTime
func (file *File) ModTime() time.Time {
return file.modTime
}
func (f *File) Read(data []byte) (int, error) {
if err := f.load(); err != nil {
func (file *File) Read(data []byte) (int, error) {
if err := file.load(); err != nil {
return 0, err
}
return f.reader.Read(data)
return file.reader.Read(data)
}
func (f *File) WriteTo(writer io.Writer) (int64, error) {
if err := f.load(); err != nil {
func (file *File) WriteTo(writer io.Writer) (int64, error) {
if err := file.load(); err != nil {
return 0, err
}
return f.reader.WriteTo(writer)
return file.reader.WriteTo(writer)
}
func (f *File) Seek(offset int64, whence int) (int64, error) {
if f.reader == nil && offset == 0 && (whence == os.SEEK_SET || whence == os.SEEK_CUR) {
func (file *File) Seek(offset int64, whence int) (int64, error) {
if file.reader == nil && offset == 0 && (whence == os.SEEK_SET || whence == os.SEEK_CUR) {
return 0, nil
}
if err := f.load(); err != nil {
if err := file.load(); err != nil {
return 0, err
}
return f.reader.Seek(offset, whence)
return file.reader.Seek(offset, whence)
}
type FilesByPath []*File
func (f FilesByPath) Len() int {
return len(f)
}
func (f FilesByPath) Swap(i, j int) {
f[i], f[j] = f[j], f[i]
}
func (f FilesByPath) Less(i, j int) bool {
return strings.Compare(f[i].Path(), f[j].Path()) < 0
}
func (f *File) export(targetDir string) error {
targetPath := filepath.Join(targetDir, f.sourcePath)
if targetInfo, err := os.Stat(targetPath); err == nil && targetInfo.ModTime().After(f.ModTime()) {
func (file *File) export(targetDir string) error {
targetPath := filepath.Join(targetDir, file.sourcePath)
if targetInfo, err := os.Stat(targetPath); err == nil && targetInfo.ModTime().After(file.ModTime()) {
return nil
}
@ -108,8 +94,8 @@ func (f *File) export(targetDir string) error {
}
defer fw.Close()
if f.reader == nil {
fr, err := os.Open(f.dataPath)
if file.reader == nil {
fr, err := os.Open(file.dataPath)
if err != nil {
return err
}
@ -119,11 +105,11 @@ func (f *File) export(targetDir string) error {
return err
}
} else {
if _, err := f.Seek(0, os.SEEK_SET); err != nil {
if _, err := file.Seek(0, os.SEEK_SET); err != nil {
return err
}
if _, err := f.WriteTo(fw); err != nil {
if _, err := file.WriteTo(fw); err != nil {
return err
}
}
@ -131,50 +117,64 @@ func (f *File) export(targetDir string) error {
return nil
}
func (f *File) load() error {
if f.reader != nil {
func (file *File) load() error {
if file.reader != nil {
return nil
}
data, err := ioutil.ReadFile(f.dataPath)
data, err := ioutil.ReadFile(file.dataPath)
if err != nil {
return err
}
f.reader = bytes.NewReader(data)
file.reader = bytes.NewReader(data)
return nil
}
func (f *File) hash() (uint32, error) {
if f.hashValid {
return f.hashValue, nil
func (file *File) hash() (uint32, error) {
if file.hashValid {
return file.hashValue, nil
}
if err := f.load(); err != nil {
if err := file.load(); err != nil {
return 0, err
}
offset, err := f.Seek(0, os.SEEK_CUR)
offset, err := file.Seek(0, os.SEEK_CUR)
if err != nil {
return 0, err
}
if _, err := f.Seek(0, os.SEEK_SET); err != nil {
if _, err := file.Seek(0, os.SEEK_SET); err != nil {
return 0, err
}
hasher := crc32.NewIEEE()
if _, err := io.Copy(hasher, f.reader); err != nil {
if _, err := io.Copy(hasher, file.reader); err != nil {
return 0, err
}
if _, err := f.Seek(offset, os.SEEK_SET); err != nil {
if _, err := file.Seek(offset, os.SEEK_SET); err != nil {
return 0, err
}
f.hashValue = hasher.Sum32()
f.hashValid = true
return f.hashValue, nil
file.hashValue = hasher.Sum32()
file.hashValid = true
return file.hashValue, nil
}
type FilesByPath []*File
func (file FilesByPath) Len() int {
return len(file)
}
func (file FilesByPath) Swap(i, j int) {
file[i], file[j] = file[j], file[i]
}
func (file FilesByPath) Less(i, j int) bool {
return strings.Compare(file[i].Path(), file[j].Path()) < 0
}
type fileInfo struct {

View File

@ -17,128 +17,128 @@ type Goldsmith struct {
fileRefs map[string]bool
fileFilters []Filter
fileCache *fileCache
fileCache *cache
errors []error
errorMtx sync.Mutex
mutex sync.Mutex
}
func Begin(sourceDir string) *Goldsmith {
gs := &Goldsmith{
goldsmith := &Goldsmith{
sourceDir: sourceDir,
contextHasher: crc32.NewIEEE(),
fileRefs: make(map[string]bool),
}
gs.Chain(new(loader))
return gs
goldsmith.Chain(new(loader))
return goldsmith
}
func (gs *Goldsmith) Cache(cacheDir string) *Goldsmith {
gs.fileCache = &fileCache{cacheDir}
return gs
func (goldsmith *Goldsmith) Cache(cacheDir string) *Goldsmith {
goldsmith.fileCache = &cache{cacheDir}
return goldsmith
}
func (gs *Goldsmith) Chain(plugin Plugin) *Goldsmith {
gs.contextHasher.Write([]byte(plugin.Name()))
func (goldsmith *Goldsmith) Chain(plugin Plugin) *Goldsmith {
goldsmith.contextHasher.Write([]byte(plugin.Name()))
context := &Context{
goldsmith: gs,
goldsmith: goldsmith,
plugin: plugin,
hash: gs.contextHasher.Sum32(),
hash: goldsmith.contextHasher.Sum32(),
outputFiles: make(chan *File),
}
context.fileFilters = append(context.fileFilters, gs.fileFilters...)
context.fileFilters = append(context.fileFilters, goldsmith.fileFilters...)
if len(gs.contexts) > 0 {
context.inputFiles = gs.contexts[len(gs.contexts)-1].outputFiles
if len(goldsmith.contexts) > 0 {
context.inputFiles = goldsmith.contexts[len(goldsmith.contexts)-1].outputFiles
}
gs.contexts = append(gs.contexts, context)
return gs
goldsmith.contexts = append(goldsmith.contexts, context)
return goldsmith
}
func (gs *Goldsmith) FilterPush(filter Filter) *Goldsmith {
gs.fileFilters = append(gs.fileFilters, filter)
return gs
func (goldsmith *Goldsmith) FilterPush(filter Filter) *Goldsmith {
goldsmith.fileFilters = append(goldsmith.fileFilters, filter)
return goldsmith
}
func (gs *Goldsmith) FilterPop() *Goldsmith {
count := len(gs.fileFilters)
func (goldsmith *Goldsmith) FilterPop() *Goldsmith {
count := len(goldsmith.fileFilters)
if count == 0 {
panic("attempted to pop empty filter stack")
}
gs.fileFilters = gs.fileFilters[:count-1]
return gs
goldsmith.fileFilters = goldsmith.fileFilters[:count-1]
return goldsmith
}
func (gs *Goldsmith) End(targetDir string) []error {
gs.targetDir = targetDir
func (goldsmith *Goldsmith) End(targetDir string) []error {
goldsmith.targetDir = targetDir
for _, context := range gs.contexts {
for _, context := range goldsmith.contexts {
go context.step()
}
context := gs.contexts[len(gs.contexts)-1]
context := goldsmith.contexts[len(goldsmith.contexts)-1]
for file := range context.outputFiles {
gs.exportFile(file)
goldsmith.exportFile(file)
}
gs.removeUnreferencedFiles()
return gs.errors
goldsmith.removeUnreferencedFiles()
return goldsmith.errors
}
func (gs *Goldsmith) retrieveFile(context *Context, outputPath string, inputFiles []*File) *File {
if gs.fileCache != nil {
outputFile, _ := gs.fileCache.retrieveFile(context, outputPath, inputFiles)
func (goldsmith *Goldsmith) retrieveFile(context *Context, outputPath string, inputFiles []*File) *File {
if goldsmith.fileCache != nil {
outputFile, _ := goldsmith.fileCache.retrieveFile(context, outputPath, inputFiles)
return outputFile
}
return nil
}
func (gs *Goldsmith) storeFile(context *Context, outputFile *File, inputFiles []*File) {
if gs.fileCache != nil {
gs.fileCache.storeFile(context, outputFile, inputFiles)
func (goldsmith *Goldsmith) storeFile(context *Context, outputFile *File, inputFiles []*File) {
if goldsmith.fileCache != nil {
goldsmith.fileCache.storeFile(context, outputFile, inputFiles)
}
}
func (gs *Goldsmith) removeUnreferencedFiles() {
func (goldsmith *Goldsmith) removeUnreferencedFiles() {
infos := make(chan fileInfo)
go scanDir(gs.targetDir, infos)
go scanDir(goldsmith.targetDir, infos)
for info := range infos {
if info.path != gs.targetDir {
relPath, _ := filepath.Rel(gs.targetDir, info.path)
if contained, _ := gs.fileRefs[relPath]; !contained {
if info.path != goldsmith.targetDir {
relPath, _ := filepath.Rel(goldsmith.targetDir, info.path)
if contained, _ := goldsmith.fileRefs[relPath]; !contained {
os.RemoveAll(info.path)
}
}
}
}
func (gs *Goldsmith) exportFile(file *File) error {
if err := file.export(gs.targetDir); err != nil {
func (goldsmith *Goldsmith) exportFile(file *File) error {
if err := file.export(goldsmith.targetDir); err != nil {
return err
}
for pathSeg := cleanPath(file.sourcePath); pathSeg != "."; pathSeg = filepath.Dir(pathSeg) {
gs.fileRefs[pathSeg] = true
goldsmith.fileRefs[pathSeg] = true
}
return nil
}
func (gs *Goldsmith) fault(pluginName string, file *File, err error) {
gs.errorMtx.Lock()
defer gs.errorMtx.Unlock()
func (goldsmith *Goldsmith) fault(pluginName string, file *File, err error) {
goldsmith.mutex.Lock()
defer goldsmith.mutex.Unlock()
faultError := &Error{Name: pluginName, Err: err}
if file != nil {
faultError.Path = file.sourcePath
}
gs.errors = append(gs.errors, faultError)
goldsmith.errors = append(goldsmith.errors, faultError)
}

View File

@ -1,15 +1,15 @@
package goldsmith
type Initializer interface {
Initialize(ctx *Context) (Filter, error)
Initialize(context *Context) (Filter, error)
}
type Processor interface {
Process(ctx *Context, file *File) error
Process(context *Context, file *File) error
}
type Finalizer interface {
Finalize(ctx *Context) error
Finalize(context *Context) error
}
type Component interface {
@ -18,7 +18,7 @@ type Component interface {
type Filter interface {
Component
Accept(ctx *Context, file *File) (bool, error)
Accept(context *Context, file *File) (bool, error)
}
type Plugin interface {