From 0337ec8cbc294f924c3f9b793da918473c501eee Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Wed, 23 Dec 2015 16:44:55 +0900 Subject: [PATCH 01/15] Changing error def --- goldsmith.go | 6 +++++- types.go | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/goldsmith.go b/goldsmith.go index 98f0962..ab22320 100644 --- a/goldsmith.go +++ b/goldsmith.go @@ -126,7 +126,11 @@ func (gs *goldsmith) referenceFile(path string) { func (gs *goldsmith) fault(f *file, err error) { gs.errorMtx.Lock() - gs.errors = append(gs.errors, &Error{f, err}) + ferr := &Error{err: err} + if f != nil { + ferr.path = f.path + } + gs.errors = append(gs.errors, ferr) gs.errorMtx.Unlock() } diff --git a/types.go b/types.go index 2e2bcfe..1a59c31 100644 --- a/types.go +++ b/types.go @@ -79,7 +79,7 @@ type Context interface { } type Error struct { - file File + path string err error } From b0041d3b0878f0cdfdf4b2e7e3b081eefd9e8868 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Wed, 23 Dec 2015 16:52:54 +0900 Subject: [PATCH 02/15] Error fixup --- goldsmith.go | 4 ++-- types.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/goldsmith.go b/goldsmith.go index ab22320..b744a37 100644 --- a/goldsmith.go +++ b/goldsmith.go @@ -126,9 +126,9 @@ func (gs *goldsmith) referenceFile(path string) { func (gs *goldsmith) fault(f *file, err error) { gs.errorMtx.Lock() - ferr := &Error{err: err} + ferr := &Error{Err: err} if f != nil { - ferr.path = f.path + ferr.Path = f.path } gs.errors = append(gs.errors, ferr) gs.errorMtx.Unlock() diff --git a/types.go b/types.go index 1a59c31..e6eacd8 100644 --- a/types.go +++ b/types.go @@ -79,12 +79,12 @@ type Context interface { } type Error struct { - path string - err error + Err error + Path string } -func (e *Error) Error() string { - return e.err.Error() +func (e Error) Error() string { + return e.Err.Error() } type Initializer interface { From 24db16ccf7bca97223e6d6246e0950b410d1d3ab Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Mon, 28 Dec 2015 21:21:13 +0900 Subject: [PATCH 03/15] Cleanup --- types.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/types.go b/types.go index e6eacd8..0a3ad02 100644 --- a/types.go +++ b/types.go @@ -92,15 +92,15 @@ type Initializer interface { } type Accepter interface { - Accept(ctx Context, file File) bool -} - -type Finalizer interface { - Finalize(ctx Context) error + Accept(ctx Context, f File) bool } type Processor interface { Process(ctx Context, f File) error } +type Finalizer interface { + Finalize(ctx Context) error +} + type Plugin interface{} From 23ba57e6a81d74e6629443570f9dcae8b8327c11 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Tue, 29 Dec 2015 21:08:41 +0900 Subject: [PATCH 04/15] Hiding the Meta parameter --- file.go | 18 ++++++++++++------ types.go | 9 +++++---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/file.go b/file.go index 463f8d7..719fd28 100644 --- a/file.go +++ b/file.go @@ -32,7 +32,7 @@ import ( type file struct { path string - meta map[string]interface{} + Meta map[string]interface{} reader *bytes.Reader asset string @@ -93,13 +93,19 @@ func (f *file) Path() string { return f.path } -func (f *file) Meta() map[string]interface{} { - return f.meta +func (f *file) Value(key string) (interface{}, bool) { + value, ok := f.Meta[key] + return value, ok } -func (f *file) Apply(m map[string]interface{}) { - for key, value := range m { - f.meta[key] = value +func (f *file) SetValue(key string, value interface{}) { + f.Meta[key] = value +} + +func (f *file) CopyValues(src File) { + rf := src.(*file) + for name, value := range rf.Meta { + f.SetValue(name, value) } } diff --git a/types.go b/types.go index 0a3ad02..2451a9a 100644 --- a/types.go +++ b/types.go @@ -46,8 +46,9 @@ func New(srcDir, dstDir string) Goldsmith { type File interface { Path() string - Meta() map[string]interface{} - Apply(m map[string]interface{}) + Value(key string) (interface{}, bool) + SetValue(key string, value interface{}) + CopyValues(src File) Read(p []byte) (int, error) WriteTo(w io.Writer) (int64, error) @@ -57,7 +58,7 @@ type File interface { func NewFileFromData(path string, data []byte) File { return &file{ path: path, - meta: make(map[string]interface{}), + Meta: make(map[string]interface{}), reader: bytes.NewReader(data), } } @@ -65,7 +66,7 @@ func NewFileFromData(path string, data []byte) File { func NewFileFromAsset(path, asset string) File { return &file{ path: path, - meta: make(map[string]interface{}), + Meta: make(map[string]interface{}), asset: asset, } } From 04ab83ae22101fde2a4ae395e9c594c7a2ddf2b8 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Wed, 30 Dec 2015 11:06:29 +0900 Subject: [PATCH 05/15] Minor API update --- file.go | 6 +++--- types.go | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/file.go b/file.go index 719fd28..9491770 100644 --- a/file.go +++ b/file.go @@ -32,7 +32,7 @@ import ( type file struct { path string - Meta map[string]interface{} + Meta map[string]Meta reader *bytes.Reader asset string @@ -93,12 +93,12 @@ func (f *file) Path() string { return f.path } -func (f *file) Value(key string) (interface{}, bool) { +func (f *file) Value(key string) (Meta, bool) { value, ok := f.Meta[key] return value, ok } -func (f *file) SetValue(key string, value interface{}) { +func (f *file) SetValue(key string, value Meta) { f.Meta[key] = value } diff --git a/types.go b/types.go index 2451a9a..a66bb1e 100644 --- a/types.go +++ b/types.go @@ -46,8 +46,8 @@ func New(srcDir, dstDir string) Goldsmith { type File interface { Path() string - Value(key string) (interface{}, bool) - SetValue(key string, value interface{}) + Value(key string) (Meta, bool) + SetValue(key string, value Meta) CopyValues(src File) Read(p []byte) (int, error) @@ -58,7 +58,7 @@ type File interface { func NewFileFromData(path string, data []byte) File { return &file{ path: path, - Meta: make(map[string]interface{}), + Meta: make(map[string]Meta), reader: bytes.NewReader(data), } } @@ -66,7 +66,7 @@ func NewFileFromData(path string, data []byte) File { func NewFileFromAsset(path, asset string) File { return &file{ path: path, - Meta: make(map[string]interface{}), + Meta: make(map[string]Meta), asset: asset, } } @@ -105,3 +105,5 @@ type Finalizer interface { } type Plugin interface{} + +type Meta interface{} From bf2bc2c180f6784e1a327ffcaafb136b92da94d8 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Fri, 1 Jan 2016 17:47:28 +0900 Subject: [PATCH 06/15] Removing Meta interface --- file.go | 6 +++--- types.go | 10 ++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/file.go b/file.go index 9491770..719fd28 100644 --- a/file.go +++ b/file.go @@ -32,7 +32,7 @@ import ( type file struct { path string - Meta map[string]Meta + Meta map[string]interface{} reader *bytes.Reader asset string @@ -93,12 +93,12 @@ func (f *file) Path() string { return f.path } -func (f *file) Value(key string) (Meta, bool) { +func (f *file) Value(key string) (interface{}, bool) { value, ok := f.Meta[key] return value, ok } -func (f *file) SetValue(key string, value Meta) { +func (f *file) SetValue(key string, value interface{}) { f.Meta[key] = value } diff --git a/types.go b/types.go index a66bb1e..2451a9a 100644 --- a/types.go +++ b/types.go @@ -46,8 +46,8 @@ func New(srcDir, dstDir string) Goldsmith { type File interface { Path() string - Value(key string) (Meta, bool) - SetValue(key string, value Meta) + Value(key string) (interface{}, bool) + SetValue(key string, value interface{}) CopyValues(src File) Read(p []byte) (int, error) @@ -58,7 +58,7 @@ type File interface { func NewFileFromData(path string, data []byte) File { return &file{ path: path, - Meta: make(map[string]Meta), + Meta: make(map[string]interface{}), reader: bytes.NewReader(data), } } @@ -66,7 +66,7 @@ func NewFileFromData(path string, data []byte) File { func NewFileFromAsset(path, asset string) File { return &file{ path: path, - Meta: make(map[string]Meta), + Meta: make(map[string]interface{}), asset: asset, } } @@ -105,5 +105,3 @@ type Finalizer interface { } type Plugin interface{} - -type Meta interface{} From da057ec5442880f7b359b43bbd7fdcc7cdfe1619 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Sun, 10 Jan 2016 20:17:35 +0900 Subject: [PATCH 07/15] Updating API --- context.go | 57 +++++++++++++++++++++++----------------------------- goldsmith.go | 35 ++++++++++++++------------------ loader.go | 44 ++++++++++++++++++++++++++++++++++++++++ types.go | 13 ++++-------- 4 files changed, 88 insertions(+), 61 deletions(-) create mode 100644 loader.go diff --git a/context.go b/context.go index 0e8bd3c..f85eec5 100644 --- a/context.go +++ b/context.go @@ -30,26 +30,17 @@ import ( type context struct { gs *goldsmith + plug Plugin input, output chan *file } -func newContext(gs *goldsmith) *context { - ctx := &context{gs: gs, output: make(chan *file)} - if len(gs.contexts) > 0 { - ctx.input = gs.contexts[len(gs.contexts)-1].output - } - - gs.contexts = append(gs.contexts, ctx) - return ctx -} - -func (ctx *context) chain(p Plugin) { +func (ctx *context) chain() { defer close(ctx.output) - init, _ := p.(Initializer) - accept, _ := p.(Accepter) - proc, _ := p.(Processor) - fin, _ := p.(Finalizer) + init, _ := ctx.plug.(Initializer) + accept, _ := ctx.plug.(Accepter) + proc, _ := ctx.plug.(Processor) + fin, _ := ctx.plug.(Finalizer) if init != nil { if err := init.Initialize(ctx); err != nil { @@ -58,26 +49,28 @@ func (ctx *context) chain(p Plugin) { } } - var wg sync.WaitGroup - for i := 0; i < runtime.NumCPU(); i++ { - wg.Add(1) - go func() { - defer wg.Done() - for f := range ctx.input { - if proc == nil || accept != nil && !accept.Accept(ctx, f) { - ctx.output <- f - } else { - if _, err := f.Seek(0, os.SEEK_SET); err != nil { - ctx.gs.fault(f, err) - } - if err := proc.Process(ctx, f); err != nil { - ctx.gs.fault(f, err) + if ctx.input != nil { + var wg sync.WaitGroup + for i := 0; i < runtime.NumCPU(); i++ { + wg.Add(1) + go func() { + defer wg.Done() + for f := range ctx.input { + if proc == nil || accept != nil && !accept.Accept(ctx, f) { + ctx.output <- f + } else { + if _, err := f.Seek(0, os.SEEK_SET); err != nil { + ctx.gs.fault(f, err) + } + if err := proc.Process(ctx, f); err != nil { + ctx.gs.fault(f, err) + } } } - } - }() + }() + } + wg.Wait() } - wg.Wait() if fin != nil { if err := fin.Finalize(ctx); err != nil { diff --git a/goldsmith.go b/goldsmith.go index b744a37..6af2130 100644 --- a/goldsmith.go +++ b/goldsmith.go @@ -39,24 +39,14 @@ type goldsmith struct { errorMtx sync.Mutex } -func (gs *goldsmith) queueFiles() { - files := make(chan string) - go scanDir(gs.srcDir, files, nil) +func (gs *goldsmith) pushContext(plug Plugin) *context { + ctx := &context{gs: gs, plug: plug, output: make(chan *file)} + if len(gs.contexts) > 0 { + ctx.input = gs.contexts[len(gs.contexts)-1].output + } - ctx := newContext(gs) - - go func() { - defer close(ctx.output) - for path := range files { - relPath, err := filepath.Rel(gs.srcDir, path) - if err != nil { - panic(err) - } - - f := NewFileFromAsset(relPath, path) - ctx.DispatchFile(f) - } - }() + gs.contexts = append(gs.contexts, ctx) + return ctx } func (gs *goldsmith) cleanupFiles() { @@ -139,12 +129,17 @@ func (gs *goldsmith) fault(f *file, err error) { // func (gs *goldsmith) Chain(p Plugin) Goldsmith { - ctx := newContext(gs) - go ctx.chain(p) + gs.pushContext(p) return gs } -func (gs *goldsmith) Complete() []error { +func (gs *goldsmith) End(dstDir string) []error { + gs.dstDir = dstDir + + for _, ctx := range gs.contexts { + go ctx.chain() + } + ctx := gs.contexts[len(gs.contexts)-1] for f := range ctx.output { gs.exportFile(f) diff --git a/loader.go b/loader.go new file mode 100644 index 0000000..cbf0204 --- /dev/null +++ b/loader.go @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016 Alex Yatskov + * Author: Alex Yatskov + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package goldsmith + +import "path/filepath" + +type loader struct{} + +func (*loader) Initialize(ctx Context) error { + files := make(chan string) + go scanDir(ctx.SrcDir(), files, nil) + + for path := range files { + relPath, err := filepath.Rel(ctx.SrcDir(), path) + if err != nil { + return err + } + + f := NewFileFromAsset(relPath, path) + ctx.DispatchFile(f) + } + + return nil +} diff --git a/types.go b/types.go index 2451a9a..7d702c2 100644 --- a/types.go +++ b/types.go @@ -29,17 +29,12 @@ import ( type Goldsmith interface { Chain(p Plugin) Goldsmith - Complete() []error + End(dstDir string) []error } -func New(srcDir, dstDir string) Goldsmith { - gs := &goldsmith{ - srcDir: srcDir, - dstDir: dstDir, - refs: make(map[string]bool), - } - - gs.queueFiles() +func Begin(srcDir string) Goldsmith { + gs := &goldsmith{srcDir: srcDir, refs: make(map[string]bool)} + gs.Chain(new(loader)) return gs } From 31beb8b2c02f03ba4b31d9e5696e575ce84f0d42 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Tue, 12 Jan 2016 17:04:41 +0900 Subject: [PATCH 08/15] Don't copy unmodified files from source to dest --- file.go | 4 ++++ util.go | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/file.go b/file.go index 719fd28..60c024b 100644 --- a/file.go +++ b/file.go @@ -39,6 +39,10 @@ type file struct { } func (f *file) export(dstPath string) error { + if f.reader == nil && fileCached(f.asset, dstPath) { + return nil + } + if err := os.MkdirAll(path.Dir(dstPath), 0755); err != nil { return err } diff --git a/util.go b/util.go index 12cbd58..18a032d 100644 --- a/util.go +++ b/util.go @@ -66,3 +66,17 @@ func scanDir(root string, files, dirs chan string) { return nil }) } + +func fileCached(srcPath, dstPath string) bool { + srcStat, err := os.Stat(srcPath) + if err != nil { + return false + } + + dstStat, err := os.Stat(dstPath) + if err != nil { + return false + } + + return dstStat.ModTime().Unix() >= srcStat.ModTime().Unix() +} From ebcde65f7fa613122d48975c0f3f68a39fd02317 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Tue, 12 Jan 2016 17:32:06 +0900 Subject: [PATCH 09/15] Optimization --- file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/file.go b/file.go index 60c024b..d7af136 100644 --- a/file.go +++ b/file.go @@ -39,7 +39,7 @@ type file struct { } func (f *file) export(dstPath string) error { - if f.reader == nil && fileCached(f.asset, dstPath) { + if len(f.asset) > 0 && fileCached(f.asset, dstPath) { return nil } From 16d02310e69449e9833604224baa81655a5035e4 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Tue, 12 Jan 2016 19:24:19 +0900 Subject: [PATCH 10/15] Adding README --- README.md | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..89f3e1a --- /dev/null +++ b/README.md @@ -0,0 +1,129 @@ ++++ +area = "projects" +github = "goldsmith" +tags = ["generator", "golang", "goldsmith", "mit license", "web"] +title = "Goldsmith" ++++ + +Goldsmith is a static website generator developed in Golang with flexibility, extensibility, and performance as primary +design considerations. With Goldsmith you can easily build and deploy any type of site, whether it is a personal blog, +image gallery, or a corporate homepage; the tool no assumptions are made about your layout or file structure. Goldsmith +is trivially extensible via a plugin architecture which makes it simple to perform complex data transformations +concurrently. A growing set of core plugins, [Goldsmith-Plugins](../goldsmith-plugins/), is provided to make it easier +to get started with this tool to generate static websites. + +## Motivation ## + +Why in the world would one create yet another static site generator? At first, I didn't think I needed to; after all, +there is a wide variety of open source tools freely available for use. Surely one of these applications would allow me +to build my portfolio page exactly the way I want right? + +After trying several static generators, namely [Pelican](http://blog.getpelican.com/), [Hexo](https://hexo.io/), and +[Hugo](https://gohugo.io/), I found that although sometimes coming close, no tool gave me exactly what I needed. +Although I hold the authors of these applications in high regard and sincerely appreciate their contribution to the open +source community, everyone seemed overly eager to make assumptions about content organization and presentation. + +Many of the static generators I've used feature extensive configuration files to support customization. Nevertheless, I +was disappointed to discovered that even though I could approach my planned design, I could never realize it. There was +always some architectural limitation preventing me from doing something with my site which seemed like basic +functionality. + +* Blog posts can be tagged, but static pages cannot. +* Image files cannot be stored next to content files. +* Navbar item activated when viewing blog, but not static pages. +* Auto-generated pages behave differently from normal ones. + +Upon asking on community forms, I learned that most users were content to live with such design decisions, with some +offering workarounds that would get me halfway to where I wanted to go. As I am not one to make compromises, I kept +hopping from one static site generator to another, until I discovered [Metalsmith](http://www.metalsmith.io/). Finally, +it seemed like I found a tool that gets out of my way, and lets me build my website the way I want to. After using this +tool for almost a year, I began to see its limits. + +* The extension system is complicated; it's difficult to write and debug plugins. +* Quality of existing plugins varies greatly; I found many subtle issues. +* No support for parallel processing (this is a big one if you process images). +* A full [Node.js](https://nodejs.org/) is stack (including dependencies) is required to build sites. + +Rather than making do with what I had indefinitely, I decided to use the knowledge I've obtained from using various +static site generators to build my own. The *Goldsmith* name is a reference to both the *Go* programming language I've +selected for this project, as well as to *Metalsmith*, my inspiration for what an static site generator could be. + +The motivation behind Goldsmith can be described by the following principles: + +* Keep the core small and simple. +* Enable efficient, multi-core processing. +* Add new features via user plugins. +* Customize behavior through user code. + +I originally built this tool to generate my personal homepage, but I believe it can be of use to anyone who wants to +enjoy the freedom of building a static site from ground up, especially users of Metalsmith. Why craft metal when you can +be crafting gold? + +## Usage ## + +Goldsmith is at it's core, a pipeline-based file processor. Files are loaded from the source directory, processed by any +number of plugins, and are finally output to the destination directory. Rather than explaining the process in detail +conceptually, I will show some code samples which show how this tool can be used in practice. + +* Start by copying files from a source directory to a destination directory (simplest possible use case): + ``` + goldsmith.Begin(srcDir). + End(dstDir) + ``` + +* Now let's also convert our [Markdown](https://daringfireball.net/projects/markdown/) files to HTML using the + [markdown plugin](../goldsmith-plugins/markdown): + ``` + goldsmith.Begin(srcDir). + Chain(markdown.NewCommon()). + End(dstDir) + ``` + +* If we are using *front matter* in our Markdown files, we can easily extract it by using the + [frontmatter plugin](../goldsmith-plugins/frontmatter): + ``` + goldsmith.Begin(srcDir). + Chain(frontmatter.New()). + Chain(markdown.NewCommon()). + End(dstDir) + ``` + +* Next we want to run our generated HTML through a template to add a header, footer, and a menu; for this we + can use the [layout plugin](../goldsmith-plugins/layout): + ``` + goldsmith.Begin(srcDir). + Chain(frontmatter.New()). + Chain(markdown.NewCommon()). + Chain(layout.New( + layoutFiles, // array of paths for files containing template definitions + templateNameVar, // metadata variable that contains the name of the template to use + contentStoreVar, // metadata variable configured in template to insert content + defTemplateName, // name of a default template to use if one is not specified + userFuncs, // mapping of functions which can be executed from templates + )). + End(dstDir) + ``` + +* Finally, let's [minify](https://en.wikipedia.org/wiki/Minification_(programming)) our files to reduce data transfer + and load times for our site's visitors using the [minify plugin](../goldsmith-plugins/minify). + ``` + goldsmith.Begin(srcDir). + Chain(frontmatter.New()). + Chain(markdown.NewCommon()). + Chain(layout.New(layoutFiles, templateNameVar, contentStoreVar, defTemplateName, userFuncs)). + Chain(minify.New()). + End(dstDir) + ``` + +I hope this simple example effectively illustrates the conceptual simplicity of the Goldsmith pipeline-based processing +method. Files are injected into the stream at Goldsmith initialization, processed in parallel through a series of +plugins, and are finally written out to disk upon completion. + +Files are guaranteed to flow through Goldsmith plugins in the same order, but not necessarily in the same sequence +relative to each other. Timing differences can cause certain files to finish ahead of others; fortunately this, along +with other threading characteristics of the tool is abstracted from the user. The execution, while appearing to be a +mere series chained methods, will process files using all of your system's cores. + +## License ## + +MIT From bbd32525416bf0a04f52b5025034a6235e96a566 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Wed, 13 Jan 2016 12:21:30 +0900 Subject: [PATCH 11/15] Renaming files --- core.go | 150 ++++++++++++++++++++++++++++++++++++++++++ goldsmith.go | 182 +++++++++++++++++++-------------------------------- types.go | 102 ----------------------------- 3 files changed, 217 insertions(+), 217 deletions(-) create mode 100644 core.go delete mode 100644 types.go diff --git a/core.go b/core.go new file mode 100644 index 0000000..6af2130 --- /dev/null +++ b/core.go @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2015 Alex Yatskov + * Author: Alex Yatskov + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package goldsmith + +import ( + "os" + "path/filepath" + "sync" +) + +type goldsmith struct { + srcDir, dstDir string + contexts []*context + + refs map[string]bool + refMtx sync.Mutex + + errors []error + errorMtx sync.Mutex +} + +func (gs *goldsmith) pushContext(plug Plugin) *context { + ctx := &context{gs: gs, plug: plug, output: make(chan *file)} + if len(gs.contexts) > 0 { + ctx.input = gs.contexts[len(gs.contexts)-1].output + } + + gs.contexts = append(gs.contexts, ctx) + return ctx +} + +func (gs *goldsmith) cleanupFiles() { + files := make(chan string) + dirs := make(chan string) + go scanDir(gs.dstDir, files, dirs) + + for files != nil || dirs != nil { + var ( + path string + ok bool + ) + + select { + case path, ok = <-files: + if !ok { + files = nil + continue + } + case path, ok = <-dirs: + if !ok { + dirs = nil + continue + } + default: + continue + } + + relPath, err := filepath.Rel(gs.dstDir, path) + if err != nil { + panic(err) + } + + if contained, _ := gs.refs[relPath]; contained { + continue + } + + os.RemoveAll(path) + } +} + +func (gs *goldsmith) exportFile(f *file) error { + absPath := filepath.Join(gs.dstDir, f.path) + if err := f.export(absPath); err != nil { + return err + } + + gs.referenceFile(f.path) + return nil +} + +func (gs *goldsmith) referenceFile(path string) { + gs.refMtx.Lock() + defer gs.refMtx.Unlock() + + path = cleanPath(path) + + for { + gs.refs[path] = true + if path == "." { + break + } + + path = filepath.Dir(path) + } +} + +func (gs *goldsmith) fault(f *file, err error) { + gs.errorMtx.Lock() + ferr := &Error{Err: err} + if f != nil { + ferr.Path = f.path + } + gs.errors = append(gs.errors, ferr) + gs.errorMtx.Unlock() +} + +// +// Goldsmith Implementation +// + +func (gs *goldsmith) Chain(p Plugin) Goldsmith { + gs.pushContext(p) + return gs +} + +func (gs *goldsmith) End(dstDir string) []error { + gs.dstDir = dstDir + + for _, ctx := range gs.contexts { + go ctx.chain() + } + + ctx := gs.contexts[len(gs.contexts)-1] + for f := range ctx.output { + gs.exportFile(f) + } + + gs.cleanupFiles() + return gs.errors +} diff --git a/goldsmith.go b/goldsmith.go index 6af2130..7d702c2 100644 --- a/goldsmith.go +++ b/goldsmith.go @@ -23,128 +23,80 @@ package goldsmith import ( - "os" - "path/filepath" - "sync" + "bytes" + "io" ) -type goldsmith struct { - srcDir, dstDir string - contexts []*context - - refs map[string]bool - refMtx sync.Mutex - - errors []error - errorMtx sync.Mutex +type Goldsmith interface { + Chain(p Plugin) Goldsmith + End(dstDir string) []error } -func (gs *goldsmith) pushContext(plug Plugin) *context { - ctx := &context{gs: gs, plug: plug, output: make(chan *file)} - if len(gs.contexts) > 0 { - ctx.input = gs.contexts[len(gs.contexts)-1].output - } - - gs.contexts = append(gs.contexts, ctx) - return ctx -} - -func (gs *goldsmith) cleanupFiles() { - files := make(chan string) - dirs := make(chan string) - go scanDir(gs.dstDir, files, dirs) - - for files != nil || dirs != nil { - var ( - path string - ok bool - ) - - select { - case path, ok = <-files: - if !ok { - files = nil - continue - } - case path, ok = <-dirs: - if !ok { - dirs = nil - continue - } - default: - continue - } - - relPath, err := filepath.Rel(gs.dstDir, path) - if err != nil { - panic(err) - } - - if contained, _ := gs.refs[relPath]; contained { - continue - } - - os.RemoveAll(path) - } -} - -func (gs *goldsmith) exportFile(f *file) error { - absPath := filepath.Join(gs.dstDir, f.path) - if err := f.export(absPath); err != nil { - return err - } - - gs.referenceFile(f.path) - return nil -} - -func (gs *goldsmith) referenceFile(path string) { - gs.refMtx.Lock() - defer gs.refMtx.Unlock() - - path = cleanPath(path) - - for { - gs.refs[path] = true - if path == "." { - break - } - - path = filepath.Dir(path) - } -} - -func (gs *goldsmith) fault(f *file, err error) { - gs.errorMtx.Lock() - ferr := &Error{Err: err} - if f != nil { - ferr.Path = f.path - } - gs.errors = append(gs.errors, ferr) - gs.errorMtx.Unlock() -} - -// -// Goldsmith Implementation -// - -func (gs *goldsmith) Chain(p Plugin) Goldsmith { - gs.pushContext(p) +func Begin(srcDir string) Goldsmith { + gs := &goldsmith{srcDir: srcDir, refs: make(map[string]bool)} + gs.Chain(new(loader)) return gs } -func (gs *goldsmith) End(dstDir string) []error { - gs.dstDir = dstDir +type File interface { + Path() string - for _, ctx := range gs.contexts { - go ctx.chain() - } + Value(key string) (interface{}, bool) + SetValue(key string, value interface{}) + CopyValues(src File) - ctx := gs.contexts[len(gs.contexts)-1] - for f := range ctx.output { - gs.exportFile(f) - } - - gs.cleanupFiles() - return gs.errors + Read(p []byte) (int, error) + WriteTo(w io.Writer) (int64, error) + Seek(offset int64, whence int) (int64, error) } + +func NewFileFromData(path string, data []byte) File { + return &file{ + path: path, + Meta: make(map[string]interface{}), + reader: bytes.NewReader(data), + } +} + +func NewFileFromAsset(path, asset string) File { + return &file{ + path: path, + Meta: make(map[string]interface{}), + asset: asset, + } +} + +type Context interface { + DispatchFile(f File) + ReferenceFile(path string) + + SrcDir() string + DstDir() string +} + +type Error struct { + Err error + Path string +} + +func (e Error) Error() string { + return e.Err.Error() +} + +type Initializer interface { + Initialize(ctx Context) error +} + +type Accepter interface { + Accept(ctx Context, f File) bool +} + +type Processor interface { + Process(ctx Context, f File) error +} + +type Finalizer interface { + Finalize(ctx Context) error +} + +type Plugin interface{} diff --git a/types.go b/types.go deleted file mode 100644 index 7d702c2..0000000 --- a/types.go +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (c) 2015 Alex Yatskov - * Author: Alex Yatskov - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to - * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of - * the Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package goldsmith - -import ( - "bytes" - "io" -) - -type Goldsmith interface { - Chain(p Plugin) Goldsmith - End(dstDir string) []error -} - -func Begin(srcDir string) Goldsmith { - gs := &goldsmith{srcDir: srcDir, refs: make(map[string]bool)} - gs.Chain(new(loader)) - return gs -} - -type File interface { - Path() string - - Value(key string) (interface{}, bool) - SetValue(key string, value interface{}) - CopyValues(src File) - - Read(p []byte) (int, error) - WriteTo(w io.Writer) (int64, error) - Seek(offset int64, whence int) (int64, error) -} - -func NewFileFromData(path string, data []byte) File { - return &file{ - path: path, - Meta: make(map[string]interface{}), - reader: bytes.NewReader(data), - } -} - -func NewFileFromAsset(path, asset string) File { - return &file{ - path: path, - Meta: make(map[string]interface{}), - asset: asset, - } -} - -type Context interface { - DispatchFile(f File) - ReferenceFile(path string) - - SrcDir() string - DstDir() string -} - -type Error struct { - Err error - Path string -} - -func (e Error) Error() string { - return e.Err.Error() -} - -type Initializer interface { - Initialize(ctx Context) error -} - -type Accepter interface { - Accept(ctx Context, f File) bool -} - -type Processor interface { - Process(ctx Context, f File) error -} - -type Finalizer interface { - Finalize(ctx Context) error -} - -type Plugin interface{} From 95079bca8e8fbf34abc19e81ab9ed8df4ddf7c26 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Wed, 13 Jan 2016 12:52:45 +0900 Subject: [PATCH 12/15] Cleanup --- core.go | 17 ++++++++++------- loader.go | 6 +----- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/core.go b/core.go index 6af2130..198986c 100644 --- a/core.go +++ b/core.go @@ -40,7 +40,12 @@ type goldsmith struct { } func (gs *goldsmith) pushContext(plug Plugin) *context { - ctx := &context{gs: gs, plug: plug, output: make(chan *file)} + ctx := &context{ + gs: gs, + plug: plug, + output: make(chan *file), + } + if len(gs.contexts) > 0 { ctx.input = gs.contexts[len(gs.contexts)-1].output } @@ -75,11 +80,7 @@ func (gs *goldsmith) cleanupFiles() { continue } - relPath, err := filepath.Rel(gs.dstDir, path) - if err != nil { - panic(err) - } - + relPath, _ := filepath.Rel(gs.dstDir, path) if contained, _ := gs.refs[relPath]; contained { continue } @@ -116,12 +117,14 @@ func (gs *goldsmith) referenceFile(path string) { func (gs *goldsmith) fault(f *file, err error) { gs.errorMtx.Lock() + defer gs.errorMtx.Unlock() + ferr := &Error{Err: err} if f != nil { ferr.Path = f.path } + gs.errors = append(gs.errors, ferr) - gs.errorMtx.Unlock() } // diff --git a/loader.go b/loader.go index cbf0204..1bc8113 100644 --- a/loader.go +++ b/loader.go @@ -31,11 +31,7 @@ func (*loader) Initialize(ctx Context) error { go scanDir(ctx.SrcDir(), files, nil) for path := range files { - relPath, err := filepath.Rel(ctx.SrcDir(), path) - if err != nil { - return err - } - + relPath, _ := filepath.Rel(ctx.SrcDir(), path) f := NewFileFromAsset(relPath, path) ctx.DispatchFile(f) } From beb43687df83ed3d3c7624e6b65eea93abd90b3b Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Wed, 13 Jan 2016 13:12:50 +0900 Subject: [PATCH 13/15] Cleanup plugin interface --- context.go | 4 ---- core.go | 26 ++++++++------------------ file.go | 4 +++- goldsmith.go | 1 - 4 files changed, 11 insertions(+), 24 deletions(-) diff --git a/context.go b/context.go index f85eec5..2e7879c 100644 --- a/context.go +++ b/context.go @@ -87,10 +87,6 @@ func (ctx *context) DispatchFile(f File) { ctx.output <- f.(*file) } -func (ctx *context) ReferenceFile(path string) { - ctx.gs.referenceFile(path) -} - func (ctx *context) SrcDir() string { return ctx.gs.srcDir } diff --git a/core.go b/core.go index 198986c..9288239 100644 --- a/core.go +++ b/core.go @@ -31,9 +31,7 @@ import ( type goldsmith struct { srcDir, dstDir string contexts []*context - - refs map[string]bool - refMtx sync.Mutex + refs map[string]bool errors []error errorMtx sync.Mutex @@ -90,29 +88,21 @@ func (gs *goldsmith) cleanupFiles() { } func (gs *goldsmith) exportFile(f *file) error { - absPath := filepath.Join(gs.dstDir, f.path) - if err := f.export(absPath); err != nil { + if err := f.export(gs.dstDir); err != nil { return err } - gs.referenceFile(f.path) - return nil -} - -func (gs *goldsmith) referenceFile(path string) { - gs.refMtx.Lock() - defer gs.refMtx.Unlock() - - path = cleanPath(path) - + pathSeg := cleanPath(f.path) for { - gs.refs[path] = true - if path == "." { + gs.refs[pathSeg] = true + if pathSeg == "." { break } - path = filepath.Dir(path) + pathSeg = filepath.Dir(pathSeg) } + + return nil } func (gs *goldsmith) fault(f *file, err error) { diff --git a/file.go b/file.go index d7af136..b1984ab 100644 --- a/file.go +++ b/file.go @@ -28,6 +28,7 @@ import ( "io/ioutil" "os" "path" + "path/filepath" ) type file struct { @@ -38,7 +39,8 @@ type file struct { asset string } -func (f *file) export(dstPath string) error { +func (f *file) export(dstDir string) error { + dstPath := filepath.Join(dstDir, f.path) if len(f.asset) > 0 && fileCached(f.asset, dstPath) { return nil } diff --git a/goldsmith.go b/goldsmith.go index 7d702c2..7d5d927 100644 --- a/goldsmith.go +++ b/goldsmith.go @@ -68,7 +68,6 @@ func NewFileFromAsset(path, asset string) File { type Context interface { DispatchFile(f File) - ReferenceFile(path string) SrcDir() string DstDir() string From a495091b0106a00915b8fa4d910d0e1af1bc7427 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Thu, 14 Jan 2016 12:14:22 +0900 Subject: [PATCH 14/15] Fixing README --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 89f3e1a..8cb3664 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,4 @@ -+++ -area = "projects" -github = "goldsmith" -tags = ["generator", "golang", "goldsmith", "mit license", "web"] -title = "Goldsmith" -+++ +# Goldsmith # Goldsmith is a static website generator developed in Golang with flexibility, extensibility, and performance as primary design considerations. With Goldsmith you can easily build and deploy any type of site, whether it is a personal blog, From 53b4d4702c7c1d1342d51b3f800ed140976597fc Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Sun, 5 Jun 2016 22:02:09 -0700 Subject: [PATCH 15/15] Add directory query --- file.go | 4 ++++ goldsmith.go | 1 + 2 files changed, 5 insertions(+) diff --git a/file.go b/file.go index b1984ab..a69ce94 100644 --- a/file.go +++ b/file.go @@ -99,6 +99,10 @@ func (f *file) Path() string { return f.path } +func (f *file) Dir() string { + return path.Dir(f.path) +} + func (f *file) Value(key string) (interface{}, bool) { value, ok := f.Meta[key] return value, ok diff --git a/goldsmith.go b/goldsmith.go index 7d5d927..c4186f2 100644 --- a/goldsmith.go +++ b/goldsmith.go @@ -40,6 +40,7 @@ func Begin(srcDir string) Goldsmith { type File interface { Path() string + Dir() string Value(key string) (interface{}, bool) SetValue(key string, value interface{})