1
This commit is contained in:
Alex Yatskov 2015-09-18 16:55:39 +09:00
parent e480b4cb61
commit ca12e619ee

View File

@ -23,6 +23,7 @@
package main package main
import ( import (
"log"
"net/url" "net/url"
"sync" "sync"
@ -146,13 +147,28 @@ func scrape(url string, scr scraper) ([]review, error) {
out := make(chan review, 128) out := make(chan review, 128)
in := make(chan review, 128) in := make(chan review, 128)
go decodeReviews(in, out, scr) var (
err := scrapeIndex(url, in, scr) reviews []review
wg sync.WaitGroup
)
var reviews []review wg.Add(1)
go func() {
defer wg.Done()
for rev, ok := <-out; ok; { for rev, ok := <-out; ok; {
if rev.err == nil {
reviews = append(reviews, rev) reviews = append(reviews, rev)
} }
log.Printf("%s: %s", rev.name, rev.err)
}
}()
go decodeReviews(in, out, scr)
err := scrapeIndex(url, in, scr)
wg.Wait()
return reviews, err return reviews, err
} }