Cleanup

build/scrape.go

@@ -23,7 +23,6 @@
 package main
 
 import (
-	"log"
 	"net/url"
 	"sync"
 
@@ -47,6 +46,8 @@ type review struct {
 
 	closestStnName string
 	closestStnDist float64
+
+	err error
 }
 
 type scraper interface {
@@ -71,80 +72,78 @@ func makeAbsUrl(ref, base string) (string, error) {
 }
 
 func decodeReviews(in chan review, out chan review, scr scraper) {
-	for {
+	for rev, ok := <-in; ok; {
-		if res, ok := <-in; ok {
+		if rev.err == nil {
-			var err error
+			rev.latitude, rev.longitude, rev.err = scr.decode(rev.address)
-			res.latitude, res.longitude, err = scr.decode(res.address)
-			if err == nil {
-				out <- res
-			} else {
-				log.Printf("failed to decode address for %s (%v)", res.url, err)
 		}
-		} else {
+
+		out <- rev
+	}
+
 	close(out)
-			return
-		}
-	}
 }
 
 func scrapeReview(url string, out chan review, scr scraper, group *sync.WaitGroup) {
 	defer group.Done()
 
-	doc, err := scr.load(url)
+	var (
-	if err != nil {
+		doc *goquery.Document
-		log.Printf("failed to load review at %s (%v)", url, err)
+		rev = review{url: url}
-		return
+	)
+
+	if doc, rev.err = scr.load(rev.url); rev.err == nil {
+		rev.name, rev.address, rev.features, rev.err = scr.review(doc)
 	}
 
-	name, address, features, err := scr.review(doc)
+	out <- rev
-	if err != nil {
-		log.Printf("failed to scrape review at %s (%v)", url, err)
-		return
 }
 
-	out <- review{
+func scrapeIndex(indexUrl string, out chan review, scr scraper) error {
-		name:     name,
+	var group sync.WaitGroup
-		address:  address,
-		features: features,
-		url:      url}
-}
 
-func scrapeIndex(indexUrl string, out chan review, scr scraper) {
+	defer func() {
+		group.Wait()
+		close(out)
+	}()
+
+	for {
 		doc, err := scr.load(indexUrl)
 		if err != nil {
-		log.Printf("failed to load index at %s (%v)", indexUrl, err)
+			return err
-		return
 		}
 
 		nextIndexUrl, reviewUrls := scr.index(doc)
 		if err != nil {
-		log.Fatal(err)
+			return err
 		}
 
-	var group sync.WaitGroup
 		for _, reviewUrl := range reviewUrls {
 			absUrl, err := makeAbsUrl(reviewUrl, indexUrl)
 			if err != nil {
-			log.Fatal(err)
+				return err
 			}
 
 			group.Add(1)
 			go scrapeReview(absUrl, out, scr, &group)
 		}
-	group.Wait()
+
+		if err != nil {
+			return err
+		}
 
 		if nextIndexUrl == "" {
-		close(out)
+			break
-	} else {
-		absUrl, err := makeAbsUrl(nextIndexUrl, indexUrl)
-		if err != nil {
-			log.Fatal(err)
 		}
 
-		scrapeIndex(absUrl, out, scr)
+		indexUrl, err = makeAbsUrl(nextIndexUrl, indexUrl)
+		if err != nil {
+			return err
 		}
 	}
 
+	return nil
+}
+
 func scrape(url string, scr scraper) []review {
 	out := make(chan review, 128)
 	in := make(chan review, 128)
@@ -152,12 +151,10 @@ func scrape(url string, scr scraper) []review {
 	go scrapeIndex(url, in, scr)
 	go decodeReviews(in, out, scr)
 
-	var results []review
+	var reviews []review
-	for {
+	for rev, ok := <-out; ok; {
-		if res, ok := <-out; ok {
+		reviews = append(reviews, rev)
-			results = append(results, res)
-		} else {
-			return results
-		}
 	}
+
+	return reviews
 }