Cleanup

build/scrape.go

@@ -23,7 +23,6 @@
 package main
 
 import (
-	"log"
 	"net/url"
 	"sync"
 
@@ -47,6 +46,8 @@ type review struct {
 
 	closestStnName string
 	closestStnDist float64
+
+	err error
 }
 
 type scraper interface {
@@ -71,78 +72,76 @@ func makeAbsUrl(ref, base string) (string, error) {
 }
 
 func decodeReviews(in chan review, out chan review, scr scraper) {
-	for {
+	for rev, ok := <-in; ok; {
-		if res, ok := <-in; ok {
+		if rev.err == nil {
-			var err error
+			rev.latitude, rev.longitude, rev.err = scr.decode(rev.address)
-			res.latitude, res.longitude, err = scr.decode(res.address)
-			if err == nil {
-				out <- res
-			} else {
-				log.Printf("failed to decode address for %s (%v)", res.url, err)
-			}
-		} else {
-			close(out)
-			return
 		}
+
+		out <- rev
 	}
+
+	close(out)
 }
 
 func scrapeReview(url string, out chan review, scr scraper, group *sync.WaitGroup) {
 	defer group.Done()
 
-	doc, err := scr.load(url)
+	var (
-	if err != nil {
+		doc *goquery.Document
-		log.Printf("failed to load review at %s (%v)", url, err)
+		rev = review{url: url}
-		return
+	)
+
+	if doc, rev.err = scr.load(rev.url); rev.err == nil {
+		rev.name, rev.address, rev.features, rev.err = scr.review(doc)
 	}
 
-	name, address, features, err := scr.review(doc)
+	out <- rev
-	if err != nil {
-		log.Printf("failed to scrape review at %s (%v)", url, err)
-		return
-	}
-
-	out <- review{
-		name:     name,
-		address:  address,
-		features: features,
-		url:      url}
 }
 
-func scrapeIndex(indexUrl string, out chan review, scr scraper) {
+func scrapeIndex(indexUrl string, out chan review, scr scraper) error {
-	doc, err := scr.load(indexUrl)
-	if err != nil {
-		log.Printf("failed to load index at %s (%v)", indexUrl, err)
-		return
-	}
-
-	nextIndexUrl, reviewUrls := scr.index(doc)
-	if err != nil {
-		log.Fatal(err)
-	}
-
 	var group sync.WaitGroup
-	for _, reviewUrl := range reviewUrls {
-		absUrl, err := makeAbsUrl(reviewUrl, indexUrl)
-		if err != nil {
-			log.Fatal(err)
-		}
 
-		group.Add(1)
+	defer func() {
-		go scrapeReview(absUrl, out, scr, &group)
+		group.Wait()
-	}
-	group.Wait()
-
-	if nextIndexUrl == "" {
 		close(out)
-	} else {
+	}()
-		absUrl, err := makeAbsUrl(nextIndexUrl, indexUrl)
+
+	for {
+		doc, err := scr.load(indexUrl)
 		if err != nil {
-			log.Fatal(err)
+			return err
 		}
 
-		scrapeIndex(absUrl, out, scr)
+		nextIndexUrl, reviewUrls := scr.index(doc)
+		if err != nil {
+			return err
+		}
+
+		for _, reviewUrl := range reviewUrls {
+			absUrl, err := makeAbsUrl(reviewUrl, indexUrl)
+			if err != nil {
+				return err
+			}
+
+			group.Add(1)
+			go scrapeReview(absUrl, out, scr, &group)
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if nextIndexUrl == "" {
+			break
+		}
+
+		indexUrl, err = makeAbsUrl(nextIndexUrl, indexUrl)
+		if err != nil {
+			return err
+		}
 	}
+
+	return nil
 }
 
 func scrape(url string, scr scraper) []review {
@@ -152,12 +151,10 @@ func scrape(url string, scr scraper) []review {
 	go scrapeIndex(url, in, scr)
 	go decodeReviews(in, out, scr)
 
-	var results []review
+	var reviews []review
-	for {
+	for rev, ok := <-out; ok; {
-		if res, ok := <-out; ok {
+		reviews = append(reviews, rev)
-			results = append(results, res)
-		} else {
-			return results
-		}
 	}
+
+	return reviews
 }