WIP

build/build.go

@@ -52,29 +52,6 @@ func (s scrapeCtx) load(url string) (*goquery.Document, error) {
 	return s.wc.load(url)
 }
 
-type semantics struct {
-	accomodating float64
-	affordable   float64
-	atmospheric  float64
-	delicious    float64
-}
-
-func (s semantics) combine(other semantics, weight float64) semantics {
-	return semantics{
-		s.accomodating + other.accomodating*weight,
-		s.affordable + other.affordable*weight,
-		s.atmospheric + other.atmospheric*weight,
-		s.delicious + other.delicious*weight}
-}
-
-func (s semantics) reduce(weight float64) semantics {
-	return semantics{
-		s.accomodating / weight,
-		s.affordable / weight,
-		s.atmospheric / weight,
-		s.delicious / weight}
-}
-
 type restaurant struct {
 	name    string
 	address string
@@ -251,10 +228,10 @@ func dumpData(dbPath string, restaraunts map[uint64]*restaurant) error {
 			) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 			rest.name,
 			rest.address,
-			rest.sem.delicious,
+			rest.sem.Delicious,
-			rest.sem.accomodating,
+			rest.sem.Accomodating,
-			rest.sem.affordable,
+			rest.sem.Affordable,
-			rest.sem.atmospheric,
+			rest.sem.Atmospheric,
 			rest.latitude,
 			rest.longitude,
 			rest.closestStnDist,

build/data/tripadvisor.toml

@@ -1,5 +1,6 @@
-[index]
+name = "tripadvisor"
-    [index.item]
+
+[index.items]
     path = "a.property_title"
     attr = "href"
 
@@ -7,9 +8,6 @@
     path = "div.deckTools.btm a.nav.next.rndBtn.rndBtnGreen.taLnk"
     attr = "href"
 
-[item]
-    scale = 5.0
-
 [item.name]
     path = "h1#HEADING"
 
@@ -22,25 +20,45 @@
 
 [item.semantics]
     [item.semantics.service]
+        accomodating = 1.0
+        affordable = 0.0
+        atmospheric = 0.0
+        delicious: 0.0
+        scale = 5.0
+
         path = "ul.barChart div.ratingRow img:nth-child(2)"
         regex = "^([0-9]*\.?[0-9]+)"
         attr = "alt"
-        def = {"accomodating": 1.0, "affordable": 0.0, "atmospheric": 0.0, "delicious": 0.0}
 
     [item.semantics.food]
+        accomodating = 0.0
+        affordable = 0.0
+        atmospheric = 0.0
+        delicious: 1.0
+        scale = 5.0
+
         path = "ul.barChart div.ratingRow img:nth-child(1)"
         regex = "^([0-9]*\.?[0-9]+)"
         attr = "alt"
-        def = {"accomodating": 0.0, "affordable": 0.0, "atmospheric": 0.0, "delicious": 1.0}
 
     [item.semantics.value]
+        accomodating = 0.0
+        affordable = 1.0
+        atmospheric = 0.0
+        delicious: 0.0
+        scale = 5.0
+
         path = "ul.barChart div.ratingRow img:nth-child(3)"
         regex = "^([0-9]*\.?[0-9]+)"
         attr = "alt"
-        def = {"accomodating": 0.0, "affordable": 1.0, "atmospheric": 0.0, "delicious": 0.0}
 
     [item.semantics.atmosphere]
+        accomodating = 0.0
+        affordable = 0.0
+        atmospheric = 1.0
+        delicious: 0.0
+        scale = 5.0
+
         path = "ul.barChart div.ratingRow img:nth-child(4)"
         regex = "^([0-9]*\.?[0-9]+)"
         attr = "alt"
-        def = {"accomodating": 0.0, "affordable": 0.0, "atmospheric": 1.0, "delicious": 0.0}

build/descriptor.go

@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) 2015 Alex Yatskov <alex@foosoft.net>
+ * Author: Alex Yatskov <alex@foosoft.net>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+ * the Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/naoina/toml"
+)
+
+//
+// descriptor
+//
+type descriptor struct {
+	gc *geoCache
+	wc *webCache
+
+	conf descConf
+}
+
+//
+// semantics
+//
+type semantics struct {
+	Accomodating float64
+	Affordable   float64
+	Atmospheric  float64
+	Delicious    float64
+}
+
+func (s semantics) combine(other semantics, weight float64) semantics {
+	return semantics{
+		s.Accomodating + other.Accomodating*weight,
+		s.Affordable + other.Affordable*weight,
+		s.Atmospheric + other.Atmospheric*weight,
+		s.Delicious + other.Delicious*weight,
+	}
+}
+
+func (s semantics) reduce(weight float64) semantics {
+	return semantics{
+		s.Accomodating / weight,
+		s.Affordable / weight,
+		s.Atmospheric / weight,
+		s.Delicious / weight,
+	}
+}
+
+//
+// pathConf
+//
+type pathConf struct {
+	Path  string
+	Attr  string
+	RegEx string
+
+	regExComp *regexp.Regexp
+}
+
+func (c *pathConf) locateStrings(doc *goquery.Document) ([]string, error) {
+	var err error
+	if len(c.RegEx) > 0 && c.regExComp == nil {
+		if c.regExComp, err = regexp.Compile(c.RegEx); err != nil {
+			return nil, err
+		}
+	}
+
+	var strs []string
+	doc.Find(c.Path).Each(func(index int, sel *goquery.Selection) {
+		var str string
+		if len(c.Attr) > 0 {
+			str, _ = sel.Attr(c.Attr)
+		} else {
+			str = sel.Text()
+		}
+
+		if c.regExComp != nil {
+			if matches := c.regExComp.FindStringSubmatch(str); len(matches) > 1 {
+				str = matches[1]
+			} else {
+				str = ""
+			}
+		}
+
+		strs = append(strs, str)
+	})
+
+	return strs, err
+
+}
+
+func (c *pathConf) locateString(doc *goquery.Document) (string, error) {
+	strs, err := c.locateStrings(doc)
+	if err != nil {
+		return "", err
+	}
+
+	return strings.Join(strs, " "), nil
+}
+
+func (c *pathConf) locateInt(doc *goquery.Document) (int64, error) {
+	str, err := c.locateString(doc)
+	if err != nil {
+		return 0, err
+	}
+
+	return strconv.ParseInt(str, 10, 8)
+}
+
+func (c *pathConf) locateFloat(doc *goquery.Document) (float64, error) {
+	str, err := c.locateString(doc)
+	if err != nil {
+		return 0.0, err
+	}
+
+	return strconv.ParseFloat(str, 8)
+}
+
+//
+// semConf
+//
+type semConf struct {
+	semantics
+	pathConf
+
+	Scale float64
+}
+
+//
+// descConf
+//
+type descConf struct {
+	Index struct {
+		Items pathConf
+		Next  pathConf
+	}
+
+	Item struct {
+		Name    pathConf
+		Address pathConf
+		Count   pathConf
+		Sem     map[string]semConf
+	}
+}
+
+func newDescriptor(filename string, gc *geoCache, wc *webCache) (*descriptor, error) {
+	desc := &descriptor{gc: gc, wc: wc}
+
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := toml.Unmarshal(bytes, &desc.conf); err != nil {
+		return nil, err
+	}
+
+	return desc, nil
+}
+
+func (d descriptor) define(keyword string) semantics {
+	return d.conf.Item.Sem[keyword].semantics
+}
+
+func (d descriptor) index(doc *goquery.Document) (next string, items []string, err error) {
+	if items, err = d.conf.Index.Items.locateStrings(doc); err != nil {
+		return
+	}
+
+	if next, err = d.conf.Index.Next.locateString(doc); err != nil {
+		return
+	}
+
+	return
+}
+
+func (d descriptor) review(doc *goquery.Document) (name, address string, features map[string]float64, count int64, err error) {
+	if name, err = d.conf.Item.Name.locateString(doc); err != nil || len(name) == 0 {
+		err = errors.New("invalid name")
+		return
+	}
+
+	if address, err = d.conf.Item.Address.locateString(doc); err != nil || len(address) == 0 {
+		err = errors.New("invalid address")
+		return
+	}
+
+	if count, err = d.conf.Item.Count.locateInt(doc); err != nil {
+		err = errors.New("invalid review count")
+		return
+	}
+
+	features = make(map[string]float64)
+	for n, s := range d.conf.Item.Sem {
+		var value float64
+		if value, err = s.pathConf.locateFloat(doc); err != nil {
+			err = fmt.Errorf("invalid feature value for %s", n)
+			return
+		}
+
+		if s.Scale != 0.0 {
+			value /= s.Scale
+		}
+
+		features[n] = value
+	}
+
+	return
+}

build/tabelog.go

@@ -37,11 +37,11 @@ type tabelog struct {
 
 func (tabelog) define(keyword string) semantics {
 	return map[string]semantics{
-		"dishes":     {accomodating: 0.0, affordable: 0.0, atmospheric: 0.0, delicious: 0.8},
+		"dishes":     {Accomodating: 0.0, Affordable: 0.0, Atmospheric: 0.0, Delicious: 0.8},
-		"drinks":     {accomodating: 0.0, affordable: 0.0, atmospheric: 0.0, delicious: 0.2},
+		"drinks":     {Accomodating: 0.0, Affordable: 0.0, Atmospheric: 0.0, Delicious: 0.2},
-		"service":    {accomodating: 1.0, affordable: 0.0, atmospheric: 0.0, delicious: 0.0},
+		"service":    {Accomodating: 1.0, Affordable: 0.0, Atmospheric: 0.0, Delicious: 0.0},
-		"cost":       {accomodating: 0.0, affordable: 1.0, atmospheric: 0.0, delicious: 0.0},
+		"cost":       {Accomodating: 0.0, Affordable: 1.0, Atmospheric: 0.0, Delicious: 0.0},
-		"atmosphere": {accomodating: 0.0, affordable: 0.0, atmospheric: 1.0, delicious: 0.0},
+		"atmosphere": {Accomodating: 0.0, Affordable: 0.0, Atmospheric: 1.0, Delicious: 0.0},
 	}[keyword]
 }
 

build/tripadvisor.go

@@ -37,10 +37,10 @@ type tripadvisor struct {
 
 func (tripadvisor) define(keyword string) semantics {
 	return map[string]semantics{
-		"food":       {accomodating: 0.0, affordable: 0.0, atmospheric: 0.0, delicious: 1.0},
+		"food":       {Accomodating: 0.0, Affordable: 0.0, Atmospheric: 0.0, Delicious: 1.0},
-		"service":    {accomodating: 1.0, affordable: 0.0, atmospheric: 0.0, delicious: 0.0},
+		"service":    {Accomodating: 1.0, Affordable: 0.0, Atmospheric: 0.0, Delicious: 0.0},
-		"value":      {accomodating: 0.0, affordable: 1.0, atmospheric: 0.0, delicious: 0.0},
+		"value":      {Accomodating: 0.0, Affordable: 1.0, Atmospheric: 0.0, Delicious: 0.0},
-		"atmosphere": {accomodating: 0.0, affordable: 0.0, atmospheric: 1.0, delicious: 0.0},
+		"atmosphere": {Accomodating: 0.0, Affordable: 0.0, Atmospheric: 1.0, Delicious: 0.0},
 	}[keyword]
 }