1

Adding dumping to sqlite3

This commit is contained in:
Alex Yatskov 2015-08-23 15:16:01 +09:00
parent 6b010e7dfc
commit 51e6a08eff

130
scrape.go
View File

@ -24,9 +24,12 @@ package main
import (
"bufio"
"database/sql"
"errors"
"net/url"
"os"
_ "github.com/mattn/go-sqlite3"
)
func scrapeUrls(filename string, wc *webCache, gc *geoCache) ([]restaurant, error) {
@ -97,13 +100,136 @@ func processData(restaurants []restaurant, stationsPath string) error {
return nil
}
func dumpData(dbPath string, restaraunts []restaurant) error {
db, err := sql.Open("sqlite3", dbPath)
if err != nil {
return err
}
defer db.Close()
//
// Restaurants
//
_, err = db.Exec(`
DROP TABLE IF EXISTS reviews;
CREATE TABLE reviews(
name VARCHAR(100) NOT NULL,
url VARCHAR(200) NOT NULL,
delicious FLOAT NOT NULL,
accommodating FLOAT NOT NULL,
affordable FLOAT NOT NULL,
atmospheric FLOAT NOT NULL,
latitude FLOAT NOT NULL,
longitude FLOAT NOT NULL,
distanceToStn FLOAT NOT NULL,
closestStn VARCHAR(100) NOT NULL,
accessCount INT NOT NULL,
id INT NOT NULL AUTO_INCREMENT PRIMARY KEY
) DEFAULT CHARACTER SET utf8`)
if err != nil {
return err
}
for _, r := range restaraunts {
_, err = db.Exec(`
INSERT INTO reviews(
name,
url,
delicious,
accommodating,
affordable,
atmospheric,
latitude,
longitude,
closestStnDist,
closestStnName,
accessCount
) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
r.name,
r.url,
0,
0,
0,
0,
r.longitude,
r.latitude,
r.closestStnDist,
r.closestStnName)
if err != nil {
return err
}
}
//
// Categories
//
_, err = db.Exec(`
DROP TABLE IF EXISTS categories;
CREATE TABLE categories(
description VARCHAR(200) NOT NULL,
id INT NOT NULL AUTO_INCREMENT PRIMARY KEY))`)
if err != nil {
return err
}
for _, category := range []string{"I prefer quiet places", "I enjoy Mexican Food", "I drive a car"} {
if _, err := db.Exec("INSERT INTO categories(description) VALUES (?)", category); err != nil {
return err
}
}
//
// History
//
_, err = db.Exec(`
DROP TABLE IF EXISTS history;
CREATE TABLE history(
date DATETIME NOT NULL,
reviewId INT NOT NULL,
id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
FOREIGN KEY(reviewId) REFERENCES reviews(id))`)
if err != nil {
return err
}
//
// HistoryGroup
//
_, err = db.Exec(`
DROP TABLE IF EXISTS historyGroups;
CREATE TABLE historyGroups(
categoryId INT NOT NULL,
categoryValue FLOAT NOT NULL,
historyId INT NOT NULL,
FOREIGN KEY(historyId) REFERENCES history(id),
FOREIGN KEY(categoryId) REFERENCES categories(id))`)
if err != nil {
return err
}
return nil
}
func main() {
restaraunts, err := scrapeData("data/urls.txt", "cache/geocache.json", "cache/webcache")
restaurants, err := scrapeData("data/urls.txt", "cache/geocache.json", "cache/webcache")
if err != nil {
panic(err)
}
if err := processData(restaraunts, "data/stations.json"); err != nil {
if err := processData(restaurants, "data/stations.json"); err != nil {
panic(err)
}
if err := dumpData("db.sqlite3", restaurants); err != nil {
panic(err)
}
}