alex/rtk-scrape
1
Fork 0
Code Releases Activity

Add retry count

Browse Source
This commit is contained in:
Alex Yatskov 2020-01-19 10:52:26 -08:00
parent 95232a44b9
commit 43d5d16ee8
2 changed files with 19 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.sum
View File

@ -8,6 +8,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/go-ole/go-ole v1.2.5-0.20190708054153-938323a72016/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ole/go-ole v1.2.5-0.20190708054153-938323a72016/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
github.com/headzoo/surf v1.0.0 h1:d2h9ftKeQYj7tKqAjQtAA0lJVkO8cTxvzdXLynmNnHM= github.com/headzoo/surf v1.0.0 h1:d2h9ftKeQYj7tKqAjQtAA0lJVkO8cTxvzdXLynmNnHM=
github.com/headzoo/surf v1.0.0/go.mod h1:/bct0m/iMNEqpn520y01yoaWxsAEigGFPnvyR1ewR5M= github.com/headzoo/surf v1.0.0/go.mod h1:/bct0m/iMNEqpn520y01yoaWxsAEigGFPnvyR1ewR5M=
github.com/headzoo/ut v0.0.0-20181013193318-a13b5a7a02ca h1:utFgFwgxaqx5OthzE3DSGrtOq7rox5r2sxZ2wbfTuK0=
github.com/headzoo/ut v0.0.0-20181013193318-a13b5a7a02ca/go.mod h1:8926sG02TCOX4RFRzIMFIzRw4xuc/TwO2gtN7teMJZ4= github.com/headzoo/ut v0.0.0-20181013193318-a13b5a7a02ca/go.mod h1:8926sG02TCOX4RFRzIMFIzRw4xuc/TwO2gtN7teMJZ4=
github.com/itchio/go-brotli v0.0.0-20190702114328-3f28d645a45c h1:Jf20xV/yR/O6eSUqLTuXhka/+54YR59sGwN7b3MkxYk= github.com/itchio/go-brotli v0.0.0-20190702114328-3f28d645a45c h1:Jf20xV/yR/O6eSUqLTuXhka/+54YR59sGwN7b3MkxYk=
github.com/itchio/go-brotli v0.0.0-20190702114328-3f28d645a45c/go.mod h1:oRXh43p/JW9kWosasd+2kHfDpb1ec4m7YrZ5E39s1iI= github.com/itchio/go-brotli v0.0.0-20190702114328-3f28d645a45c/go.mod h1:oRXh43p/JW9kWosasd+2kHfDpb1ec4m7YrZ5E39s1iI=

24
main.go
View File

@ -149,6 +149,7 @@ func main() {
var ( var (
username = flag.String("username", "", "login username for kanji.koohii.com") username = flag.String("username", "", "login username for kanji.koohii.com")
password = flag.String("password", "", "login password for kanji.koohii.com") password = flag.String("password", "", "login password for kanji.koohii.com")
retryCount = flag.Int("retryCount", 3, "scrape retry count")
firstFrame = flag.Int("firstFrame", 1, "kanji first frame") firstFrame = flag.Int("firstFrame", 1, "kanji first frame")
lastFrame = flag.Int("lastFrame", 3030, "kanji last frame") lastFrame = flag.Int("lastFrame", 3030, "kanji last frame")
) )
@ -186,14 +187,25 @@ func main() {
var kanjiList []*KanjiEntry var kanjiList []*KanjiEntry
for _, lookup := range lookups { for _, lookup := range lookups {
log.Printf("scraping %s...", lookup) var kanji *KanjiEntry
kanji, err := scrape(br, lookup) var err error
if err != nil {
log.Fatal(err) for i := 0; i < *retryCount; i++ {
log.Printf("scraping %s...", lookup)
kanji, err = scrape(br, lookup)
time.Sleep(2 * time.Second)
if err == nil {
break
}
} }
kanjiList = append(kanjiList, kanji) if err == nil {
time.Sleep(2 * time.Second) kanjiList = append(kanjiList, kanji)
} else {
log.Fatal(err)
}
} }
log.Printf("saving to %s...", args[0]) log.Printf("saving to %s...", args[0])