alex/rtk-scrape
1
Fork 0
Code Releases Activity

Update flags

Browse Source
This commit is contained in:
Alex Yatskov 2024-04-01 21:51:47 -07:00
parent c6dc096a0e
commit bf315663eb
1 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
main.go
View File

@ -135,11 +135,10 @@ func save(path string, kanjiList []*KanjiEntry) error {
func main() {
var (
username = flag.String("username", "", "login username for kanji.koohii.com")
password = flag.String("password", "", "login password for kanji.koohii.com")
retryCount = flag.Int("retryCount", 3, "scrape retry count")
firstFrame = flag.Int("firstFrame", 1, "kanji first frame")
lastFrame = flag.Int("lastFrame", 3030, "kanji last frame")
retryCount = flag.Int("retryCount", 3, "scrape retry count")
scrapeDelay = flag.Int("scrapeDelay", 2, "scrape delay in seconds")
)
flag.Usage = func() {
@ -151,18 +150,28 @@ func main() {
flag.Parse()
args := flag.Args()
if len(*username) == 0 || len(*password) == 0 || len(args) == 0 || *firstFrame > *lastFrame {
if len(args) == 0 || *firstFrame > *lastFrame {
flag.Usage()
os.Exit(2)
}
username, ok := os.LookupEnv("RTK_USER")
if !ok {
log.Fatalf("environment variable RTK_USER environment is not set")
}
password, ok := os.LookupEnv("RTK_PASS")
if !ok {
log.Fatalf("environment variable RTK_PASS environment is not set")
}
br := surf.NewBrowser()
br.SetUserAgent(agent.Firefox())
br.AddRequestHeader("Accept", "text/html")
br.AddRequestHeader("Accept-Charset", "utf8")
log.Println("logging in...")
if err := login(br, *username, *password); err != nil {
if err := login(br, username, password); err != nil {
log.Fatal(err)
}
@ -188,7 +197,7 @@ func main() {
log.Printf("scraping %s...", lookup)
kanji, err = scrape(br, lookup)
time.Sleep(2 * time.Second)
time.Sleep(time.Second * time.Duration(*scrapeDelay))
if err == nil {
break