93 lines
2.1 KiB
Go
93 lines
2.1 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"git.dvdrw.dev/nsmarter/scraper/scraper"
|
|
"log"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
var Log = log.Default()
|
|
|
|
const defaultChunkSize = 5
|
|
const defaultLimitQuerySize = 0
|
|
|
|
var stationDbPath string
|
|
var innerCityOnly bool = false
|
|
var chunkSize = 5
|
|
var apiEndpoint string = "https://online.nsmart.rs/publicapi/v1/announcement/announcement.php"
|
|
var apiKey string
|
|
var limitQuerySize = 0
|
|
|
|
func parseEnvars() {
|
|
for _, e := range os.Environ() {
|
|
pair := strings.SplitN(e, "=", 2)
|
|
switch pair[0] {
|
|
case "STATIONS_DB_PATH":
|
|
stationDbPath = pair[1]
|
|
case "INNER_CITY_ONLY":
|
|
innerCityOnly = pair[1] == "1"
|
|
case "CHUNK_SIZE":
|
|
var err error
|
|
chunkSize, err = strconv.Atoi(pair[1])
|
|
if err != nil {
|
|
Log.Printf("WARN: Invalid value for CHUNK_SIZE. Falling back to default value (%v)\n", defaultChunkSize)
|
|
chunkSize = defaultChunkSize
|
|
}
|
|
case "LIMIT_QUERY_SIZE":
|
|
var err error
|
|
limitQuerySize, err = strconv.Atoi(pair[1])
|
|
if err != nil {
|
|
Log.Printf("WARN: Invalid value for LIMIT_QUERY_SIZE. Falling back to default value (%v)\n", defaultLimitQuerySize)
|
|
limitQuerySize = defaultLimitQuerySize
|
|
}
|
|
case "API_ENDPOINT":
|
|
apiEndpoint = pair[1]
|
|
case "API_KEY":
|
|
apiKey = pair[1]
|
|
}
|
|
}
|
|
}
|
|
|
|
func main() {
|
|
// Parse out config from environment variables
|
|
parseEnvars()
|
|
|
|
if stationDbPath == "" {
|
|
log.Fatal("Environment variable STATIONS_DB_PATH empty!")
|
|
}
|
|
|
|
stations, err := readDb()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
Log.Printf("Finished importing stations! (Total: %v)", len(stations))
|
|
|
|
// Split stations so as to be scraped in chunks
|
|
var stationChunks [][]scraper.Station
|
|
for i := 0; i < len(stations); i += chunkSize {
|
|
end := i + chunkSize
|
|
|
|
if end > len(stations) {
|
|
end = len(stations)
|
|
}
|
|
|
|
stationChunks = append(stationChunks, stations[i:end])
|
|
}
|
|
|
|
results := make(chan []scraper.ScrapeResult, 200)
|
|
for _, chunk := range stationChunks {
|
|
go scraper.ScheduleScrape(chunk,
|
|
results,
|
|
scraper.ApiConfig{Endpoint: apiEndpoint,
|
|
Key: apiKey})
|
|
}
|
|
|
|
for r := range results {
|
|
fmt.Printf("Received data: %#v\n", r)
|
|
}
|
|
}
|