feat: feed data to upstream over REST instead of printing to stdout

main/main.go

@@ -1,12 +1,16 @@
 package main
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
-	"git.dvdrw.dev/nsmarter/scraper/scraper"
 	"log"
+	"net/http"
 	"os"
 	"strconv"
 	"strings"
+
+	"git.dvdrw.dev/nsmarter/scraper/scraper"
 )
 
 var Log = log.Default()
@@ -20,6 +24,7 @@ var chunkSize = 5
 var apiEndpoint string = "https://online.nsmart.rs/publicapi/v1/announcement/announcement.php"
 var apiKey string
 var limitQuerySize = 0
+var fillerUrl string = "localhost"
 
 func parseEnvars() {
 	for _, e := range os.Environ() {
@@ -47,6 +52,8 @@ func parseEnvars() {
 			apiEndpoint = pair[1]
 		case "API_KEY":
 			apiKey = pair[1]
+		case "FILLER_URL":
+			fillerUrl = pair[1]
 		}
 	}
 }
@@ -88,5 +95,22 @@ func main() {
 
 	for r := range results {
 		fmt.Printf("Received data: %#v\n", r)
+
+		json, err := json.Marshal(r)
+		if err != nil {
+			fmt.Print("Couldn't serialise struct to JSON: ", err)
+		}
+
+		request, err := http.NewRequest("POST", fillerUrl+"/v1/submit", bytes.NewBuffer(json))
+		request.Header.Set("Content-Type", "application/json; charset=UTF-8")
+
+		go (func(req *http.Request) {
+			res, err := http.DefaultClient.Do(req)
+			if err != nil {
+				fmt.Print("Error while sending data to filler: ", err)
+			} else if res.StatusCode != 200 {
+				fmt.Print("Non-200 while sending data to filler!")
+			}
+		})(request)
 	}
 }

scraper/scrape.go

@@ -3,49 +3,51 @@ package scraper
 import (
 	"encoding/json"
 	"fmt"
+	"log"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
-	"log"
 )
 
 type Coords struct {
-	Lat, Lon float64
+	Lat float64 `json:"lat"`
+	Lon float64 `json:"lon"`
 }
 
 type Vehicle struct {
-	GarageId      string
+	GarageId      string `json:"garageId"`
-	Coords        Coords
+	Coords        Coords `json:"coords"`
-	AtStationId   int
+	AtStationId   int    `json:"atStationId"`
-	AtStationName string
+	AtStationName string `json:"atStationName"`
 }
 
 type StationInfo struct {
-	Id       int
+	Id       int    `json:"id"`
-	CityId   int
+	CityId   int    `json:"cityId"`
-	CityName string
+	CityName string `json:"cityName"`
-	Name     string
+	Name     string `json:"name"`
-	Coords   Coords
+	Coords   Coords `json:"coords"`
 }
 
 type LineInfo struct {
-	Name     string
+	Id       int           `json:"id"`
-	Title    string
+	Name     string        `json:"name"`
-	Stations []StationInfo
+	Title    string        `json:"title"`
-	Route    []Coords
+	Stations []StationInfo `json:"stations"`
+	Route    []Coords      `json:"route"`
 }
 
 type ScrapeResult struct {
-	Id          int
+	Id          int       `json:"id"`
-	Success     bool
+	Success     bool      `json:"success"`
-	SecondsLeft int
+	SecondsLeft int       `json:"secondsLeft"`
-	LineInfo    LineInfo
+	LineInfo    LineInfo  `json:"lineInfo"`
-	Vehicles    []Vehicle
+	Vehicles    []Vehicle `json:"vehicles"`
 }
 
 type Station struct {
-	Id int
+	Id int `json:"id"`
 }
 
 var Log = log.Default()
@@ -139,6 +141,13 @@ func Scrape(stations []Station, c ApiConfig) ([]ScrapeResult, error) {
 
 			var lineInfo LineInfo
 
+			lineId, err := strconv.Atoi(bus["id"].(string))
+			if err != nil {
+				Log.Printf("WARN: Failed to parse vehicle for stationId=%v\n\t%v", id, err)
+				continue
+			}
+			lineInfo.Id = lineId
+
 			lineName, ok := bus["line_number"].(string)
 			if !ok {
 				lineName = ""