commit cda11cef2e69305e83e0e4ace8d54b87d3d6eff7
| author | 斟酌 鵬兄 <tgckpg@gmail.com> |
|---|
| date | 2026-02-27T06:43:02Z |
|---|
| subject | Mimic browser requests because kmb blocked Go httpGet clients |
|---|
commit cda11cef2e69305e83e0e4ace8d54b87d3d6eff7
Author: 斟酌 鵬兄 <tgckpg@gmail.com>
Date: 2026-02-27T06:43:02Z
Mimic browser requests because kmb blocked Go httpGet clients
---
datasources/kmb/routestops.go | 258 ++++++++++++++++++++----------------------
utils/shortcuts.go | 27 +++++
2 files changed, 152 insertions(+), 133 deletions(-)
diff --git a/datasources/kmb/routestops.go b/datasources/kmb/routestops.go
index 26b97f4..b5bb91d 100644
--- a/datasources/kmb/routestops.go
+++ b/datasources/kmb/routestops.go
@@ -1,147 +1,139 @@
package kmb
import (
- "bytes"
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "path/filepath"
- // "strings"
-
- query "github.com/tgckpg/golifehk/query"
- "github.com/tgckpg/golifehk/utils"
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "log"
+ "path/filepath"
+
+ query "github.com/tgckpg/golifehk/query"
+ "github.com/tgckpg/golifehk/utils"
)
-var JSON_ROUTESTOPS string = filepath.Join( utils.WORKDIR, "kmb-routestops.json" )
-var JSON_BUSSTOPS string = filepath.Join( utils.WORKDIR, "kmb-busstops.json" )
-
-func readRouteStopsData( busStops *map[string] *BusStop, buff *bytes.Buffer ) ( *[] *RouteStop, error ) {
-
- routeStopsData := RouteStops{}
- err := json.Unmarshal( buff.Bytes(), &routeStopsData )
- if err != nil {
- return nil, err
- }
-
- // routeStops[ Route ][ ServiceType ][ Direction ][ Seq ] = RouteStop
- routeStops := map[string] *map[string] *map[ string ] *map[ int ] *RouteStop{}
- allRouteStops := [] *RouteStop{}
-
- for _, entry := range routeStopsData.RouteStops {
-
- busStop := (*busStops)[ entry.StationId ]
- if busStop == nil {
- busStop = &BusStop {
- BusStopId: entry.StationId,
- Name_en: "???", Name_tc: "???", Name_sc: "???",
- }
- busStop.Reload()
-
- (*busStops)[ entry.StationId ] = busStop
- }
-
- if busStop.Routes == nil {
- busStopRoutes := [] *RouteStop{}
- busStop.Routes = &busStopRoutes
- }
-
- (*busStop.Routes) = append( (*busStop.Routes), entry )
- entry.BusStop = busStop
-
- route := routeStops[ entry.RouteId ]
- if route == nil {
- route = &map[string] *map[ string ] *map[ int ] *RouteStop{}
- routeStops[ entry.RouteId ] = route
- }
-
- service := (*route)[ entry.ServiceType ]
- if service == nil {
- service = &map[ string ] *map[ int ] *RouteStop{}
- (*route)[ entry.ServiceType ] = service
- }
-
- direction := (*service)[ entry.Direction ]
- if direction == nil {
- direction = &map[ int ] *RouteStop{}
- (*service)[ entry.Direction ] = direction
- }
- entry.RouteStops = direction
-
- seq := (*direction)[ entry.StationSeq ]
- if seq == nil {
- (*direction)[ entry.StationSeq ] = entry
- }
- allRouteStops = append( allRouteStops, entry )
- }
-
- return &allRouteStops, nil
+var JSON_ROUTESTOPS string = filepath.Join(utils.WORKDIR, "kmb-routestops.json")
+var JSON_BUSSTOPS string = filepath.Join(utils.WORKDIR, "kmb-busstops.json")
+
+func readRouteStopsData(busStops *map[string]*BusStop, buff *bytes.Buffer) (*[]*RouteStop, error) {
+
+ routeStopsData := RouteStops{}
+ err := json.Unmarshal(buff.Bytes(), &routeStopsData)
+ if err != nil {
+ return nil, err
+ }
+
+ // routeStops[ Route ][ ServiceType ][ Direction ][ Seq ] = RouteStop
+ routeStops := map[string]*map[string]*map[string]*map[int]*RouteStop{}
+ allRouteStops := []*RouteStop{}
+
+ for _, entry := range routeStopsData.RouteStops {
+
+ busStop := (*busStops)[entry.StationId]
+ if busStop == nil {
+ busStop = &BusStop{
+ BusStopId: entry.StationId,
+ Name_en: "???", Name_tc: "???", Name_sc: "???",
+ }
+ busStop.Reload()
+
+ (*busStops)[entry.StationId] = busStop
+ }
+
+ if busStop.Routes == nil {
+ busStopRoutes := []*RouteStop{}
+ busStop.Routes = &busStopRoutes
+ }
+
+ (*busStop.Routes) = append((*busStop.Routes), entry)
+ entry.BusStop = busStop
+
+ route := routeStops[entry.RouteId]
+ if route == nil {
+ route = &map[string]*map[string]*map[int]*RouteStop{}
+ routeStops[entry.RouteId] = route
+ }
+
+ service := (*route)[entry.ServiceType]
+ if service == nil {
+ service = &map[string]*map[int]*RouteStop{}
+ (*route)[entry.ServiceType] = service
+ }
+
+ direction := (*service)[entry.Direction]
+ if direction == nil {
+ direction = &map[int]*RouteStop{}
+ (*service)[entry.Direction] = direction
+ }
+ entry.RouteStops = direction
+
+ seq := (*direction)[entry.StationSeq]
+ if seq == nil {
+ (*direction)[entry.StationSeq] = entry
+ }
+ allRouteStops = append(allRouteStops, entry)
+ }
+
+ return &allRouteStops, nil
}
-func readBusStopsData( buff *bytes.Buffer ) ( *map[string]*BusStop, error ) {
- busStopsData := BusStops{}
- err := json.Unmarshal( buff.Bytes(), &busStopsData )
- if err != nil {
- return nil, err
- }
+func readBusStopsData(buff *bytes.Buffer) (*map[string]*BusStop, error) {
+ busStopsData := BusStops{}
+ err := json.Unmarshal(buff.Bytes(), &busStopsData)
+ if err != nil {
+ return nil, err
+ }
- busStopMap := map[string] *BusStop{}
- for _, entry := range busStopsData.BusStops {
+ busStopMap := map[string]*BusStop{}
+ for _, entry := range busStopsData.BusStops {
- entry.Reload()
+ entry.Reload()
- if _, ok := busStopMap[ entry.BusStopId ]; ok {
- return nil, fmt.Errorf( "Duplicated BusStop: %s", entry.BusStopId )
- }
+ if _, ok := busStopMap[entry.BusStopId]; ok {
+ return nil, fmt.Errorf("Duplicated BusStop: %s", entry.BusStopId)
+ }
- busStopMap[ entry.BusStopId ] = entry
- }
- return &busStopMap, nil
+ busStopMap[entry.BusStopId] = entry
+ }
+ return &busStopMap, nil
}
-func getRouteStops() (*[] query.ISearchable, error) {
-
- QUERY_FUNC := func() ( io.ReadCloser, error ) {
- resp, err := http.Get( "https://data.etabus.gov.hk/v1/transport/kmb/stop" )
- if err != nil {
- return nil, err
- }
- return resp.Body, nil
- }
-
- buff, err := utils.CacheStream( JSON_BUSSTOPS, QUERY_FUNC, 7 * 24 * 3600 )
- if err != nil {
- return nil, err
- }
-
- busStopMap, err := readBusStopsData( buff )
- if err != nil {
- return nil, err
- }
-
- QUERY_FUNC = func() ( io.ReadCloser, error ) {
- resp, err := http.Get( "https://data.etabus.gov.hk/v1/transport/kmb/route-stop" )
- if err != nil {
- return nil, err
- }
- return resp.Body, nil
- }
-
- buff, err = utils.CacheStream( JSON_ROUTESTOPS, QUERY_FUNC, 7 * 24 * 3600 )
- if err != nil {
- return nil, err
- }
-
- routeStops, err := readRouteStopsData( busStopMap, buff )
- if err != nil {
- return nil, err
- }
-
- searchables := [] query.ISearchable{}
- for _, routeStop := range *routeStops {
- searchables = append( searchables, routeStop )
- routeStop.Reload()
- }
-
- return &searchables, nil
+func getRouteStops() (*[]query.ISearchable, error) {
+
+ QUERY_FUNC := func() (io.ReadCloser, error) {
+ return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/stop")
+ }
+
+ buff, err := utils.CacheStream(JSON_BUSSTOPS, QUERY_FUNC, 7*24*3600)
+ if err != nil {
+ return nil, err
+ }
+
+ busStopMap, err := readBusStopsData(buff)
+ if err != nil {
+ return nil, err
+ }
+
+ QUERY_FUNC = func() (io.ReadCloser, error) {
+ return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/route-stop")
+ }
+
+ buff, err = utils.CacheStream(JSON_ROUTESTOPS, QUERY_FUNC, 7*24*3600)
+ if err != nil {
+ return nil, err
+ }
+
+ routeStops, err := readRouteStopsData(busStopMap, buff)
+ if err != nil {
+ log.Printf("Unable to parse RouteStopsData: %s", err)
+ return nil, err
+ }
+
+ searchables := []query.ISearchable{}
+ for _, routeStop := range *routeStops {
+ searchables = append(searchables, routeStop)
+ routeStop.Reload()
+ }
+
+ return &searchables, nil
}
diff --git a/utils/shortcuts.go b/utils/shortcuts.go
index be95073..7aa2ab1 100644
--- a/utils/shortcuts.go
+++ b/utils/shortcuts.go
@@ -2,9 +2,12 @@ package utils
import (
"bytes"
+ "io"
+ "net/http"
"os"
"strconv"
"strings"
+ "time"
)
var MARKDOWN_ESC []string = []string{"_", "*", "[", "]", "(", ")", "~", "`", ">", "#", "+", "-", "=", "|", "{", "}", ".", "!"}
@@ -117,3 +120,27 @@ func TryGetEnv[T any](name string, fallback T) T {
return fallback
}
+
+func HttpGet(url string) (io.ReadCloser, error) {
+ client := &http.Client{
+ Timeout: 15 * time.Second,
+ }
+
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ ua := "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
+ req.Header.Set("User-Agent", ua)
+ req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
+ req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+ req.Header.Set("Connection", "keep-alive")
+
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+
+ return resp.Body, nil
+}