commit afb73b70ffbd9065fc762a78ae457d671398bd56
| author | 斟酌 鵬兄 <tgckpg@gmail.com> |
|---|
| date | 2022-09-14T09:12:48Z |
|---|
| subject | Generalize cache get with expire param |
|---|
commit afb73b70ffbd9065fc762a78ae457d671398bd56
Author: 斟酌 鵬兄 <tgckpg@gmail.com>
Date: 2022-09-14T09:12:48Z
Generalize cache get with expire param
---
.gitignore | 1 +
businfo/mtr/busstops_test.go | 22 --------
businfo/mtr/k66.json | 0
.../mtr => datasources/mtr/bus}/busschedule.go | 56 +++++++++---------
.../mtr/bus}/busschedule_test.go | 2 +-
{businfo/mtr => datasources/mtr/bus}/busstops.go | 63 ++++-----------------
datasources/mtr/bus/busstops_test.go | 8 +++
{businfo/mtr => datasources/mtr/bus}/query.go | 18 +++++-
{businfo/mtr => datasources/mtr/bus}/query_test.go | 4 +-
utils/cache.go | 66 ++++++++++++++++++++++
10 files changed, 130 insertions(+), 110 deletions(-)
diff --git a/.gitignore b/.gitignore
index 343a25c..3c9b48b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
*.csv
*.json
+*.swp
diff --git a/businfo/mtr/busstops_test.go b/businfo/mtr/busstops_test.go
deleted file mode 100644
index aa81062..0000000
--- a/businfo/mtr/busstops_test.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package mtr
-
-import (
- "fmt"
- "testing"
-)
-
-func TestAll(t *testing.T) {
- /*
- entries, err := pullRemote()
- if err != nil {
- t.Error( err )
- }
- fmt.Println( entries )
- */
-
- entries, err := pullLocal()
- if err != nil {
- t.Error( err )
- }
- fmt.Println( entries )
-}
diff --git a/businfo/mtr/k66.json b/businfo/mtr/k66.json
deleted file mode 100644
index e69de29..0000000
diff --git a/businfo/mtr/busschedule.go b/datasources/mtr/bus/busschedule.go
similarity index 55%
rename from businfo/mtr/busschedule.go
rename to datasources/mtr/bus/busschedule.go
index c796394..2734a3b 100644
--- a/businfo/mtr/busschedule.go
+++ b/datasources/mtr/bus/busschedule.go
@@ -1,12 +1,13 @@
-package mtr
+package bus
import (
- "fmt"
- // "byte"
- // "net/http"
- "io"
- "os"
+ "bytes"
"encoding/json"
+ "io"
+ "net/http"
+ "path/filepath"
+
+ "github.com/tgckpg/golifehk/utils"
)
type Location struct {
@@ -38,41 +39,38 @@ type BusSchedule struct {
func getSchedule( lang string, routeName string ) ( *BusSchedule, error ) {
- /*
- values := map[string]string { "language": lang , "routeName": routeName }
-
- jsonValue, _ := json.Marshal(values)
-
- resp, err := http.Post(
- "https://rt.data.gov.hk/v1/transport/mtr/bus/getSchedule",
- "application/json",
- bytes.NewBuffer( jsonValue ),
+ CACHE_PATH := filepath.Join(
+ utils.WORKDIR,
+ "mtr_bsch" + "-" + lang + "-" + routeName + ".json",
)
- if err != nil {
- return nil, err
+ QUERY_FUNC := func() ( io.ReadCloser, error ) {
+ // Query Remote
+ values := map[string]string { "language": lang , "routeName": routeName }
+ jsonValue, _ := json.Marshal(values)
+ resp, err := http.Post(
+ "https://rt.data.gov.hk/v1/transport/mtr/bus/getSchedule",
+ "application/json",
+ bytes.NewBuffer( jsonValue ),
+ )
+
+ if err != nil {
+ return nil, err
+ }
+
+ return resp.Body, nil
}
- defer resp.Body.Close()
-
- data, err := io.ReadAll( resp.Body )
- /*/
- f, err := os.Open( "abc.json" )
- defer f.Close()
-
- data, err := io.ReadAll( f )
- //*/
+ buff, err := utils.CacheStream( CACHE_PATH, QUERY_FUNC, 60 )
if err != nil {
return nil, err
}
schedules := BusSchedule{}
-
- err = json.Unmarshal( data, &schedules )
+ err = json.Unmarshal( buff.Bytes(), &schedules )
if err != nil {
return nil, err
}
- fmt.Printf( "%+v", schedules )
return &schedules, nil
}
diff --git a/businfo/mtr/busschedule_test.go b/datasources/mtr/bus/busschedule_test.go
similarity index 92%
rename from businfo/mtr/busschedule_test.go
rename to datasources/mtr/bus/busschedule_test.go
index 9f8ccf6..e48102a 100644
--- a/businfo/mtr/busschedule_test.go
+++ b/datasources/mtr/bus/busschedule_test.go
@@ -1,4 +1,4 @@
-package mtr
+package bus
import (
"testing"
diff --git a/businfo/mtr/busstops.go b/datasources/mtr/bus/busstops.go
similarity index 65%
rename from businfo/mtr/busstops.go
rename to datasources/mtr/bus/busstops.go
index a1a05ad..aa807bc 100644
--- a/businfo/mtr/busstops.go
+++ b/datasources/mtr/bus/busstops.go
@@ -1,11 +1,10 @@
-package mtr
+package bus
import (
"encoding/csv"
"fmt"
"io"
"net/http"
- "os"
"path/filepath"
"strconv"
"strings"
@@ -26,7 +25,7 @@ type BusStop struct {
var mBusStops *map[string]BusStop
-var DEF_CSV string = filepath.Join( utils.WORKDIR, "mtr_bus_stops.csv" )
+var CSV_BUSSTOPS string = filepath.Join( utils.WORKDIR, "mtr_bus_stops.csv" )
func readBusStopData( r io.Reader ) ( *map[string]BusStop, error ) {
@@ -82,60 +81,18 @@ func readBusStopData( r io.Reader ) ( *map[string]BusStop, error ) {
func getBusStops() (*map[string]BusStop, error) {
- if mBusStops != nil {
- return mBusStops, nil
- }
-
- mBusStops, err := pullLocal()
- if mBusStops != nil {
- return mBusStops, err
- }
-
- mBusStops, err = pullRemote()
- if mBusStops != nil {
- return mBusStops, err
- }
-
- return nil, err
-}
-
-func pullRemote() (*map[string]BusStop, error) {
-
- err := os.MkdirAll( filepath.Dir( DEF_CSV ), 0750 )
- if err != nil {
- return nil, err
- }
-
- resp, err := http.Get( "https://opendata.mtr.com.hk/data/mtr_bus_stops.csv" )
- if err != nil {
- return nil, err
- }
-
- defer resp.Body.Close()
-
- f, err := os.OpenFile( DEF_CSV, os.O_CREATE | os.O_RDWR, 0644 )
- if err != nil {
- return nil, err
- }
-
- defer f.Close()
-
- _, err = io.Copy( f, resp.Body )
- if err != nil {
- return nil, err
+ QUERY_FUNC := func() ( io.ReadCloser, error ) {
+ resp, err := http.Get( "https://opendata.mtr.com.hk/data/mtr_bus_stops.csv" )
+ if err != nil {
+ return nil, err
+ }
+ return resp.Body, nil
}
- f.Seek( 0, 0 )
- return readBusStopData( f )
-}
-
-func pullLocal() (*map[string]BusStop, error) {
- f, err := os.Open( DEF_CSV )
+ buff, err := utils.CacheStream( CSV_BUSSTOPS, QUERY_FUNC, 7 * 24 * 3600 )
if err != nil {
return nil, err
}
- defer f.Close()
-
- return readBusStopData( f )
+ return readBusStopData( buff )
}
diff --git a/datasources/mtr/bus/busstops_test.go b/datasources/mtr/bus/busstops_test.go
new file mode 100644
index 0000000..5d3c382
--- /dev/null
+++ b/datasources/mtr/bus/busstops_test.go
@@ -0,0 +1,8 @@
+package bus
+
+import (
+ "testing"
+)
+
+func TestAll(t *testing.T) {
+}
diff --git a/businfo/mtr/query.go b/datasources/mtr/bus/query.go
similarity index 82%
rename from businfo/mtr/query.go
rename to datasources/mtr/bus/query.go
index e0e61d6..c2122e3 100644
--- a/businfo/mtr/query.go
+++ b/datasources/mtr/bus/query.go
@@ -1,4 +1,4 @@
-package mtr
+package bus
import (
"fmt"
@@ -8,7 +8,12 @@ import (
type QueryObject struct {
Route string
+ BusStops *[]BusStop
+}
+
+type QueryResult struct {
BusStops []BusStop
+ err string
}
type Term struct {
@@ -16,6 +21,10 @@ type Term struct {
ProblyRoute bool
}
+func Query( message string ) *QueryResult {
+ return &QueryResult{ err: "No Result" }
+}
+
func test( entry BusStop, val string ) bool {
switch true {
case strings.Contains( entry.Name_zhant, val ):
@@ -37,6 +46,7 @@ func parse( line string ) ( *QueryObject, error ) {
var searches = []string{}
matches := []BusStop{}
+ // Sanitize and assume properties for each of the keywords
terms := []Term{}
for _, val := range strings.Split( line, " " ) {
val = strings.ToUpper( strings.Trim( val, " " ) )
@@ -47,6 +57,7 @@ func parse( line string ) ( *QueryObject, error ) {
terms = append( terms, term )
}
+ // Search for route name first, otherwise search in other props
for _, entry := range *busStops {
// Search for RouteId
@@ -69,6 +80,8 @@ func parse( line string ) ( *QueryObject, error ) {
}
}
+ // If route found, filter out all other route
+ // then search within that route
if route != "" && 0 < len( searches ) {
matches_in := []BusStop{}
for _, entry := range matches {
@@ -86,6 +99,5 @@ func parse( line string ) ( *QueryObject, error ) {
matches = matches_in
}
-
- return &QueryObject{ Route: route, BusStops: matches }, err
+ return &QueryObject{ Route: route, BusStops: &matches }, err
}
diff --git a/businfo/mtr/query_test.go b/datasources/mtr/bus/query_test.go
similarity index 80%
rename from businfo/mtr/query_test.go
rename to datasources/mtr/bus/query_test.go
index db0feb0..131c852 100644
--- a/businfo/mtr/query_test.go
+++ b/datasources/mtr/bus/query_test.go
@@ -1,4 +1,4 @@
-package mtr
+package bus
import (
"fmt"
@@ -11,6 +11,6 @@ func TestQuerySchedule(t *testing.T) {
t.Error( err )
}
- fmt.Println( qo )
+ fmt.Printf( "%+v", qo )
}
diff --git a/utils/cache.go b/utils/cache.go
new file mode 100644
index 0000000..00540fc
--- /dev/null
+++ b/utils/cache.go
@@ -0,0 +1,66 @@
+package utils
+
+import (
+ "bytes"
+ "io"
+ "os"
+ "path/filepath"
+ "time"
+)
+
+
+func CacheStream( path string, readStream func() ( io.ReadCloser, error ), expires int ) ( *bytes.Buffer, error ) {
+
+ cache, err := os.Stat( path )
+
+ // Check if cache exists and not expired
+ if err == nil {
+ expired := cache.ModTime().Add( 60 * 1e9 )
+ if time.Now().Before( expired ) {
+ f, err := os.Open( path )
+ if err == nil {
+ defer f.Close()
+ writeBuff := bytes.NewBuffer( []byte{} )
+ _, err = io.Copy( writeBuff, f )
+ if err == nil {
+ return writeBuff, nil
+ }
+ }
+ }
+ }
+
+ err = os.MkdirAll( filepath.Dir( path ), 0750 )
+ if err != nil {
+ return nil, err
+ }
+
+ writeBuff := bytes.NewBuffer( []byte{} )
+
+ // Get the reader that return new data
+ s, err := readStream()
+ if err != nil {
+ return nil, err
+ }
+
+ defer s.Close()
+
+ _, err = io.Copy( writeBuff, s )
+ if err != nil {
+ return nil, err
+ }
+
+ f, err := os.OpenFile( path, os.O_CREATE | os.O_WRONLY | os.O_TRUNC, 0644 )
+ if err != nil {
+ return nil, err
+ }
+
+ defer f.Close()
+
+ data := writeBuff.Bytes()
+ _, err = io.Copy( f, bytes.NewReader( data ) )
+ if err != nil {
+ return nil, err
+ }
+
+ return writeBuff, nil
+}