Author: Paolo Lulli <paolo@lulli.net>
add metric search request
server/db/dates_test.go | 32 +++++++++++++++++++------------- server/db/metric.go | 16 ++++++++-------- server/db/metric_info.go | 2 +- server/db/parquet.go | 8 ++++---- server/db/queries.go | 14 +++++++------- server/model/models.go | 33 +++++++++++++++++++++++---------- server/rest/rest-event.go | 4 ++-- server/rest/rest-metric.go | 24 +++++++++++++++++++++--- server/service-rest.go | 4 ++--
diff --git a/server/db/dates_test.go b/server/db/dates_test.go index 86e2aebd1823b2529546c5dd89bcbc46fa8494b5..66063081155983bb621634a60165ab335b7a96ed 100644 --- a/server/db/dates_test.go +++ b/server/db/dates_test.go @@ -36,7 +36,10 @@ currentTime := time.Now() yesterday := currentTime.AddDate(0, 0, -1) fmt.Println("today YYYY-MM-DD : ", currentTime.Format("2006-02-01")) fmt.Println("yesterday YYYY-MM-DD : ", yesterday.Format("2006-02-01")) - return dates.DaysRange{yesterday.Format("2006-02-01"), currentTime.Format("2006-02-01")} + return dates.DaysRange{ + From: yesterday.Format("2006-02-01"), + To: currentTime.Format("2006-02-01"), + } } func DaysAgo(n int) dates.DaysRange { @@ -44,30 +47,33 @@ currentTime := time.Now().AddDate(0, 0, n*(-1)) yesterday := currentTime.AddDate(0, 0, -1) fmt.Println("today YYYY-MM-DD : ", currentTime.Format("2006-02-01")) fmt.Println("yesterday YYYY-MM-DD : ", yesterday.Format("2006-02-01")) - return dates.DaysRange{yesterday.Format("2006-02-01"), currentTime.Format("2006-02-01")} + return dates.DaysRange{ + From: yesterday.Format("2006-02-01"), + To: currentTime.Format("2006-02-01"), + } } func TestParquet(t *testing.T) { - event := model.EventRequest{ - ID_client: "abc", - Etime: int64(time.Now().UnixMilli()), - Name: "SomeTest", + event := model.EventSaveRequest{ + IdClient: "abc", + Etime: int64(time.Now().UnixMilli()), + Name: "SomeTest", } tmpFile, _ := os.CreateTemp("/tmp/", "tempEvent-XXXX.parquet") - EventToParquet([]model.EventRequest{event}, tmpFile.Name()) + EventToParquet([]model.EventSaveRequest{event}, tmpFile.Name()) - metric := model.MetricRequest{ - ID_client: "mabc", - Mtime: int64(time.Now().UnixMilli()), - Name: "someName", - Value: "SomeValue", + metric := model.MetricSaveRequest{ + IdClient: "mabc", + Mtime: int64(time.Now().UnixMilli()), + Name: "someName", + Value: "SomeValue", } tmpMetricFile, _ := os.CreateTemp("/tmp/", "tempMetric-XXXX.parquet") - MetricToParquet([]model.MetricRequest{metric}, tmpMetricFile.Name()) + MetricToParquet([]model.MetricSaveRequest{metric}, tmpMetricFile.Name()) } func TestDates(t *testing.T) { diff --git a/server/db/metric.go b/server/db/metric.go index ce1280a0e76ec15bc6493bf78ae5e9906b8fd467..ccd1f566c527a4ac994977e1377a477e63539266 100644 --- a/server/db/metric.go +++ b/server/db/metric.go @@ -25,14 +25,14 @@ Name string `json:"name"` Value string `json:"value"` } -func PrintMetrics(metrics []model.MetricRequest) { +func PrintMetrics(metrics []model.MetricSaveRequest) { for _, metric := range metrics { fmt.Printf("%s\n", metric.Name) } } -func LoadMetrics(session *gocql.Session, idClient string, daysRange dates.DaysRange) []model.MetricRequest { - var metrics []model.MetricRequest +func LoadMetrics(session *gocql.Session, idClient string, daysRange dates.DaysRange) []model.MetricSaveRequest { + var metrics []model.MetricSaveRequest m := map[string]interface{}{} metricInfos := GetClientMetrics(session, idClient) @@ -44,11 +44,11 @@ q := fmt.Sprintf("SELECT id_client, mtime, name, value FROM metric where id_client='%s' and name ='%s' and mtime > '%s' and mtime < '%s'", idClient, metricName, daysRange.From, daysRange.To) fmt.Println(q) iter := session.Query(q).Iter() for iter.MapScan(m) { - metrics = append(metrics, model.MetricRequest{ - ID_client: m["id_client"].(string), - Mtime: m["mtime"].(time.Time).UnixMilli(), - Name: m["name"].(string), - Value: m["value"].(string), + metrics = append(metrics, model.MetricSaveRequest{ + IdClient: m["id_client"].(string), + Mtime: m["mtime"].(time.Time).UnixMilli(), + Name: m["name"].(string), + Value: m["value"].(string), }) m = map[string]interface{}{} diff --git a/server/db/metric_info.go b/server/db/metric_info.go index 987502176bd6a00fa53311b2fd1aa2069e479b1a..6f93d33812bc61280d9b0a157699811bc37cdb1c 100644 --- a/server/db/metric_info.go +++ b/server/db/metric_info.go @@ -26,7 +26,7 @@ fmt.Println(q) iter := session.Query(q).Iter() for iter.MapScan(m) { infos = append(infos, model.MetricInfo{ - ID_client: m["id_client"].(string), + IdClient: m["id_client"].(string), Name: m["name"].(string), Description: m["description"].(string), }) diff --git a/server/db/parquet.go b/server/db/parquet.go index d15e6984b0bdf0a599bbf7c7e064dc8e839f94d5..7f0a042fe358f952d46ff1062ad1a19242332881 100644 --- a/server/db/parquet.go +++ b/server/db/parquet.go @@ -18,7 +18,7 @@ "github.com/xitongsys/parquet-go/writer" ) -func EventToParquet(events []model.EventRequest, parquetFile string) { +func EventToParquet(events []model.EventSaveRequest, parquetFile string) { var err error w, err := os.Create(parquetFile) if err != nil { @@ -26,7 +26,7 @@ log.Println("Can't create local file", err) return } - pw, err := writer.NewParquetWriterFromWriter(w, new(model.EventRequest), 4) + pw, err := writer.NewParquetWriterFromWriter(w, new(model.EventSaveRequest), 4) if err != nil { log.Println("Can't create parquet writer", err) return @@ -46,7 +46,7 @@ log.Println("Write Finished") w.Close() } -func MetricToParquet(metrics []model.MetricRequest, parquetFile string) { +func MetricToParquet(metrics []model.MetricSaveRequest, parquetFile string) { var err error w, err := os.Create(parquetFile) @@ -55,7 +55,7 @@ log.Println("Can't create local file", err) return } - pw, err := writer.NewParquetWriterFromWriter(w, new(model.MetricRequest), 4) + pw, err := writer.NewParquetWriterFromWriter(w, new(model.MetricSaveRequest), 4) if err != nil { log.Println("Can't create parquet writer", err) return diff --git a/server/db/queries.go b/server/db/queries.go index cf158d66a694f33774983c17546bffe1dc0652c2..367dfa630af33640b48e8a670af6fcb79659953c 100644 --- a/server/db/queries.go +++ b/server/db/queries.go @@ -17,19 +17,19 @@ "time" "yats-server/model" ) -func MetricsFrom(session *gocql.Session, idClient string, metricName string, fromTime string, limit int32) []model.MetricRequest { - var metrics []model.MetricRequest +func MetricsFrom(session *gocql.Session, idClient string, metricName string, fromTime string, limit int32) []model.MetricSaveRequest { + var metrics []model.MetricSaveRequest m := map[string]interface{}{} q := fmt.Sprintf("SELECT id_client, mtime, name, value FROM metric where id_client='%s' and name ='%s' and mtime > '%s' order by mtime limit %d", idClient, metricName, fromTime, limit) fmt.Println(q) iter := session.Query(q).Iter() for iter.MapScan(m) { - metrics = append(metrics, model.MetricRequest{ - ID_client: m["id_client"].(string), - Mtime: m["mtime"].(time.Time).UnixMilli(), - Name: m["name"].(string), - Value: m["value"].(string), + metrics = append(metrics, model.MetricSaveRequest{ + IdClient: m["id_client"].(string), + Mtime: m["mtime"].(time.Time).UnixMilli(), + Name: m["name"].(string), + Value: m["value"].(string), }) m = map[string]interface{}{} diff --git a/server/model/models.go b/server/model/models.go index 95a1ee18810a6d4220fdc49dbba152411147591e..0c0b18c3897b451e339ed296843b3fda65710671 100644 --- a/server/model/models.go +++ b/server/model/models.go @@ -10,17 +10,17 @@ */ package model -type MetricRequest struct { - ID_client string `json:"id_client" parquet:"name=id_client, type=BYTE_ARRAY, convertedtype=UTF8"` - Mtime int64 `json:"mtime" parquet:"name=mtime, type=INT64, convertedtype=TIMESTAMP_MILLIS"` - Name string `json:"name" parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8"` - Value string `json:"value" parquet:"name=value, type=BYTE_ARRAY, convertedtype=UTF8"` +type MetricSaveRequest struct { + IdClient string `json:"id_client" parquet:"name=id_client, type=BYTE_ARRAY, convertedtype=UTF8"` + Mtime int64 `json:"mtime" parquet:"name=mtime, type=INT64, convertedtype=TIMESTAMP_MILLIS"` + Name string `json:"name" parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8"` + Value string `json:"value" parquet:"name=value, type=BYTE_ARRAY, convertedtype=UTF8"` } -type EventRequest struct { - ID_client string `json:"id_client" parquet:"name=id_client, type=BYTE_ARRAY, convertedtype=UTF8"` - Etime int64 `json:"etime" parquet:"name=etime, type=INT64, convertedtype=TIMESTAMP_MILLIS"` - Name string `json:"name" parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8"` +type EventSaveRequest struct { + IdClient string `json:"id_client" parquet:"name=id_client, type=BYTE_ARRAY, convertedtype=UTF8"` + Etime int64 `json:"etime" parquet:"name=etime, type=INT64, convertedtype=TIMESTAMP_MILLIS"` + Name string `json:"name" parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8"` } type ClientInfo struct { @@ -30,7 +30,20 @@ Name string } type MetricInfo struct { - ID_client string + IdClient string Name string Description string } + +type MetricSearchRequest struct { + From int64 `json:"from" parquet:"name=mtime, type=INT64, convertedtype=TIMESTAMP_MILLIS"` + To int64 `json:"to" parquet:"name=mtime, type=INT64, convertedtype=TIMESTAMP_MILLIS"` + IdClient string `json:"id_client" parquet:"name=id_client, type=BYTE_ARRAY, convertedtype=UTF8"` + Name string `json:"name" parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8"` +} + +type EventSearchRequest struct { + From int64 `json:"from" parquet:"name=mtime, type=INT64, convertedtype=TIMESTAMP_MILLIS"` + To int64 `json:"to" parquet:"name=mtime, type=INT64, convertedtype=TIMESTAMP_MILLIS"` + IdClient string `json:"id_client" parquet:"name=id_client, type=BYTE_ARRAY, convertedtype=UTF8"` +} diff --git a/server/rest/rest-event.go b/server/rest/rest-event.go index d1035f7df5e254daa01d1eb11895f338e3de219b..4832b00e9aa96c54669433b6633942ccd7b04b2f 100644 --- a/server/rest/rest-event.go +++ b/server/rest/rest-event.go @@ -20,7 +20,7 @@ "yats-server/model" ) func WriteEventNow(c *gin.Context) { - var event model.EventRequest + var event model.EventSaveRequest if err := c.BindJSON(&event); err != nil { c.IndentedJSON(http.StatusAccepted, gin.H{"ret": "-1"}) @@ -35,7 +35,7 @@ c.IndentedJSON(http.StatusAccepted, gin.H{"ret": "0"}) } func WriteEventcAt(c *gin.Context) { - var event model.EventRequest + var event model.EventSaveRequest if err := c.BindJSON(&event); err != nil { c.IndentedJSON(http.StatusAccepted, gin.H{"ret": "-1"}) diff --git a/server/rest/rest-metric.go b/server/rest/rest-metric.go index 7fa24cb07f2eda583864e0814562646bf1dd04ca..23a7422190a344c660b6a9cf2808d05252799f2f 100644 --- a/server/rest/rest-metric.go +++ b/server/rest/rest-metric.go @@ -20,7 +20,7 @@ "yats-server/model" ) func WriteMetricNow(c *gin.Context) { - var metric model.MetricRequest + var metric model.MetricSaveRequest if err := c.BindJSON(&metric); err != nil { c.IndentedJSON(http.StatusAccepted, gin.H{"ret": "-1"}) @@ -35,7 +35,7 @@ c.IndentedJSON(http.StatusAccepted, gin.H{"ret": "0"}) } func WriteMetricAt(c *gin.Context) { - var metric model.MetricRequest + var metric model.MetricSaveRequest if err := c.BindJSON(&metric); err != nil { c.IndentedJSON(http.StatusAccepted, gin.H{"ret": "-1"}) @@ -52,7 +52,25 @@ c.IndentedJSON(http.StatusAccepted, gin.H{"ret": "OK"}) } func GetMetricsFrom(c *gin.Context) { - var metric model.MetricRequest + var metric model.MetricSearchRequest + + if err := c.BindJSON(&metric); err != nil { + c.IndentedJSON(http.StatusAccepted, gin.H{"ret": "-1"}) + return + } + + clientCN := GetClientCN(c) + fmt.Printf("%s / %s ", clientCN, metric.Name) + + unixTimeUTC := time.Unix(metric.From, 0) + timeAsBytes, _ := unixTimeUTC.UTC().MarshalText() + metricsPack := db.MetricsFrom(db.Session, clientCN, metric.Name, string(timeAsBytes), 100) + + c.IndentedJSON(http.StatusAccepted, gin.H{"ret": "OK", "content": metricsPack}) +} + +func GetMetricsBetween(c *gin.Context) { + var metric model.MetricSaveRequest if err := c.BindJSON(&metric); err != nil { c.IndentedJSON(http.StatusAccepted, gin.H{"ret": "-1"}) diff --git a/server/service-rest.go b/server/service-rest.go index 1ab7c9f26d993aeea513200e0c2ea0dc9b5eea8d..f29f1cb81b0f6802f7a36dc5a41eaff7ca19f458 100644 --- a/server/service-rest.go +++ b/server/service-rest.go @@ -28,12 +28,12 @@ router.SetTrustedProxies([]string{"127.0.0.1"}) gin.SetMode(gin.ReleaseMode) - router.POST("/rt/metric", rest.WriteMetricNow) + //router.POST("/rt/metric", rest.WriteMetricNow) router.POST("/metric", rest.WriteMetricAt) router.POST("/metric/get", rest.GetMetricsFrom) - router.POST("/rt/event", rest.WriteEventNow) + //router.POST("/rt/event", rest.WriteEventNow) router.POST("/event", rest.WriteEventcAt) router.Run(address)