yats.git

commit e0ca0badc0fb9294211c1f8dd8df34d8e06c31a9

Author: Paolo Lulli <paolo@lulli.net>

Merge branch 'master' of dev.lulli.net:customer/net.lulli/yats

 client/io.go | 6 ++
 client/main.go | 113 +++++++++++++++++++++++----------------
 server/rest/rest-metric.go | 3 


diff --git a/client/io.go b/client/io.go
index 450f0865ff95312f52a30271fde31a658c717fa1..32f8e2784af121598830f93fdc00211f0d670d70 100644
--- a/client/io.go
+++ b/client/io.go
@@ -29,6 +29,12 @@ 	json.Unmarshal([]byte(decoded), &dataMap)
 	return dataMap, false
 }
 
+func getMaxpage(list string) int64 {
+	var result map[string]int64
+	json.Unmarshal([]byte(list), &result)
+	return result["maxpage"]
+}
+
 func getDataJson(list string) ([]byte, bool) {
 	var result map[string]interface{}
 	json.Unmarshal([]byte(list), &result)




diff --git a/client/main.go b/client/main.go
index bf1d6a8de418282082ae3500713031f5ebe343ad..e48659d360b0e653e0a6001337e73b23042f33ec 100644
--- a/client/main.go
+++ b/client/main.go
@@ -162,23 +162,20 @@ 	}
 
 	if *listEvents {
 		listEvents := yatsClient.EventList(*sourceOption, fromTimestamp, toTimestamp)
-		if *reformatOutput {
-			dataJson, errDataPortion := getDataJson(listEvents)
-			if errDataPortion {
-				fmt.Printf("Could not parse data attribute: %v", errDataPortion)
-				return
-			}
+		eventListModel, maxpage, _ := responseToEventArray(listEvents)
+		var composedList []EventModel
+		composedList = append(composedList, eventListModel...)
 
-			var eventListModel []EventModel
-			err := json.Unmarshal(dataJson, &eventListModel)
+		for (len(eventListModel) == 100) && !stopSearching(maxpage, toTimestamp) {
+			listEvents = yatsClient.EventList(*sourceOption, maxpage/1000, toTimestamp)
+			eventListModel, maxpage, _ = responseToEventArray(listEvents)
+			composedList = append(composedList, eventListModel...)
+		}
 
-			if err == nil {
-				for _, event := range eventListModel {
-					eventTime := formattedUtcTimeInSec(event.Etime)
-					fmt.Printf("%s %s\n", eventTime, event.Name)
-				}
-			} else {
-				fmt.Println(err)
+		if *reformatOutput {
+			for _, event := range composedList {
+				eventTime := formattedUtcTimeInSec(event.Etime)
+				fmt.Printf("%s %s\n", eventTime, event.Name)
 			}
 		} else {
 			fmt.Printf("%s", listEvents)
@@ -188,21 +185,20 @@ 	}
 
 	if "" != *listMetrics {
 		listResponse, err := yatsClient.MetricList(*sourceOption, *listMetrics, fromTimestamp, toTimestamp)
+		metricListModel, maxpage, _ := responseToMetricArray(listResponse)
+		var composedList []MetricModel
+		composedList = append(composedList, metricListModel...)
+
+		for (len(metricListModel) == 100) && !stopSearching(maxpage, toTimestamp) {
+			listResponse, _ = yatsClient.MetricList(*sourceOption, *listMetrics, maxpage/1000, toTimestamp)
+			metricListModel, maxpage, _ = responseToMetricArray(listResponse)
+			composedList = append(composedList, metricListModel...)
+		}
 
 		if err == nil {
 			if *reformatOutput {
-
-				dataJson, errDataPortion := getDataJson(listResponse)
-				if errDataPortion {
-					fmt.Printf("Could not parse data attribute: %v", errDataPortion)
-					return
-				}
-
-				var metricListModel []MetricModel
-				err := json.Unmarshal(dataJson, &metricListModel)
-
 				if err == nil {
-					for _, metric := range metricListModel {
+					for _, metric := range composedList {
 						fmt.Printf("%d %s=%s\n", metric.Mtime, metric.Name, metric.Value)
 					}
 				} else {
@@ -211,17 +207,8 @@ 				}
 				os.Exit(0)
 			}
 			if *logOutput {
-				dataJson, errDataPortion := getDataJson(listResponse)
-				if errDataPortion {
-					fmt.Printf("Could not parse data attribute: %v", errDataPortion)
-					return
-				}
-
-				var metricListModel []MetricModel
-				err := json.Unmarshal(dataJson, &metricListModel)
-
 				if err == nil {
-					for _, metric := range metricListModel {
+					for _, metric := range composedList {
 						if *decodeBase64Output {
 							decodedBytes, err := base64.StdEncoding.DecodeString(metric.Value)
 							if err != nil {
@@ -240,17 +227,8 @@ 				os.Exit(0)
 			}
 
 			if *unpackJsonPayload {
-				dataJson, errDataPortion := getDataJson(listResponse)
-				if errDataPortion {
-					fmt.Printf("Could not parse data attribute: %v", errDataPortion)
-					return
-				}
-
-				var metricListModel []MetricModel
-				err := json.Unmarshal(dataJson, &metricListModel)
-
 				if err == nil {
-					metric := metricListModel[0]
+					metric := composedList[0]
 					dataMap, errNoValue := mapFromBase64(metric.Value)
 					if errNoValue {
 						return
@@ -266,7 +244,7 @@ 					for _, k := range keys {
 						fmt.Printf("\"%s\"%s", k, recordSeparator)
 					}
 					fmt.Printf("%s", rowSeparator)
-					for _, row := range metricListModel {
+					for _, row := range composedList {
 						rowMap, errNoValue := mapFromBase64(row.Value)
 						if errNoValue {
 							return
@@ -299,3 +277,44 @@ 		fmt.Printf("%s", listResponse)
 		os.Exit(0)
 	}
 }
+
+func responseToMetricArray(listResponse string) ([]MetricModel, int64, bool) {
+	dataJson, errDataPortion := getDataJson(listResponse)
+	maxpage := getMaxpage(listResponse)
+	if errDataPortion {
+		fmt.Printf("Could not parse data attribute: %v", errDataPortion)
+		return nil, 0, true
+	}
+
+	var metricListModel []MetricModel
+	err := json.Unmarshal(dataJson, &metricListModel)
+	if err == nil {
+		return metricListModel, maxpage, true
+	}
+	return metricListModel, maxpage, false
+}
+
+func responseToEventArray(listResponse string) ([]EventModel, int64, bool) {
+	dataJson, errDataPortion := getDataJson(listResponse)
+	maxpage := getMaxpage(listResponse)
+	if errDataPortion {
+		fmt.Printf("Could not parse data attribute: %v", errDataPortion)
+		return nil, 0, true
+	}
+
+	var eventListModel []EventModel
+	err := json.Unmarshal(dataJson, &eventListModel)
+	if err == nil {
+		return eventListModel, maxpage, true
+	}
+	return eventListModel, maxpage, false
+}
+func stopSearching(maxpage int64, toTimestamp int64) bool {
+	if toTimestamp == 0 {
+		return false
+	}
+	if maxpage >= toTimestamp {
+		return false
+	}
+	return true
+}




diff --git a/server/rest/rest-metric.go b/server/rest/rest-metric.go
index d0e04554a17f92231017dd5f6746e726c4f997e7..4a75165c163115bf77c9f7ce47ded705dddf2c33 100644
--- a/server/rest/rest-metric.go
+++ b/server/rest/rest-metric.go
@@ -12,7 +12,6 @@ package rest
 
 import (
 	"fmt"
-	"github.com/gin-gonic/gin"
 	"log"
 	"net/http"
 	"os"
@@ -22,6 +21,8 @@ 	"yats-server/config"
 	"yats-server/db"
 	"yats-server/model"
 	"yats-server/util"
+
+	"github.com/gin-gonic/gin"
 )
 
 var (