yats.git

commit 50c609d05a1149eef5535c33306198163e2bb041

Author: Paolo Lulli <paolo@lulli.net>

Multiple calls for 100 rows pages

 client/io.go | 6 +++
 client/main.go | 71 +++++++++++++++++++++------------------
 server/rest/rest-metric.go | 9 ++--


diff --git a/client/io.go b/client/io.go
index 450f0865ff95312f52a30271fde31a658c717fa1..32f8e2784af121598830f93fdc00211f0d670d70 100644
--- a/client/io.go
+++ b/client/io.go
@@ -29,6 +29,12 @@ 	json.Unmarshal([]byte(decoded), &dataMap)
 	return dataMap, false
 }
 
+func getMaxpage(list string) int64 {
+	var result map[string]int64
+	json.Unmarshal([]byte(list), &result)
+	return result["maxpage"]
+}
+
 func getDataJson(list string) ([]byte, bool) {
 	var result map[string]interface{}
 	json.Unmarshal([]byte(list), &result)




diff --git a/client/main.go b/client/main.go
index bf1d6a8de418282082ae3500713031f5ebe343ad..4dd423392144d63b74bdb3d1c257cc1a61835a83 100644
--- a/client/main.go
+++ b/client/main.go
@@ -188,21 +188,20 @@ 	}
 
 	if "" != *listMetrics {
 		listResponse, err := yatsClient.MetricList(*sourceOption, *listMetrics, fromTimestamp, toTimestamp)
+		metricListModel, maxpage, _ := responseToArray(listResponse)
+		var composedList []MetricModel
+		composedList = append(composedList, metricListModel...)
+
+		for (len(metricListModel) == 100) && !stopSearching(maxpage, toTimestamp) {
+			listResponse, _ = yatsClient.MetricList(*sourceOption, *listMetrics, maxpage/1000, toTimestamp)
+			metricListModel, maxpage, _ = responseToArray(listResponse)
+			composedList = append(composedList, metricListModel...)
+		}
 
 		if err == nil {
 			if *reformatOutput {
-
-				dataJson, errDataPortion := getDataJson(listResponse)
-				if errDataPortion {
-					fmt.Printf("Could not parse data attribute: %v", errDataPortion)
-					return
-				}
-
-				var metricListModel []MetricModel
-				err := json.Unmarshal(dataJson, &metricListModel)
-
 				if err == nil {
-					for _, metric := range metricListModel {
+					for _, metric := range composedList {
 						fmt.Printf("%d %s=%s\n", metric.Mtime, metric.Name, metric.Value)
 					}
 				} else {
@@ -211,17 +210,8 @@ 				}
 				os.Exit(0)
 			}
 			if *logOutput {
-				dataJson, errDataPortion := getDataJson(listResponse)
-				if errDataPortion {
-					fmt.Printf("Could not parse data attribute: %v", errDataPortion)
-					return
-				}
-
-				var metricListModel []MetricModel
-				err := json.Unmarshal(dataJson, &metricListModel)
-
 				if err == nil {
-					for _, metric := range metricListModel {
+					for _, metric := range composedList {
 						if *decodeBase64Output {
 							decodedBytes, err := base64.StdEncoding.DecodeString(metric.Value)
 							if err != nil {
@@ -240,17 +230,8 @@ 				os.Exit(0)
 			}
 
 			if *unpackJsonPayload {
-				dataJson, errDataPortion := getDataJson(listResponse)
-				if errDataPortion {
-					fmt.Printf("Could not parse data attribute: %v", errDataPortion)
-					return
-				}
-
-				var metricListModel []MetricModel
-				err := json.Unmarshal(dataJson, &metricListModel)
-
 				if err == nil {
-					metric := metricListModel[0]
+					metric := composedList[0]
 					dataMap, errNoValue := mapFromBase64(metric.Value)
 					if errNoValue {
 						return
@@ -266,7 +247,7 @@ 					for _, k := range keys {
 						fmt.Printf("\"%s\"%s", k, recordSeparator)
 					}
 					fmt.Printf("%s", rowSeparator)
-					for _, row := range metricListModel {
+					for _, row := range composedList {
 						rowMap, errNoValue := mapFromBase64(row.Value)
 						if errNoValue {
 							return
@@ -299,3 +280,29 @@ 		fmt.Printf("%s", listResponse)
 		os.Exit(0)
 	}
 }
+
+func responseToArray(listResponse string) ([]MetricModel, int64, bool) {
+	dataJson, errDataPortion := getDataJson(listResponse)
+	maxpage := getMaxpage(listResponse)
+	if errDataPortion {
+		fmt.Printf("Could not parse data attribute: %v", errDataPortion)
+		return nil, 0, true
+	}
+
+	var metricListModel []MetricModel
+	err := json.Unmarshal(dataJson, &metricListModel)
+	if err == nil {
+		return metricListModel, maxpage, true
+	}
+	return metricListModel, maxpage, false
+}
+
+func stopSearching(maxpage int64, toTimestamp int64) bool {
+	if toTimestamp == 0 {
+		return false
+	}
+	if toTimestamp > maxpage {
+		return false
+	}
+	return true
+}




diff --git a/server/rest/rest-metric.go b/server/rest/rest-metric.go
index adf3a8bb509e93877f190b0fc7df03c8c338d110..0cbb06193ed29dcc4a01e5107718966061382b99 100644
--- a/server/rest/rest-metric.go
+++ b/server/rest/rest-metric.go
@@ -12,7 +12,6 @@ package rest
 
 import (
 	"fmt"
-	"github.com/gin-gonic/gin"
 	"log"
 	"net/http"
 	"os"
@@ -22,12 +21,14 @@ 	"yats-server/config"
 	"yats-server/db"
 	"yats-server/model"
 	"yats-server/util"
+
+	"github.com/gin-gonic/gin"
 )
 
 var (
 	//outfile, _ = os.Create(configuration.LogFile)
-	LOG = log.New(os.Stdout, "[YATS] ", log.Ldate|log.Ltime)
-	var MaxPageSize int32 = 100
+	LOG               = log.New(os.Stdout, "[YATS] ", log.Ldate|log.Ltime)
+	MaxPageSize int32 = 100
 )
 
 // @BasePath /
@@ -144,4 +145,4 @@
 		maxpage := metricsPack[len(metricsPack)-1].Mtime
 		JsonPrint(cfg, c, http.StatusAccepted, gin.H{"maxpage": maxpage, "data": metricsPack})
 	}
-}
\ No newline at end of file
+}