Author: Paolo Lulli <paolo@lulli.net>
Multiple calls for 100 rows pages
client/io.go | 6 +++ client/main.go | 71 +++++++++++++++++++++------------------ server/rest/rest-metric.go | 9 ++--
diff --git a/client/io.go b/client/io.go index 450f0865ff95312f52a30271fde31a658c717fa1..32f8e2784af121598830f93fdc00211f0d670d70 100644 --- a/client/io.go +++ b/client/io.go @@ -29,6 +29,12 @@ json.Unmarshal([]byte(decoded), &dataMap) return dataMap, false } +func getMaxpage(list string) int64 { + var result map[string]int64 + json.Unmarshal([]byte(list), &result) + return result["maxpage"] +} + func getDataJson(list string) ([]byte, bool) { var result map[string]interface{} json.Unmarshal([]byte(list), &result) diff --git a/client/main.go b/client/main.go index bf1d6a8de418282082ae3500713031f5ebe343ad..4dd423392144d63b74bdb3d1c257cc1a61835a83 100644 --- a/client/main.go +++ b/client/main.go @@ -188,21 +188,20 @@ } if "" != *listMetrics { listResponse, err := yatsClient.MetricList(*sourceOption, *listMetrics, fromTimestamp, toTimestamp) + metricListModel, maxpage, _ := responseToArray(listResponse) + var composedList []MetricModel + composedList = append(composedList, metricListModel...) + + for (len(metricListModel) == 100) && !stopSearching(maxpage, toTimestamp) { + listResponse, _ = yatsClient.MetricList(*sourceOption, *listMetrics, maxpage/1000, toTimestamp) + metricListModel, maxpage, _ = responseToArray(listResponse) + composedList = append(composedList, metricListModel...) + } if err == nil { if *reformatOutput { - - dataJson, errDataPortion := getDataJson(listResponse) - if errDataPortion { - fmt.Printf("Could not parse data attribute: %v", errDataPortion) - return - } - - var metricListModel []MetricModel - err := json.Unmarshal(dataJson, &metricListModel) - if err == nil { - for _, metric := range metricListModel { + for _, metric := range composedList { fmt.Printf("%d %s=%s\n", metric.Mtime, metric.Name, metric.Value) } } else { @@ -211,17 +210,8 @@ } os.Exit(0) } if *logOutput { - dataJson, errDataPortion := getDataJson(listResponse) - if errDataPortion { - fmt.Printf("Could not parse data attribute: %v", errDataPortion) - return - } - - var metricListModel []MetricModel - err := json.Unmarshal(dataJson, &metricListModel) - if err == nil { - for _, metric := range metricListModel { + for _, metric := range composedList { if *decodeBase64Output { decodedBytes, err := base64.StdEncoding.DecodeString(metric.Value) if err != nil { @@ -240,17 +230,8 @@ os.Exit(0) } if *unpackJsonPayload { - dataJson, errDataPortion := getDataJson(listResponse) - if errDataPortion { - fmt.Printf("Could not parse data attribute: %v", errDataPortion) - return - } - - var metricListModel []MetricModel - err := json.Unmarshal(dataJson, &metricListModel) - if err == nil { - metric := metricListModel[0] + metric := composedList[0] dataMap, errNoValue := mapFromBase64(metric.Value) if errNoValue { return @@ -266,7 +247,7 @@ for _, k := range keys { fmt.Printf("\"%s\"%s", k, recordSeparator) } fmt.Printf("%s", rowSeparator) - for _, row := range metricListModel { + for _, row := range composedList { rowMap, errNoValue := mapFromBase64(row.Value) if errNoValue { return @@ -299,3 +280,29 @@ fmt.Printf("%s", listResponse) os.Exit(0) } } + +func responseToArray(listResponse string) ([]MetricModel, int64, bool) { + dataJson, errDataPortion := getDataJson(listResponse) + maxpage := getMaxpage(listResponse) + if errDataPortion { + fmt.Printf("Could not parse data attribute: %v", errDataPortion) + return nil, 0, true + } + + var metricListModel []MetricModel + err := json.Unmarshal(dataJson, &metricListModel) + if err == nil { + return metricListModel, maxpage, true + } + return metricListModel, maxpage, false +} + +func stopSearching(maxpage int64, toTimestamp int64) bool { + if toTimestamp == 0 { + return false + } + if toTimestamp > maxpage { + return false + } + return true +} diff --git a/server/rest/rest-metric.go b/server/rest/rest-metric.go index adf3a8bb509e93877f190b0fc7df03c8c338d110..0cbb06193ed29dcc4a01e5107718966061382b99 100644 --- a/server/rest/rest-metric.go +++ b/server/rest/rest-metric.go @@ -12,7 +12,6 @@ package rest import ( "fmt" - "github.com/gin-gonic/gin" "log" "net/http" "os" @@ -22,12 +21,14 @@ "yats-server/config" "yats-server/db" "yats-server/model" "yats-server/util" + + "github.com/gin-gonic/gin" ) var ( //outfile, _ = os.Create(configuration.LogFile) - LOG = log.New(os.Stdout, "[YATS] ", log.Ldate|log.Ltime) - var MaxPageSize int32 = 100 + LOG = log.New(os.Stdout, "[YATS] ", log.Ldate|log.Ltime) + MaxPageSize int32 = 100 ) // @BasePath / @@ -144,4 +145,4 @@ maxpage := metricsPack[len(metricsPack)-1].Mtime JsonPrint(cfg, c, http.StatusAccepted, gin.H{"maxpage": maxpage, "data": metricsPack}) } -} \ No newline at end of file +}