Author: Paolo Lulli <paolo@lulli.net>
Initialize db on first run
client/main.go | 37 ++++++++++++--- client/metric-client-rest.go | 2 clients/shell/LogNow.sh | 1 schema/doc.txt | 87 ++++++++++++++++++++++++++++++++++++++ schema/schema.cql | 37 ---------------- server/db/connection.go | 2 server/db/metric.go | 9 ++- server/dbinit_test.go | 10 ++++ server/main.go | 16 ++++++ server/rest/rest-metric.go | 4
diff --git a/client/main.go b/client/main.go index 5c9a543c540224b8208fd6883a2d579f698009d3..39763d4ca372f687c540a9d98df40195a901806d 100644 --- a/client/main.go +++ b/client/main.go @@ -28,7 +28,7 @@ if len(os.Args) == 1 { fmt.Printf("For usage: %s --help\n", os.Args[0]) os.Exit(1) } - fmt.Println("Loading default config") + configPath = os.Getenv("HOME") + "/.yats-client.json" cfg := config.GetClientConfig(configPath) @@ -36,19 +36,40 @@ var yatsClient = YatsClient{config: cfg} sourceOption := flag.StringP("source", "s", "", "Source Application") - isMetricOption := flag.BoolP("isMetric", "m", false, "Select Metric") + isMetricOption := flag.BoolP("metric", "m", false, "Metric Mode") + isEventOption := flag.BoolP("event", "e", true, "Event Mode") + isPositionOption := flag.BoolP("metric", "p", false, "Position Mode") + + writeModeOption := flag.BoolP("write", "w", false, "Write Mode") + fromOption := flag.Int64("from", 0, "From tstamp") toOption := flag.Int64("to", 0, "To tstamp") metricNameOption := flag.StringP("metricName", "n", "", "Metric name") flag.Parse() - if *isMetricOption { - listResponse := yatsClient.MetricList(*sourceOption, *metricNameOption, *fromOption, *toOption) - fmt.Printf("Metric list: %s", listResponse) + if !*writeModeOption { + if *isMetricOption { + listResponse := yatsClient.MetricList(*sourceOption, *metricNameOption, *fromOption, *toOption) + fmt.Printf("%s", listResponse) + } else if *isPositionOption { + listResponse := yatsClient.PositionList(*sourceOption, *fromOption, *toOption) + fmt.Printf("%s", listResponse) + } else { + listEvents := yatsClient.EventList(*sourceOption, *fromOption, *toOption) + fmt.Printf("%s", listEvents) + } } else { - listEvents := yatsClient.EventList(*sourceOption, *fromOption, *toOption) - fmt.Printf("Event list: %s", listEvents) + // Write Mode + if *isMetricOption { + listResponse := yatsClient.MetricList(*sourceOption, *metricNameOption, *fromOption, *toOption) + fmt.Printf("%s", listResponse) + } else if *isPositionOption { + listResponse := yatsClient.PositionList(*sourceOption, *fromOption, *toOption) + fmt.Printf("%s", listResponse) + } else { + listEvents := yatsClient.EventList(*sourceOption, *fromOption, *toOption) + fmt.Printf("%s", listEvents) + } } - } diff --git a/client/metric-client-rest.go b/client/metric-client-rest.go index 42c4dff04449ca0af7e7d1451e3f2ba6f3f463be..71eaa028082ce56c8548a522b243a7ec1c2747ad 100644 --- a/client/metric-client-rest.go +++ b/client/metric-client-rest.go @@ -10,7 +10,6 @@ */ package main import ( - "fmt" "strconv" ) @@ -24,6 +23,5 @@ body = "{\"source\": \"" + source + "\", \"name\":\"" + metricName + "\", \"from\":" + strconv.FormatInt(from, 10) + " }" } else { body = "{\"source\": \"" + source + "\",\"name\":\"" + metricName + "\", \"from\":" + strconv.FormatInt(from, 10) + ",\"to\":" + strconv.FormatInt(to, 10) + " }" } - fmt.Printf("request: [%s]\n", body) return c.ApiPost(c.config.Endpoint+"/metric/search", body) } diff --git a/clients/shell/LogNow.sh b/clients/shell/LogNow.sh index 8fb209e3038a251ff5b89b9067d029165394573e..cd3cc26440998ed9ca208c370134b32e813acf56 100755 --- a/clients/shell/LogNow.sh +++ b/clients/shell/LogNow.sh @@ -3,5 +3,6 @@ # curl \ -X POST\ --header "Content-Type: application/json"\ + --header "X-SSL-Client-CN: something"\ -d '{"id_client":"test-cli","name":"test-log","value":"A log message"}'\ http://127.0.0.1:18081/metric diff --git a/schema/doc.txt b/schema/doc.txt new file mode 100644 index 0000000000000000000000000000000000000000..4f958d807e5a243e397a8972e0feedb69e1768e9 --- /dev/null +++ b/schema/doc.txt @@ -0,0 +1,87 @@ +CREATE KEYSPACE yats WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} AND durable_writes = true; +CREATE TABLE metric ( + id_client text, + mtime timestamp, + name text, + value text, + PRIMARY KEY ((id_client,name),mtime) +); +CREATE TABLE event ( + id_client text, + etime timestamp, + name text, + PRIMARY KEY (id_client,etime) +); +CREATE TABLE client ( + id text, + name text, + created timestamp, + PRIMARY KEY(id) +); +CREATE TABLE metric_info ( + id_client text, + name text, + description text, + PRIMARY KEY (id_client,name) +); +CREATE TABLE position ( + id_client text, + ptime timestamp, + lat double, + lon double, + name text, + PRIMARY KEY (id_client,ptime) +); +CREATE TABLE sources ( + id_client text, + name text, + type text, + app text, + description text, + PRIMARY KEY (id_client,app,type,name) +); +CREATE INDEX ON sources(id_client); + +CREATE TABLE location ( + hash text, + lname text, + lon double, + lat double, + code text, + cdist double, + PRIMARY KEY (code,cdist) +); +CREATE TABLE location_label ( + hash text, + ltype text, + name text, + PRIMARY KEY (ltype,hash) +); + +// select * from location where lat='' and lon='' +// select * from location where ltype='' and code='' order by cdist +CREATE TABLE location ( + ltype text, + code text, + cdist double, + hash text, + lat double, + lname text, + lon double, + PRIMARY KEY ((ltype, code), cdist) +) + +use case: +1) actual fixed locations ---> index with H3 + precision (or maidenhead + precision) +2) position points on a path with id_client and optionally a timestamp : no space partitioning +select * from location where hash='x' +select * from position where id_user='x' and timestamp>'x' + +CREATE TABLE position ( + id_client text, + ptime timestamp, + lat double, + lon double, + name text, + PRIMARY KEY (id_client,ptime) +); diff --git a/schema/schema.cql b/schema/schema.cql index 4f958d807e5a243e397a8972e0feedb69e1768e9..cc4a266f038bd49cf8f5acefb3e3f5c3f49e6130 100644 --- a/schema/schema.cql +++ b/schema/schema.cql @@ -1,4 +1,3 @@ -CREATE KEYSPACE yats WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} AND durable_writes = true; CREATE TABLE metric ( id_client text, mtime timestamp, @@ -41,42 +40,6 @@ description text, PRIMARY KEY (id_client,app,type,name) ); CREATE INDEX ON sources(id_client); - -CREATE TABLE location ( - hash text, - lname text, - lon double, - lat double, - code text, - cdist double, - PRIMARY KEY (code,cdist) -); -CREATE TABLE location_label ( - hash text, - ltype text, - name text, - PRIMARY KEY (ltype,hash) -); - -// select * from location where lat='' and lon='' -// select * from location where ltype='' and code='' order by cdist -CREATE TABLE location ( - ltype text, - code text, - cdist double, - hash text, - lat double, - lname text, - lon double, - PRIMARY KEY ((ltype, code), cdist) -) - -use case: -1) actual fixed locations ---> index with H3 + precision (or maidenhead + precision) -2) position points on a path with id_client and optionally a timestamp : no space partitioning -select * from location where hash='x' -select * from position where id_user='x' and timestamp>'x' - CREATE TABLE position ( id_client text, ptime timestamp, diff --git a/server/db/connection.go b/server/db/connection.go index c9455a74a635b5d8b17fabaf0050dd26934b7f5b..56522cb9305f78ff304fe73b54b72336157ca145 100644 --- a/server/db/connection.go +++ b/server/db/connection.go @@ -24,5 +24,7 @@ cluster.Keyspace = configuration.DbName Session, _ := cluster.CreateSession() + //Session.Query("CONSISTENCY LOCAL_QUORUM").Exec() + return Session } diff --git a/server/db/metric.go b/server/db/metric.go index bb357e5fcbb1d591eda31abbca0ee087910ba829..375aac940d493d0eeef8ad12341ba4a3ed6e6a57 100644 --- a/server/db/metric.go +++ b/server/db/metric.go @@ -19,10 +19,10 @@ "yats-server/model" ) type MetricModel struct { - ID_client string `json:"id_client"` - Mtime time.Time `json:"mtime"` - Name string `json:"name"` - Value string `json:"value"` + IdClient string `json:"id_client"` + Mtime time.Time `json:"mtime"` + Name string `json:"name"` + Value string `json:"value"` } func PrintMetrics(metrics []model.MetricModel) { @@ -71,6 +71,7 @@ } func SaveMetric(idClient string, metricName string, metricValue string) { q := fmt.Sprintf("insert into metric ( id_client, mtime, name, value) values ('%s',toTimestamp(now()),'%s','%s');", idClient, metricName, metricValue) + fmt.Println(q) Session.Query(q).Exec() } diff --git a/server/dbinit_test.go b/server/dbinit_test.go new file mode 100644 index 0000000000000000000000000000000000000000..9a0d92a782d05b613d7fa39d35f46fbfd65524f8 --- /dev/null +++ b/server/dbinit_test.go @@ -0,0 +1,10 @@ +package main + +import ( + "testing" +) + +func TestDbInitialization(t *testing.T) { + + CreateTables("../schema/schema.cql") +} diff --git a/server/main.go b/server/main.go index 7e6686ff6aa947018b88decc6e22ee66ee1f4b35..f3c1e011474da0e4bf09ec819ef051efefb1777f 100644 --- a/server/main.go +++ b/server/main.go @@ -13,6 +13,7 @@ import ( "fmt" "os" + "strings" "yats-server/config" "yats-server/db" "yats-server/docs" @@ -66,3 +67,18 @@ fmt.Printf("Starting REST Server on address: %s\n", configuration.RestAddress) RestService(configuration) } + +func CreateTables(schemafile string) { + filestring, err := os.ReadFile(schemafile) + + if err != nil { + fmt.Print(err) + } + fullFile := string(filestring) + fullFile = strings.Replace(fullFile, "\n", "", -1) + queries := strings.Split(fullFile, ";") + for _, query := range queries { + fmt.Printf("query: [%s]\n", query) + //Session.Query(q).Exec() + } +} diff --git a/server/rest/rest-metric.go b/server/rest/rest-metric.go index 4134388f156b45037b6bab4fcb10ac7363025b36..f3e2ddc79da488506ff25395b228361b44af1eb5 100644 --- a/server/rest/rest-metric.go +++ b/server/rest/rest-metric.go @@ -104,7 +104,7 @@ metricsPack = db.MetricsFrom(db.Session, clientCN, metric.Name, util.Int64ToTimeUTC(metric.From), 100) } else if metric.From != 0 { metricsPack = db.MetricsBetween(db.Session, clientCN, metric.Name, util.Int64ToTimeUTC(metric.From), util.Int64ToTimeUTC(metric.To), 100) } - c.IndentedJSON(http.StatusAccepted, gin.H{"ret": "OK", "data": metricsPack}) + c.IndentedJSON(http.StatusAccepted, gin.H{"data": metricsPack}) } } @@ -138,6 +138,6 @@ var metricsPack []model.MetricModel metricsPack = db.MetricsFrom(db.Session, clientCN, nameParam, util.Int64ToTimeUTC(fromParamInt64), 100) - c.IndentedJSON(http.StatusAccepted, gin.H{"ret": "OK", "data": metricsPack}) + c.IndentedJSON(http.StatusAccepted, gin.H{"data": metricsPack}) } }