From d8137b631fd02e60a3ac598e6817b6cb30854b82 Mon Sep 17 00:00:00 2001 From: Michele Date: Thu, 25 Oct 2018 17:26:59 +0200 Subject: [PATCH 01/12] debug insert b2 --- consumer.go | 7 ++++--- main.go | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/consumer.go b/consumer.go index 5a68f9e..169ec7a 100644 --- a/consumer.go +++ b/consumer.go @@ -139,7 +139,7 @@ func consumer() { tipo: "err", val: len(prod.logins), } - if opts.Test { + if opts.Test || opts.Debug { log.Printf("ERR: %s - %+v\n", prod.user, prod.logins) } continue @@ -148,12 +148,13 @@ func consumer() { tipo: "dup", val: strings.Count(err.Error(), "E11000"), } + if opts.Debug { + log.Printf("DUP: %s - %+v\n", prod.user, prod.logins) + } } } else { if opts.Test { log.Printf("OK: %s - %+v\n", prod.user, prod.logins) - } - if opts.Debug { log.Printf("BulkResult: %s - %+v\n", prod.user, result) } counter <- Counterchan{ diff --git a/main.go b/main.go index 66b7216..f5db7f9 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( ) const ( - _Version = "v4.4.5b1" + _Version = "v4.4.5b2" _Producer = 0 _Consumer = 1 _Remover = 2 From 8e1b5033f1fb4026cf5a8c1c1923d5be78722d72 Mon Sep 17 00:00:00 2001 From: Michele Date: Wed, 7 Nov 2018 16:44:03 +0100 Subject: [PATCH 02/12] counter INS = len(bulk) --- consumer.go | 2 +- main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/consumer.go b/consumer.go index 169ec7a..ef827f2 100644 --- a/consumer.go +++ b/consumer.go @@ -159,7 +159,7 @@ func consumer() { } counter <- Counterchan{ tipo: "ins", - val: len(prod.logins), + val: len(bulk), } } } diff --git a/main.go b/main.go index f5db7f9..d5d2efd 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( ) const ( - _Version = "v4.4.5b2" + _Version = "v4.4.5b3" _Producer = 0 _Consumer = 1 _Remover = 2 From 3e3e36f81ea6e86238291b074536d3dba140dd71 Mon Sep 17 00:00:00 2001 From: Michele Date: Wed, 7 Nov 2018 16:47:38 +0100 Subject: [PATCH 03/12] counter INS = len(AllLogins) --- consumer.go | 2 +- main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/consumer.go b/consumer.go index ef827f2..34505e3 100644 --- a/consumer.go +++ b/consumer.go @@ -159,7 +159,7 @@ func consumer() { } counter <- Counterchan{ tipo: "ins", - val: len(bulk), + val: len(allLogins), } } } diff --git a/main.go b/main.go index d5d2efd..d478d2b 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( ) const ( - _Version = "v4.4.5b3" + _Version = "v4.4.5b4" _Producer = 0 _Consumer = 1 _Remover = 2 From 58234ad80a63300bf223b92333c4b7656a2d766f Mon Sep 17 00:00:00 2001 From: Michele Date: Wed, 7 Nov 2018 16:50:53 +0100 Subject: [PATCH 04/12] v4.4.5 --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index d478d2b..2eec5e4 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( ) const ( - _Version = "v4.4.5b4" + _Version = "v4.4.5" _Producer = 0 _Consumer = 1 _Remover = 2 From 5bc0076ecf9067dc3faf89638baed4517cc2d754 Mon Sep 17 00:00:00 2001 From: Michele Fadda Date: Thu, 22 Nov 2018 12:29:08 +0100 Subject: [PATCH 05/12] auto discovery (by consul) --- consul.go | 30 ++++++++++++++++++++++++++++++ main.go | 2 +- options.go | 7 +++++-- 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 consul.go diff --git a/consul.go b/consul.go new file mode 100644 index 0000000..6c2c3d0 --- /dev/null +++ b/consul.go @@ -0,0 +1,30 @@ +package main + +import ( + "fmt" + "log" + "strconv" + "strings" + "time" + + "github.com/hashicorp/consul/api" +) + +type Consul struct { + RedisTTL time.Duration + Timeout time.Duration + MaxError int + Influxdb string + Month string + Retention float64 + MongoURI string + Database string + RedisURI string +} + +func getClient() { + client, err := api.NewClient(api.DefaultConfig()) + if err != nil { + panic(err) + } +} diff --git a/main.go b/main.go index 2eec5e4..3221956 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( ) const ( - _Version = "v4.4.5" + _Version = "v5.0.0" _Producer = 0 _Consumer = 1 _Remover = 2 diff --git a/options.go b/options.go index b0c6319..32c33d3 100644 --- a/options.go +++ b/options.go @@ -27,6 +27,7 @@ type Options struct { Month string Queue int Retention float64 + Consul string } var ( @@ -40,11 +41,12 @@ func usage() { fmt.Println(`Usage: llmongo -m -d - -r + -r -t -l -T - -i + -i + -C -q -R -v @@ -69,6 +71,7 @@ func init() { flag.StringVar(&dbs.Database, "d", "", "Mongodb Database") flag.StringVar(&dbs.RedisURI, "r", "", "Redis") flag.StringVar(&opts.LogFile, "l", opts.LogFile, "Logs filename") + flag.StringVar(&opts.Consul, "C", opts.Consul, "consul client") flag.DurationVar(&opts.RedisTTL, "t", opts.RedisTTL, "Redis keys TTL") flag.BoolVar(&opts.Version, "v", false, "Version") flag.DurationVar(&opts.Timeout, "T", 0, "Running timeout") From a2fc00df74e0c5eae4267acbd71fe682a546e32e Mon Sep 17 00:00:00 2001 From: Michele Fadda Date: Tue, 27 Nov 2018 15:10:19 +0100 Subject: [PATCH 06/12] update --- consul.go | 22 ++++++++++++++++++---- counter.go | 2 ++ options.go | 3 +++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/consul.go b/consul.go index 6c2c3d0..a959a24 100644 --- a/consul.go +++ b/consul.go @@ -3,13 +3,12 @@ package main import ( "fmt" "log" - "strconv" - "strings" "time" "github.com/hashicorp/consul/api" ) +// Consul client structure type Consul struct { RedisTTL time.Duration Timeout time.Duration @@ -20,11 +19,26 @@ type Consul struct { MongoURI string Database string RedisURI string + Client *api.Client } -func getClient() { +// NewClient New consul client +func NewClient() *Consul { client, err := api.NewClient(api.DefaultConfig()) if err != nil { - panic(err) + log.Fatalf("Consul error: %+v\n", err) + if opts.Debug { + fmt.Printf("Consul error: %+v\n", err) + } } + + consul := &Consul{ + Client: client, + RedisTTL: opts.RedisTTL, + Timeout: opts.Timeout, + MaxError: opts.MaxError, + Influxdb: opts.Influxdb, + } + + return consul } diff --git a/counter.go b/counter.go index f9505d2..1b616a0 100644 --- a/counter.go +++ b/counter.go @@ -6,6 +6,7 @@ import ( "time" ) +// Counterchan counter structure type Counterchan struct { tipo string val int @@ -38,6 +39,7 @@ func NewCounter() *Counter { } } +// Run open channel to receive counter func (c *Counter) Run() { for { tocount := <-counter diff --git a/options.go b/options.go index 32c33d3..96ab622 100644 --- a/options.go +++ b/options.go @@ -28,6 +28,7 @@ type Options struct { Queue int Retention float64 Consul string + Port int } var ( @@ -47,6 +48,7 @@ func usage() { -T -i -C + -P [port] ## used by consul to check services ## -q -R -v @@ -72,6 +74,7 @@ func init() { flag.StringVar(&dbs.RedisURI, "r", "", "Redis") flag.StringVar(&opts.LogFile, "l", opts.LogFile, "Logs filename") flag.StringVar(&opts.Consul, "C", opts.Consul, "consul client") + flag.IntVar(&opts.Port, "P", 10500, "service check port") flag.DurationVar(&opts.RedisTTL, "t", opts.RedisTTL, "Redis keys TTL") flag.BoolVar(&opts.Version, "v", false, "Version") flag.DurationVar(&opts.Timeout, "T", 0, "Running timeout") From 2d3d9e43d3e40ab2b85aad21fd6f376f07fe5719 Mon Sep 17 00:00:00 2001 From: Michele Fadda Date: Tue, 27 Nov 2018 15:11:59 +0100 Subject: [PATCH 07/12] added --- mongo_scripts.txt | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 mongo_scripts.txt diff --git a/mongo_scripts.txt b/mongo_scripts.txt new file mode 100644 index 0000000..6dc7834 --- /dev/null +++ b/mongo_scripts.txt @@ -0,0 +1,34 @@ + +################ +var dd = function(x) { var s = x.toString(); if(s.length > 1){return s;} else { return "0"+s;}} + +for (var i=1; i<18; i++) { + var col = "ll_1806"+dd(i); + print(col); + db.getCollection(col).createIndex({date: -1}, {expireAfterSeconds: 15552000, name: "expire", background: true}); +} + +########### +var dd = function(x) { var s = x.toString(); if(s.length > 1){return s;} else { return "0"+s;}} + +for (var m=1; m<=12; m++) { + for (var d=1; d<=31; d++) { + var col = "ll_15"+dd(m)+dd(d); + if (db.getCollection(col).exists != null) { + print(col); + print(db.getCollection(col).totalSize()); + } + } +} + + +#### +var ar = db.getCollectionNames() +var len = ar.length + +for (var i=0; i Date: Mon, 10 Dec 2018 16:25:40 +0100 Subject: [PATCH 08/12] testing new mgo driver --- consumer.go | 3 ++- dbs.go | 3 ++- main.go | 2 +- options.go | 1 - 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/consumer.go b/consumer.go index 34505e3..0c4b7f8 100644 --- a/consumer.go +++ b/consumer.go @@ -12,7 +12,8 @@ import ( "strings" "time" - "gopkg.in/mgo.v2" + "github.com/globalsign/mgo" + // "gopkg.in/mgo.v2" ) type consumed struct { diff --git a/dbs.go b/dbs.go index f0bec39..4e28cf5 100644 --- a/dbs.go +++ b/dbs.go @@ -10,7 +10,8 @@ import ( "github.com/garyburd/redigo/redis" - "gopkg.in/mgo.v2" + "github.com/globalsign/mgo" + // "gopkg.in/mgo.v2" ) var ( diff --git a/main.go b/main.go index 3221956..0fa9a6c 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( ) const ( - _Version = "v5.0.0" + _Version = "v5.0.0b1" _Producer = 0 _Consumer = 1 _Remover = 2 diff --git a/options.go b/options.go index 96ab622..5753674 100644 --- a/options.go +++ b/options.go @@ -1,4 +1,3 @@ -// options package main import ( From f1f38601d5e6c2f89334d859edf2e77fe37ee455 Mon Sep 17 00:00:00 2001 From: Michele Fadda Date: Wed, 12 Dec 2018 13:21:39 +0100 Subject: [PATCH 09/12] influxdb hostname|ip --- docker-compose/docker-compose.yml | 4 ++-- docker-compose/mongo.sh | 2 +- main.go | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 67eeacb..cf96102 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -1,7 +1,7 @@ version: '2' services: mongodb: - image: mikif70/mongodb:3.4.7 + image: mikif70/mongodb:3.6.9 ports: - 27017:27017 container_name: ll_mongod @@ -15,4 +15,4 @@ services: ports: - 6379:6379 volumes: - - ./redis:/data \ No newline at end of file + - ./redis:/data diff --git a/docker-compose/mongo.sh b/docker-compose/mongo.sh index 4c17756..e7fadef 100755 --- a/docker-compose/mongo.sh +++ b/docker-compose/mongo.sh @@ -6,4 +6,4 @@ else HOST=${1} fi -docker run --rm -it mikif70/mongotools:3.4.7 mongo ${HOST} +docker run --rm -it mikif70/mongotools:3.6.9 mongo ${HOST} diff --git a/main.go b/main.go index 0fa9a6c..fa66fb1 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( ) const ( - _Version = "v5.0.0b1" + _Version = "v5.0.0b2" _Producer = 0 _Consumer = 1 _Remover = 2 @@ -51,7 +51,8 @@ func main() { } if opts.Influxdb != "" { - re, _ := regexp.Compile(`(\w+)@(\d+.\d+.\d+.\d+:\d+)`) + var re = regexp.MustCompile(`(?m)(\w+)@([\w.]+):(\d+)`) + // re, _ := regexp.Compile(`(\w+)@(\d+.\d+.\d+.\d+:\d+)`) if re.MatchString(opts.Influxdb) { match := re.FindStringSubmatch(opts.Influxdb) infhost = match[1] From 5c45a79d12cfc0f5eed440f49033d669bace1658 Mon Sep 17 00:00:00 2001 From: Michele Fadda Date: Wed, 12 Dec 2018 13:33:26 +0100 Subject: [PATCH 10/12] split host@addr:port --- docker-compose/test.sh | 1 + influxdb.go | 3 ++- main.go | 7 +++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docker-compose/test.sh b/docker-compose/test.sh index 87d6691..233ebbf 100755 --- a/docker-compose/test.sh +++ b/docker-compose/test.sh @@ -11,5 +11,6 @@ docker run \ -r 192.168.0.1:6379 \ -m 192.168.0.1:27017 \ -d lastlogin \ + -i test@10.39.253.206:8086 \ -T 80s \ $@ diff --git a/influxdb.go b/influxdb.go index e0cdae4..3b96f37 100644 --- a/influxdb.go +++ b/influxdb.go @@ -15,7 +15,8 @@ var ( func writeStats(start time.Time) { if opts.Debug { - fmt.Printf("writing to influxdb server: %s", opts.Influxdb) + fmt.Printf("writing to influxdb server: %s\n", opts.Influxdb) + fmt.Printf("host: %s -- addr: %s\n", infhost, infdb) } c, err := influxdb.NewHTTPClient(influxdb.HTTPConfig{ diff --git a/main.go b/main.go index fa66fb1..1ff7150 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( ) const ( - _Version = "v5.0.0b2" + _Version = "v5.0.0b3" _Producer = 0 _Consumer = 1 _Remover = 2 @@ -55,8 +55,11 @@ func main() { // re, _ := regexp.Compile(`(\w+)@(\d+.\d+.\d+.\d+:\d+)`) if re.MatchString(opts.Influxdb) { match := re.FindStringSubmatch(opts.Influxdb) + if opts.Debug { + fmt.Printf("Influxdb match: %+v\n", match) + } infhost = match[1] - infdb = match[2] + infdb = match[2] + ":" + match[3] } else { opts.Influxdb = "" } From ee28cf2d89ca7834488cdbc9c249b899fd64e518 Mon Sep 17 00:00:00 2001 From: Michele Fadda Date: Wed, 12 Dec 2018 14:35:13 +0100 Subject: [PATCH 11/12] influxdb verbose errors --- influxdb.go | 10 +++++++--- main.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/influxdb.go b/influxdb.go index 3b96f37..242fe7c 100644 --- a/influxdb.go +++ b/influxdb.go @@ -3,6 +3,7 @@ package main import ( "fmt" + "log" "time" influxdb "github.com/influxdata/influxdb/client/v2" @@ -24,7 +25,8 @@ func writeStats(start time.Time) { Timeout: 2 * time.Second, }) if err != nil { - fmt.Printf("Error: %+v\n", err) + fmt.Printf("InfluxDB connect Error: %+v\n", err) + log.Printf("InfluxDB connect Error: %+v\n", err) return } defer c.Close() @@ -34,7 +36,8 @@ func writeStats(start time.Time) { Precision: "s", }) if err != nil { - fmt.Printf("Error: %+v\n", err) + fmt.Printf("InfluxDB batch Error: %+v\n", err) + log.Printf("InfluxDB batch Error: %+v\n", err) return } @@ -49,7 +52,8 @@ func writeStats(start time.Time) { } pt, err := influxdb.NewPoint("ll2mongo", tags, fields, start) if err != nil { - fmt.Printf("Error: %+v\n", err) + fmt.Printf("InfluxDB point Error: %+v\n", err) + log.Printf("InfluxDB point Error: %+v\n", err) return } diff --git a/main.go b/main.go index 1ff7150..150966d 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( ) const ( - _Version = "v5.0.0b3" + _Version = "v5.0.0b4" _Producer = 0 _Consumer = 1 _Remover = 2 From abca7e21e71c040af6cd2ee66b2b84ae4f221572 Mon Sep 17 00:00:00 2001 From: Michele Fadda Date: Wed, 12 Dec 2018 14:57:08 +0100 Subject: [PATCH 12/12] influxdb write verbose --- influxdb.go | 11 ++++++++++- main.go | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/influxdb.go b/influxdb.go index 242fe7c..be821d2 100644 --- a/influxdb.go +++ b/influxdb.go @@ -57,8 +57,17 @@ func writeStats(start time.Time) { return } + if opts.Debug { + fmt.Printf("InfluxDB pt: %+v\n", pt) + } + bp.AddPoint(pt) // Write the batch - c.Write(bp) + err = c.Write(bp) + if err != nil { + fmt.Printf("InfluxDB write Error: %+v\n", err) + log.Printf("InfluxDB write Error: %+v\n", err) + return + } } diff --git a/main.go b/main.go index 150966d..304e716 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( ) const ( - _Version = "v5.0.0b4" + _Version = "v5.0.0b6" _Producer = 0 _Consumer = 1 _Remover = 2