primo test

aggregate.go

@@ -4,62 +4,60 @@ package main
 import (
 	"fmt"
 	"log"
+	//	"sort"
 	"time"
 
 	"gopkg.in/mgo.v2/bson"
 )
 
-func aggregate(ys time.Time, ye time.Time) {
+func consolidate(ys time.Time, ye time.Time) {
 
-	qStart := time.Now()
+	limit := 10000
 
-	p := dbs.ll.Pipe([]bson.M{{"$match": bson.M{"date": bson.M{"$gte": ys, "$lte": ye}}}, {"$group": bson.M{"_id": "$user"}}, {"$project": bson.M{"user": "$_id"}}})
+	pre := dbs.mdbDst.DB("dovecot").C("pre_lastlogin_day")
+
+	query := pre.Find(bson.M{})
 
-	idb.Pipe = time.Since(qStart)
 	if opts.Debug {
-		fmt.Printf("Aggregate user: %s\n\r", time.Since(qStart))
+		tot, _ := query.Count()
+		fmt.Printf("Users: %d\n", tot)
 	}
 
-	fmt.Printf("Date: %s - %s\n", ys, ye)
+	query.Batch(limit)
-	log.Printf("Date: %s - %s\n", ys, ye)
+	iter := query.Iter()
 
-	ar := Users{}
+	if opts.Bulk {
-	it := p.Iter()
+		dbs.bulk = dbs.lc.Bulk()
+		dbs.bulk.Unordered()
+	}
 
-	for it.Next(&ar) {
+	result := Users{}
+
+	for iter.Next(&result) {
 		idb.CountTOT += 1
 		ll := LastLoginDay{}
-		ll.User = ar.User
+		ll.User = result.User
 		ll.Date = ys
 
 		// DEBUG
-		if opts.Debug {
-			fmt.Printf("User: %s\n\r", ar.User)
-		}
 
-		qStart = time.Now()
+		logins := result.Logins
-		nq := dbs.ll.Find(bson.M{"date": bson.M{"$gte": ys, "$lte": ye}, "user": ar.User}).Sort("date")
-		//fmt.Printf("User: %s -> %s\n\r", ar[u], time.Since(qStart)	)
-		iter := nq.Iter()
 
-		if idb.Find == 0 {
-			idb.Find = time.Since(qStart)
-		} else {
-			idb.Find = (idb.Find + time.Since(qStart)) / 2
-		}
-
-		result := LastLogin{}
 		ips := []IPs{}
 		lastip := IPs{}
-		for iter.Next(&result) {
+		for l := range logins {
-			if result.IP == lastip.IP && result.Date.Sub(lastip.Date) < opts.Interval {
+			if logins[l].IP == lastip.IP && logins[l].Date.Sub(lastip.Date) < opts.Interval {
-				//fmt.Println("IPs: ", result.IP, result.Date, result.Date.Sub(lastip.Date))
+				/*
+					if opts.Debug {
+						fmt.Println("IPs: ", logins[l].IP, logins[l].Date, logins[l].Date.Sub(lastip.Date))
+					}
+				*/
 			} else {
-				ips = append(ips, IPs{IP: result.IP, Date: result.Date, Protocol: result.Protocol})
+				ips = append(ips, IPs{IP: logins[l].IP, Date: logins[l].Date, Protocol: logins[l].Protocol})
-				lastip.Date = result.Date
+				lastip.Date = logins[l].Date
-				lastip.IP = result.IP
+				lastip.IP = logins[l].IP
 			}
-			switch result.Protocol {
+			switch logins[l].Protocol {
 			case "pop3", "pop":
 				ll.Protocols.Pop += 1
 			case "imap":
@@ -68,23 +66,80 @@ func aggregate(ys time.Time, ye time.Time) {
 				ll.Protocols.Web += 1
 			}
 		}
-		if err := iter.Close(); err != nil {
-			log.Println("Iter: ", err)
-		}
 		ll.IPs = ips
-		//fmt.Printf("Upsert %+v\n\r", ll)
+
+		/*
+			if opts.Debug {
+				fmt.Printf("Insert %+v\n\r", ll)
+			}
+		*/
+
 		iStart := time.Now()
-		_, err := dbs.lc.Upsert(Index{User: ll.User, Date: ll.Date}, ll)
+
-		if err != nil {
+		if opts.Bulk {
-			log.Println("Insert error: ", err)
+			//dbs.bulk.Upsert(Index{User: ll.User, Date: ll.Date}, ll)
-		}
+			dbs.bulk.Insert(ll)
-		if idb.Insert == 0 {
-			idb.Insert = time.Since(iStart)
 		} else {
-			idb.Insert = (idb.Insert + time.Since(iStart)) / 2
+			info, err := dbs.lc.Upsert(Index{User: ll.User, Date: ll.Date}, ll)
+			if err != nil {
+				log.Println("Insert error: ", err)
+				continue
+			}
+			fmt.Printf("Change: %+v\n", info)
 		}
+		idb.Insert += time.Since(iStart)
 
 		idb.CountOK += 1
 	}
 
+	if opts.Bulk {
+		res, err := dbs.bulk.Run()
+		if err != nil {
+			log.Println("Insert error: ", err)
+		}
+		fmt.Printf("Bulk res: %+v\n", res)
+	}
+
+	if opts.Debug {
+		fmt.Printf("Insert: %d\n", idb.CountOK)
+	}
+
+	pre.DropCollection()
+}
+
+func aggregate(ys time.Time, ye time.Time) {
+
+	qStart := time.Now()
+
+	p := dbs.ll.Pipe([]bson.M{
+		{"$match": bson.M{"date": bson.M{"$gte": ys, "$lte": ye}}},
+		{"$group": bson.M{"_id": "$user"}},
+		{"$project": bson.M{"user": "$_id"}}})
+
+	/*
+		p := dbs.ll.Pipe([]bson.M{
+			{"$match": bson.M{"date": bson.M{"$gte": ys, "$lte": ye}}},
+			//		{"$sort": bson.M{"user": -1, "date": 1}},
+			{"$group": bson.M{"_id": "$user",
+				"logins": bson.M{"$push": bson.M{"protocol": "$protocol", "date": "$date", "ip": "$ip"}}}}})
+		//		{"$sort": bson.M{"_id": -1}},
+		//		{"$project": bson.M{"user": "$_id", "_id": 0, "logins": 1}},
+		//		{"$out": "pre_lastlogin_day"}})
+	*/
+
+	idb.Pipe = idb.Pipe + time.Since(qStart)
+	if opts.Debug {
+		res := new(interface{})
+		err := p.Explain(res)
+		fmt.Printf("Pipe: %+v\nErr: %+v\n", *res, err)
+		res = new(interface{})
+		err = p.One(res)
+		fmt.Printf("user: %+v\nErr: %+v\n", *res, err)
+		fmt.Printf("Aggregate user: %s\n\r", time.Since(qStart))
+	}
+
+	consolidate(ys, ye)
+
+	fmt.Printf("Date: %s - %s\n", ys, ye)
+	log.Printf("Date: %s - %s\n", ys, ye)
 }

dbs.go

@@ -2,10 +2,11 @@
 package main
 
 import (
-	"gopkg.in/mgo.v2"
 	"log"
 	"os"
 	"time"
+
+	"gopkg.in/mgo.v2"
 )
 
 type Dbs struct {
@@ -13,6 +14,7 @@ type Dbs struct {
 	mdbDst *mgo.Session
 	ll     *mgo.Collection
 	lc     *mgo.Collection
+	bulk   *mgo.Bulk
 }
 
 type LastLogin struct {
@@ -47,9 +49,16 @@ type Index struct {
 	Date time.Time `json:"date"`
 }
 
+type Users struct {
+	User   string `json:"user"`
+	Logins []IPs  `json:"ips"`
+}
+
+/*
 type Users struct {
 	User string `json:"user"`
 }
+*/
 
 func connectMongo() {
 
@@ -77,5 +86,4 @@ func connectMongo() {
 		}
 		dbs.lc = dbs.mdbDst.DB("dovecot").C("lastlogin_day")
 	}
-
 }

lastlogin_consolidate.go

@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	_VERSION = "v1.2.3"
+	_VERSION = "v1.3.0"
 	_tformat = "2006-01-02"
 	_24h     = (time.Hour * 23) + (time.Minute * 59) + (time.Second * 59)
 	_10m     = (time.Minute * 10)
@@ -70,7 +70,7 @@ func main() {
 
 	connectMongo()
 	defer dbs.mdbSrc.Close()
-	defer dbs.mdbDst.Clone()
+	defer dbs.mdbDst.Close()
 
 	y, err := time.Parse(_tformat, opts.StartDate)
 	if err != nil {
@@ -84,18 +84,23 @@ func main() {
 	var ys []time.Time
 	var ye []time.Time
 
-	if opts.Duration <= (time.Hour * 24) {
+	//	if opts.Duration <= (time.Hour * 24) {
-		ys = append(ys, time.Date(y.Year(), y.Month(), y.Day(), 0, 0, 0, 0, time.UTC))
+	ys = append(ys, time.Date(y.Year(), y.Month(), y.Day(), 0, 0, 0, 0, time.UTC))
-		ye = append(ye, ys[0].Add(opts.Duration))
+	ye = append(ye, ys[0].Add(opts.Duration))
-	} else {
+
-		for i := 0; i < int(opts.Duration/(time.Hour*24)); i++ {
+	/*
-			//			fmt.Println(i)
+		} else {
-			yt := y.Add(time.Hour * time.Duration(24*i))
+			for i := 0; i <= int(opts.Duration/(time.Hour*24)); i++ {
-			//	fmt.Println(yt)
+				yt := y.Add(time.Hour * time.Duration(24*i))
-			ys = append(ys, time.Date(yt.Year(), yt.Month(), yt.Day(), 0, 0, 0, 0, time.UTC))
+				if opts.Debug {
-			ye = append(ye, ys[i].Add(_24h))
+					fmt.Println(i)
+					fmt.Println(yt)
+				}
+				ys = append(ys, time.Date(yt.Year(), yt.Month(), yt.Day(), 0, 0, 0, 0, time.UTC))
+				ye = append(ye, ys[i].Add(_24h))
+			}
 		}
-	}
+	*/
 
 	// DEBUG
 	if opts.Debug {

options.go

@@ -23,10 +23,11 @@ type Options struct {
 	Hostname  string
 	Version   bool
 	Debug     bool
+	Bulk      bool
 }
 
 func usage() {
-	fmt.Println("Usage: lastlogin_consolidate -ms <mongo source mongodb://IP:PORT> -md <mongo destination mongodb://IP:PORT> -l <logfile> -d <date> -dd <duration> -i <interval> -I <influxdb uri> -v\n")
+	fmt.Println("Usage: lastlogin_consolidate -ms <mongo source mongodb://IP:PORT> -md <mongo destination mongodb://IP:PORT> -l <logfile> -d <date> -i <interval> -I <influxdb uri> -v\n")
 	os.Exit(0)
 }
 
@@ -48,4 +49,5 @@ func init() {
 	flag.DurationVar(&opts.Interval, "i", opts.Interval, "Duration")
 	flag.BoolVar(&opts.Version, "v", false, "Version")
 	flag.BoolVar(&opts.Debug, "debug", false, "Debug")
+	flag.BoolVar(&opts.Bulk, "bulk", false, "Bulk")
 }