implementata la scrittura Bulk e lo split dell'aggregato per username
This commit is contained in:
parent
0eca19256c
commit
e3d7b1dae5
2 changed files with 82 additions and 91 deletions
53
aggregate.go
53
aggregate.go
|
|
@ -11,21 +11,20 @@ import (
|
|||
)
|
||||
|
||||
func bulkWrite() {
|
||||
res, err := dbs.bulk.Run()
|
||||
_, err := dbs.bulk.Run()
|
||||
if err != nil {
|
||||
log.Println("Insert error: ", err)
|
||||
}
|
||||
fmt.Printf("Bulk res: %+v\n", res)
|
||||
//fmt.Printf("Bulk res: %+v\n", res)
|
||||
}
|
||||
|
||||
func consolidate(users []Users, ys time.Time, ye time.Time) {
|
||||
func consolidate(user Users, ys time.Time, ye time.Time) {
|
||||
|
||||
if opts.Bulk {
|
||||
dbs.bulk = dbs.lc.Bulk()
|
||||
dbs.bulk.Unordered()
|
||||
}
|
||||
|
||||
for _, user := range users {
|
||||
idb.CountTOT += 1
|
||||
ll := LastLoginDay{}
|
||||
ll.User = user.User
|
||||
|
|
@ -66,64 +65,56 @@ func consolidate(users []Users, ys time.Time, ye time.Time) {
|
|||
//dbs.bulk.Upsert(Index{User: ll.User, Date: ll.Date}, ll)
|
||||
dbs.bulk.Insert(ll)
|
||||
} else {
|
||||
info, err := dbs.lc.Upsert(Index{User: ll.User, Date: ll.Date}, ll)
|
||||
_, err := dbs.lc.Upsert(Index{User: ll.User, Date: ll.Date}, ll)
|
||||
if err != nil {
|
||||
log.Println("Insert error: ", err)
|
||||
continue
|
||||
}
|
||||
fmt.Printf("Change: %+v\n", info)
|
||||
// fmt.Printf("Change: %+v\n", info)
|
||||
}
|
||||
idb.Insert += time.Since(iStart)
|
||||
|
||||
idb.CountOK += 1
|
||||
}
|
||||
|
||||
if opts.Bulk {
|
||||
bulkWrite()
|
||||
}
|
||||
|
||||
if opts.Debug {
|
||||
fmt.Printf("Insert: %d in %v\n", idb.CountOK, idb.Insert)
|
||||
}
|
||||
|
||||
// pre.DropCollection()
|
||||
// if opts.Debug {
|
||||
// fmt.Printf("Insert: %d in %v\n", idb.CountOK, idb.Insert)
|
||||
// }
|
||||
}
|
||||
|
||||
func aggregate(ys time.Time, ye time.Time) {
|
||||
|
||||
skip := 0
|
||||
limit := 100000
|
||||
groups := []string{"[^a-z]", "[be]", "[rv]", "[dt]", "[li]", "[pzjkwxy]", "[fn]", "[co]", "[gu]", "[sh]", "[aq]", "[m]"}
|
||||
|
||||
for {
|
||||
for g := range groups {
|
||||
|
||||
qStart := time.Now()
|
||||
|
||||
p := dbs.ll.Pipe([]bson.M{
|
||||
{"$match": bson.M{"date": bson.M{"$gte": ys, "$lte": ye}}},
|
||||
{"$match": bson.M{"date": bson.M{"$gte": ys, "$lte": ye},
|
||||
"user": bson.RegEx{"^" + groups[g], ""}}},
|
||||
{"$sort": bson.M{"user": -1, "date": 1}},
|
||||
{"$group": bson.M{"_id": "$user",
|
||||
"logins": bson.M{"$push": bson.M{"protocol": "$protocol", "date": "$date", "ip": "$ip"}}}},
|
||||
{"$sort": bson.M{"_id": -1}},
|
||||
{"$skip": skip},
|
||||
{"$limit": limit},
|
||||
{"$project": bson.M{"user": "$_id", "_id": 0, "logins": 1}}}).AllowDiskUse()
|
||||
"logins": bson.M{"$push": bson.M{"protocol": "$protocol", "date": "$date", "ip": "$ip"}}}}}).AllowDiskUse()
|
||||
|
||||
result := make([]Users, limit)
|
||||
p.All(&result)
|
||||
if len(result) <= 0 {
|
||||
break
|
||||
iter := p.Iter()
|
||||
defer iter.Close()
|
||||
|
||||
var result Users
|
||||
for iter.Next(&result) {
|
||||
consolidate(result, ys, ye)
|
||||
}
|
||||
|
||||
if opts.Debug {
|
||||
last := len(result) - 1
|
||||
fmt.Printf("Res %d: %d - first: %+v - last: %+v - time: %+v\n", skip, len(result), result[0].User, result[last].User, time.Since(qStart))
|
||||
fmt.Printf("Group %v: %+v\n", groups[g], time.Since(qStart))
|
||||
}
|
||||
|
||||
// p.All(&result)
|
||||
|
||||
idb.Pipe = idb.Pipe + time.Since(qStart)
|
||||
|
||||
consolidate(result, ys, ye)
|
||||
|
||||
skip += limit
|
||||
}
|
||||
|
||||
fmt.Printf("Date: %s - %s\n", ys, ye)
|
||||
|
|
|
|||
38
dbs.go
38
dbs.go
|
|
@ -18,40 +18,40 @@ type Dbs struct {
|
|||
}
|
||||
|
||||
type LastLogin struct {
|
||||
User string `json: "user"`
|
||||
Protocol string `json: "protocol"`
|
||||
IP string `json: "ip"`
|
||||
Date time.Time `json: "date"`
|
||||
ID string `json: "_id"`
|
||||
User string `json: "user" bson:"user"`
|
||||
Protocol string `json: "protocol" bson:"protocol"`
|
||||
IP string `json: "ip" bson:"ip"`
|
||||
Date time.Time `json: "date" bson:"date"`
|
||||
ID string `json: "_id" bson:"_id"`
|
||||
}
|
||||
|
||||
type LastLoginDay struct {
|
||||
User string `json:"user"`
|
||||
Date time.Time `json:"date"`
|
||||
Protocols Protocols `json:"protocols"`
|
||||
IPs []IPs `json:"ips"`
|
||||
User string `json:"user" bson:"user"`
|
||||
Date time.Time `json:"date" bson:"date"`
|
||||
Protocols Protocols `json:"protocols" bson:"protocols"`
|
||||
IPs []IPs `json:"ips" bson:"ips"`
|
||||
}
|
||||
|
||||
type IPs struct {
|
||||
IP string `json:"ip"`
|
||||
Date time.Time `json:"date"`
|
||||
Protocol string `json:"protocol"`
|
||||
IP string `json:"ip" bson:"ip"`
|
||||
Date time.Time `json:"date" bson:"date"`
|
||||
Protocol string `json:"protocol" bson:"protocol"`
|
||||
}
|
||||
|
||||
type Protocols struct {
|
||||
Pop int `json:"pop"`
|
||||
Imap int `json:"imap"`
|
||||
Web int `json:"web"`
|
||||
Pop int `json:"pop" bson:"pop"`
|
||||
Imap int `json:"imap" bson:"imap"`
|
||||
Web int `json:"web" bson:"web"`
|
||||
}
|
||||
|
||||
type Index struct {
|
||||
User string `json:"user"`
|
||||
Date time.Time `json:"date"`
|
||||
User string `json:"user" bson:"user"`
|
||||
Date time.Time `json:"date" bson:"date"`
|
||||
}
|
||||
|
||||
type Users struct {
|
||||
User string `json:"user"`
|
||||
Logins []IPs `json:"ips"`
|
||||
User string `json:"_id" bson:"_id"`
|
||||
Logins []IPs `json:"logins" bson:"logins"`
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue