120 lines
2.5 KiB
Go
120 lines
2.5 KiB
Go
// consumer
|
|
package main
|
|
|
|
import (
|
|
"fmt"
|
|
// "github.com/garyburd/redigo/redis"
|
|
"log"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gopkg.in/mgo.v2"
|
|
)
|
|
|
|
type consumed struct {
|
|
user string
|
|
error bool
|
|
logins []string
|
|
empty bool
|
|
}
|
|
|
|
func contains(s []Ips, e string) bool {
|
|
for _, a := range s {
|
|
if a.IP == e {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func consumer() {
|
|
|
|
for {
|
|
|
|
prod := <-consume
|
|
|
|
var bulk = make(map[string]*mgo.Bulk)
|
|
|
|
bulk[opts.Month] = dbs.ll.Bulk()
|
|
bulk[opts.Month].Unordered()
|
|
|
|
cons := consumed{
|
|
user: prod.user,
|
|
logins: make([]string, 0),
|
|
error: false,
|
|
empty: true,
|
|
}
|
|
|
|
start := time.Now()
|
|
for i := range prod.logins {
|
|
login := prod.logins[i]
|
|
// se la riga di login e' vuota
|
|
if login == "" {
|
|
log.Println("Login empty: ", prod.user)
|
|
cons.logins = append(cons.logins, login)
|
|
continue
|
|
}
|
|
sval := strings.Split(login, ":")
|
|
// se il formato della riga di login non e' corretto
|
|
if sval[1] == "" {
|
|
log.Println("Login format error: ", login, prod.user)
|
|
cons.logins = append(cons.logins, login)
|
|
continue
|
|
}
|
|
// se il timestamp della riga di login non e' corretto
|
|
date, err := strconv.ParseInt(sval[1], 10, 64)
|
|
if err != nil {
|
|
log.Printf("Date Error: %+v - %s - %s\n", err, prod.user, login)
|
|
cons.logins = append(cons.logins, login)
|
|
continue
|
|
}
|
|
ml := MongoLogin{
|
|
// genera l' _ID con user e timestamp
|
|
ID: fmt.Sprintf("%s_%s", prod.user, time.Unix(date, 0).Format("20060102T150405")),
|
|
User: prod.user,
|
|
Protocol: sval[0],
|
|
IP: sval[2],
|
|
Date: time.Unix(date, 0),
|
|
Insert: time.Now(),
|
|
}
|
|
|
|
if opts.Month != ml.Date.Format("0601") {
|
|
dt := fmt.Sprintf("lastlogin_%s", ml.Date.Format("0601"))
|
|
if _, ok := bulk[dt]; !ok {
|
|
bulk[dt] = dbs.mdb.DB("lastlogin").C(dt).Bulk()
|
|
bulk[dt].Unordered()
|
|
}
|
|
bulk[dt].Insert(ml)
|
|
} else {
|
|
// inserisce il login su Mongodb
|
|
|
|
bulk[opts.Month].Insert(ml)
|
|
// inserisce last timestamp su redis per consolidamento
|
|
}
|
|
|
|
cons.logins = append(cons.logins, login)
|
|
}
|
|
|
|
for key, _ := range bulk {
|
|
_, err := bulk[key].Run()
|
|
if err != nil {
|
|
|
|
fmt.Printf("Err: %+v\n", err)
|
|
}
|
|
}
|
|
|
|
count.AddLog(len(prod.logins))
|
|
|
|
if opts.MaxLogins > -1 && len(prod.logins) < opts.MaxLogins {
|
|
cons.empty = true
|
|
}
|
|
|
|
if opts.Debug {
|
|
fmt.Printf("CONS: user=%s logins=%d in %v - active=%d\n", prod.user, len(prod.logins), time.Since(start), dbs.rdb.ActiveCount())
|
|
}
|
|
|
|
wg.Done()
|
|
// remove <- cons
|
|
}
|
|
}
|