MongoDB/llmongodb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

poco efficente.... implemento iterator

Browse source
This commit is contained in:
Miki 2015-11-19 17:12:53 +01:00
parent 12a27241f7
commit ba85c9ecc9
4 changed files with 64 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
consumer.go
View file

@ -10,14 +10,14 @@ import (
"time"
)
func consumer() {
func consumer(user string) {
var date int64
var lastval string
var conn = dbs.rdb.Get()
defer conn.Close()
start := time.Now()
for {
user := <-msgs
// Estrae l'ultimo login dell'utente 'user'
val, err := redis.String(conn.Do("LINDEX", user, "-1"))
if err != nil {
@ -25,24 +25,34 @@ func consumer() {
log.Printf("LINDEX error: %+v - %s\n\r", err, val)
fmt.Printf("LINDEX error: %+v - %s\n\r", err, val)
}
// se ha trovato user e righe di login
if lastval != "" {
// reinserisce l'ultimo login e imposta il ttl su Redis
retval, _ := conn.Do("lpush", user, lastval)
ttl, _ := conn.Do("expire", user, opts.RedisTTL.Seconds())
if strings.Contains(err.Error(), "nil returned") {
// se ha trovato user e righe di login
if lastval != "" {
// reinserisce l'ultimo login e imposta il ttl su Redis
retval, errr := conn.Do("lpush", user, lastval)
ttl, errt := conn.Do("expire", user, int(opts.RedisTTL.Seconds()))
if opts.Debug {
log.Printf("LPUSH retval: %+v %+v %+v %+v %s %s %d\n\r", retval, errr, ttl, errt, user, lastval, int(opts.RedisTTL.Seconds()))
fmt.Printf("LPUSH retval: %+v %+v %+v %+v %s %s %d\n\r", retval, errr, ttl, errt, user, lastval, int(opts.RedisTTL.Seconds()))
}
}
} else {
if opts.Debug {
log.Println("LPUSH retval: ", retval, ttl, user, lastval, opts.RedisTTL.Seconds())
fmt.Println("LPUSH retval: ", retval, ttl, user, lastval, opts.RedisTTL.Seconds())
fmt.Printf("SADD error: %+v - %s\n\r", err, val)
}
val, err := conn.Do("sadd", "llindex", user)
if err != nil {
log.Printf("SADD error: %+v - %s\n\r", err, val)
}
}
// break
continue
break
}
// se la riga di login e' vuota
if val == "" {
log.Println("Login empty: ", user)
retval, _ := conn.Do("lrem", user, "-1", val)
log.Println("LREM retval: ", user, val, retval)
//return
continue
}
sval := strings.Split(val, ":")
@ -51,12 +61,14 @@ func consumer() {
log.Println("Login format error: ", val, user)
retval, _ := conn.Do("lrem", user, "-1", val)
log.Println("LREM retval: ", user, val, retval)
//return
continue
}
// se il timestamp della riga di login non e' corretto
date, err = strconv.ParseInt(sval[1], 10, 64)
if err != nil {
log.Printf("Date Error: %+v - %s\n", err, user)
//return
continue
}
ml := MongoLogin{
@ -77,6 +89,7 @@ func consumer() {
// se l'errore non e' "duplicate key error" salta al prossimo senza cancellare niente
if !strings.Contains(err.Error(), "E11000") {
errCount += 1
//return
continue
}
}
@ -88,4 +101,6 @@ func consumer() {
}
lastval = val
}
fmt.Printf("CONS: user=%s in %v - active=%d\n\r", user, time.Since(start), dbs.rdb.ActiveCount())
log.Printf("CONS: user=%s in %v - active=%d\n\r", user, time.Since(start), dbs.rdb.ActiveCount())
}

6
dbs.go
View file

@ -47,8 +47,10 @@ type Index struct {
func (db *Dbs) poolRedis() {
dbs.rdb = &redis.Pool{
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
MaxIdle: 128,
MaxActive: 1000,
Wait: true,
IdleTimeout: 1 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", db.RedisUri)
if err != nil {

37
main.go
View file

@ -6,8 +6,10 @@ import (
"fmt"
"log"
"os"
"os/signal"
"path"
"path/filepath"
"syscall"
"time"
)
@ -19,10 +21,11 @@ type Options struct {
Timeout bool
Debug bool
Version bool
BufferSize int
}
const (
_VERSION = "v2.0.0"
_VERSION = "v2.1.0"
)
var (
@ -34,14 +37,14 @@ var (
loop = true
ttl = time.Second * 55
done = make(chan bool)
msgs = make(chan string)
msgs chan string
count = 0
errCount = 0
)
func usage() {
fmt.Println("Usage: llmongo -m <mongo uri> -r <redis uri> -t <ttl> -l <logfile> -T -D -v\n")
fmt.Println("Usage: llmongo -m <mongo uri> -r <redis uri> -t <ttl> -l <logfile> -b <buffer size> -T -D -v\n")
os.Exit(0)
}
@ -63,6 +66,7 @@ func init() {
flag.BoolVar(&opts.Version, "v", false, "Version")
flag.BoolVar(&opts.Timeout, "T", false, "Timeout")
flag.BoolVar(&opts.Debug, "D", false, "Debug")
flag.IntVar(&opts.BufferSize, "b", 1, "Buffer size")
}
func stopLoop() {
@ -78,6 +82,14 @@ func main() {
os.Exit(0)
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
signal.Notify(c, syscall.SIGTERM)
go func() {
<-c
stopLoop()
}()
fs, err := os.OpenFile(opts.LogFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
fmt.Println("Log file error: ", err.Error())
@ -104,10 +116,19 @@ func main() {
time.AfterFunc(ttl, stopLoop)
}
go producer()
go consumer()
<-done
msgs = make(chan string, opts.BufferSize)
fmt.Printf("Stop %v - login: %d - errors: %d\n\r", time.Since(start), count, errCount)
log.Printf("Stop %v - login: %d - errors: %d\n\r", time.Since(start), count, errCount)
go producer()
for loop {
start := time.Now()
user := <-msgs
fmt.Printf("Wait: %v\n\r", time.Since(start))
if user != "" {
go consumer(user)
}
}
fmt.Printf("Stop %v - login: %d - errors: %d - conn: %d\n\r", time.Since(start), count, errCount, dbs.rdb.ActiveCount())
log.Printf("Stop %v - login: %d - errors: %d - conn: %d\n\r", time.Since(start), count, errCount, dbs.rdb.ActiveCount())
}

8
producer.go
View file

@ -5,6 +5,7 @@ import (
"fmt"
"github.com/garyburd/redigo/redis"
"log"
"time"
)
func producer() {
@ -12,11 +13,12 @@ func producer() {
defer conn.Close()
for loop {
start := time.Now()
// estrae un userid dalla lista degli utenti che hanno fatto login
user, err := redis.String(conn.Do("spop", "llindex"))
if opts.Debug {
log.Printf("SPOP: %+v %+v - conn: %d\n\r", user, err)
fmt.Printf("SPOP: %+v %+v - conn: %d\n\r", user, err)
log.Printf("SPOP: %+v - %+v\n\r", user, err)
fmt.Printf("SPOP: %+v - %+v\n\r", user, err)
}
// se non ci sono piu' userid esce
if err != nil {
@ -27,6 +29,6 @@ func producer() {
break
}
msgs <- user
fmt.Printf("PROD: user=%s in %v - conn=%d\n\r", user, time.Since(start), dbs.rdb.ActiveCount())
}
done <- true
}