compilazione statica + Docker

forzata l'ora in UTC

Docker/Dockerfile

@@ -0,0 +1,7 @@
+FROM		scratch
+
+MAINTAINER Michele Fadda "<mikif70@gmail.com>"
+
+COPY        imp_mongodb-v1.1.0 /bin/imp_mongodb
+
+ENTRYPOINT  [ "/bin/imp_mongodb" ]

Docker/run.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+docker run \
+        --rm \
+        -v /opt/impmongo/log:/data \
+        --name impmongo \
+        --log-opt max-size=2m \
+        --log-opt max-file=5 \
+        mikif70/impmongo:1.1.0 \
+                -l /data/llmongo.log \
+                -p /data/llmongo.pid \
+                -m mongodb://10.39.80.189:27017 \
+				-i http://10.39.109.107:8086 \
+                $@

build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo .

influxdb.go

@@ -0,0 +1,78 @@
+// influxdb
+package main
+
+import (
+	"fmt"
+	//	"time"
+
+	influxdb "github.com/influxdata/influxdb/client/v2"
+)
+
+func writeStats(mimp *MongoImp) {
+	if opts.Debug {
+		fmt.Printf("writing to influxdb server: %s", opts.Influxdb)
+	}
+
+	c, err := influxdb.NewHTTPClient(influxdb.HTTPConfig{
+		Addr: opts.Influxdb,
+	})
+	if err != nil {
+		fmt.Printf("Error: %+v\n", err)
+		return
+	}
+	defer c.Close()
+
+	bp, err := influxdb.NewBatchPoints(influxdb.BatchPointsConfig{
+		Database:  "dovecot",
+		Precision: "s",
+	})
+	if err != nil {
+		fmt.Printf("Error: %+v\n", err)
+		return
+	}
+
+	tags := map[string]string{"server": mimp.Host}
+	fields := map[string]interface{}{
+		"banner_legit":   mimp.Banner.Legit,
+		"banner_spam":    mimp.Banner.Spam,
+		"banner_virus":   mimp.Banner.Virus,
+		"auth_legit":     mimp.Auth.Legit,
+		"auth_spam":      mimp.Auth.Spam,
+		"auth_virus":     mimp.Auth.Virus,
+		"content_legit":  mimp.Content.Legit,
+		"content_spam":   mimp.Content.Spam,
+		"content_virus":  mimp.Content.Virus,
+		"data_legit":     mimp.Data.Legit,
+		"data_spam":      mimp.Data.Spam,
+		"data_virus":     mimp.Data.Virus,
+		"mailfrom_legit": mimp.Mailfrom.Legit,
+		"mailfrom_spam":  mimp.Mailfrom.Spam,
+		"mailfrom_virus": mimp.Mailfrom.Virus,
+		"rcptto_legit":   mimp.Rcptto.Legit,
+		"rcptto_spam":    mimp.Rcptto.Spam,
+		"rcptto_virus":   mimp.Rcptto.Virus,
+		"rate_content":   mimp.Rate.Contentp,
+		"rate_smtp":      mimp.Rate.Smtpp,
+		"rate_total":     mimp.Rate.Totalp,
+	}
+	if opts.Out {
+		pt, err := influxdb.NewPoint("cloudmark_out", tags, fields, mimp.Date)
+		if err != nil {
+			fmt.Printf("Error: %+v\n", err)
+			return
+		}
+
+		bp.AddPoint(pt)
+	} else {
+		pt, err := influxdb.NewPoint("cloudmark", tags, fields, mimp.Date)
+		if err != nil {
+			fmt.Printf("Error: %+v\n", err)
+			return
+		}
+
+		bp.AddPoint(pt)
+	}
+
+	// Write the batch
+	c.Write(bp)
+}

main.go

@@ -4,7 +4,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"gopkg.in/mgo.v2"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -12,19 +11,26 @@ import (
 	"path"
 	"path/filepath"
 	"time"
+
+	"gopkg.in/mgo.v2"
 )
 
 type Options struct {
 	MongoUri  string
+	Influxdb  string
 	mdb       *mgo.Session
 	ll        *mgo.Collection
 	LogFile   string
 	DateStart string
 	TimeStart time.Time
 	DateStop  time.Duration
+	Out       bool
+	Version   bool
+	Debug     bool
 }
 
 const (
+	_VERSION   = "v1.1.0"
 	_dashboard = "/dashboard?suser=admin&spass=FreakKitchen&type=counters&view=hour&date="
 	_domain    = ".mail.tiscali.sys"
 	_port      = "8082"
@@ -32,11 +38,12 @@ const (
 )
 
 var (
-	_imp = [...]string{"imp-1", "imp-2", "imp-3", "imp-6"}
+	_imp = [...]string{"cmgw-1", "cmgw-2", "cmgw-3", "cmgw-4"}
+	_out = [...]string{"michael", "santino"}
 	opts = Options{
 		MongoUri:  "mongodb://127.0.0.1:27018",
 		LogFile:   "log/imp.log",
-		DateStart: time.Now().Add(-time.Hour).Format(_tformat),
+		DateStart: time.Now().UTC().Add(-time.Hour).Format(_tformat),
 		DateStop:  time.Hour,
 	}
 	mimp = MongoImp{}
@@ -49,11 +56,16 @@ func connectMongo() {
 		log.Println("Mongodb connect Error: ", err.Error())
 		os.Exit(-3)
 	}
-	opts.ll = opts.mdb.DB("antispam").C("counters")
+	if opts.Out {
+		opts.ll = opts.mdb.DB("antispam").C("output")
+	} else {
+		opts.ll = opts.mdb.DB("antispam").C("counters")
+	}
+
 }
 
 func usage() {
-	fmt.Println("Usage: imp_mongo -d <date> -r <date range> -m <mongodb uri> -l <logfile>\n")
+	fmt.Println("Usage: imp_mongo -d <date (yyyy-mm-dd:hh)> -r <date range (HHh)> -m <mongodb uri> -l <logfile> -i <influxdb> -v -D\n")
 	os.Exit(0)
 }
 
@@ -65,10 +77,14 @@ func init() {
 
 	opts.LogFile = path.Join(current, opts.LogFile)
 
-	flag.StringVar(&opts.MongoUri, "m", opts.MongoUri, "Mongodb")
+	flag.StringVar(&opts.MongoUri, "m", "", "Mongodb")
 	flag.StringVar(&opts.LogFile, "l", opts.LogFile, "Logs filename")
 	flag.StringVar(&opts.DateStart, "d", opts.DateStart, "From Date <YYYY-MM-DD:HH>")
 	flag.DurationVar(&opts.DateStop, "r", opts.DateStop, "Duration <HH>h")
+	flag.StringVar(&opts.Influxdb, "i", "", "Influxdb")
+	flag.BoolVar(&opts.Out, "O", false, "Out")
+	flag.BoolVar(&opts.Debug, "D", false, "Debug")
+	flag.BoolVar(&opts.Version, "v", false, "Version")
 }
 
 func parseDate(date string) string {
@@ -93,6 +109,15 @@ func main() {
 	flag.Usage = usage
 	flag.Parse()
 
+	if opts.Version {
+		fmt.Println(os.Args[0], _VERSION)
+		os.Exit(0)
+	}
+
+	if opts.MongoUri == "" && opts.Influxdb == "" {
+		usage()
+	}
+
 	fs, err := os.OpenFile(opts.LogFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
 	if err != nil {
 		fmt.Println("Log file error: ", err.Error())
@@ -110,30 +135,50 @@ func main() {
 	fmt.Printf("Start: %+v\n", opts)
 	log.Printf("Start: %+v\n", opts)
 
-	connectMongo()
+	if opts.MongoUri != "" {
-	defer opts.mdb.Close()
+		connectMongo()
+		defer opts.mdb.Close()
+	}
 
 	for h := 0; h < int(opts.DateStop.Hours()); h++ {
 		stime := opts.TimeStart.Add(time.Hour * time.Duration(h))
 		//		fmt.Println("Time: ", stime)
-		for ind := range _imp {
-			uri := "http://" + _imp[ind] + _domain + ":" + _port + _dashboard + stime.Format("2006010215") + "00"
-			//			fmt.Println(uri)
-			resp, err := http.Get(uri)
-			if err != nil {
-				log.Fatal(err)
-			}
-			body, err := ioutil.ReadAll(resp.Body)
-			resp.Body.Close()
-			if err != nil {
-				log.Fatal(err)
-			}
 
-			parse(string(body), stime)
+		if opts.Out {
+			for ind := range _out {
+				uri := "http://" + _out[ind] + _domain + ":" + _port + _dashboard + stime.Format("2006010215") + "00"
+				//			fmt.Println(uri)
+				resp, err := http.Get(uri)
+				if err != nil {
+					log.Fatal(err)
+				}
+				body, err := ioutil.ReadAll(resp.Body)
+				resp.Body.Close()
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				parse(string(body), stime)
+			}
+		} else {
+			for ind := range _imp {
+				uri := "http://" + _imp[ind] + _domain + ":" + _port + _dashboard + stime.Format("2006010215") + "00"
+				//			fmt.Println(uri)
+				resp, err := http.Get(uri)
+				if err != nil {
+					log.Fatal(err)
+				}
+				body, err := ioutil.ReadAll(resp.Body)
+				resp.Body.Close()
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				parse(string(body), stime)
+			}
 		}
-	}
 
-	//	fmt.Printf("%+v\n", mimp)
+	}
 
 	fmt.Printf("Stop: %v\n", time.Since(start))
 	log.Printf("Stop: %v\n", time.Since(start))

parser.go

@@ -2,70 +2,97 @@
 package main
 
 import (
+	"fmt"
 	"log"
+	"strconv"
 	"strings"
 	"time"
 )
 
 type Banner struct {
-	Legit string `json:"legit" bson:"legit"`
+	Legit int `json:"legit" bson:"legit"`
-	Spam  string `json:"spam" bson:"spam"`
+	Spam  int `json:"spam" bson:"spam"`
-	Virus string `json:"virus" bson:"virus"`
+	Virus int `json:"virus" bson:"virus"`
 }
 
 type Heloehlo struct {
-	Legit string `json:"legit" bson:"legit"`
+	Legit int `json:"legit" bson:"legit"`
-	Spam  string `json:"spam" bson:"spam"`
+	Spam  int `json:"spam" bson:"spam"`
-	Virus string `json:"virus" bson:"virus"`
+	Virus int `json:"virus" bson:"virus"`
 }
 
 type Auth struct {
-	Legit string `json:"legit" bson:"legit"`
+	Legit int `json:"legit" bson:"legit"`
-	Spam  string `json:"spam" bson:"spam"`
+	Spam  int `json:"spam" bson:"spam"`
-	Virus string `json:"virus" bson:"virus"`
+	Virus int `json:"virus" bson:"virus"`
 }
 
 type Mailfrom struct {
-	Legit string `json:"legit" bson:"legit"`
+	Legit int `json:"legit" bson:"legit"`
-	Spam  string `json:"spam" bson:"spam"`
+	Spam  int `json:"spam" bson:"spam"`
-	Virus string `json:"virus" bson:"virus"`
+	Virus int `json:"virus" bson:"virus"`
 }
 
 type Rcptto struct {
-	Legit string `json:"legit" bson:"legit"`
+	Legit int `json:"legit" bson:"legit"`
-	Spam  string `json:"spam" bson:"spam"`
+	Spam  int `json:"spam" bson:"spam"`
-	Virus string `json:"virus" bson:"virus"`
+	Virus int `json:"virus" bson:"virus"`
 }
 
 type Data struct {
-	Legit string `json:"legit" bson:"legit"`
+	Legit int `json:"legit" bson:"legit"`
-	Spam  string `json:"spam" bson:"spam"`
+	Spam  int `json:"spam" bson:"spam"`
-	Virus string `json:"virus" bson:"virus"`
+	Virus int `json:"virus" bson:"virus"`
 }
 
 type Content struct {
-	Legit string `json:"legit" bson:"legit"`
+	Legit int `json:"legit" bson:"legit"`
-	Spam  string `json:"spam" bson:"spam"`
+	Spam  int `json:"spam" bson:"spam"`
-	Virus string `json:"virus" bson:"virus"`
+	Virus int `json:"virus" bson:"virus"`
 }
 
 type Rate struct {
-	Smtpp    string `json:"smtpp" bson:"smtpp"`
+	Smtpp    float32 `json:"smtpp" bson:"smtpp"`
-	Contentp string `json:"contentp" bson:"contentp"`
+	Contentp float32 `json:"contentp" bson:"contentp"`
-	Totalp   string `json:"totalp" bson:"totalp"`
+	Totalp   float32 `json:"totalp" bson:"totalp"`
 }
 
 type MongoImp struct {
-	Host     string    `json:"host"`
+	Host   string    `json:"host"`
-	Date     time.Time `json:"date"`
+	Date   time.Time `json:"date"`
-	Banner   Banner    `json:banner`
+	Banner Banner    `json:banner`
-	Heloehlo Heloehlo  `json:heloehlo`
+	//	Heloehlo Heloehlo  `json:heloehlo`
-	Auth     Auth      `json:auth`
+	Auth     Auth     `json:auth`
-	Mailfrom Mailfrom  `json:mailfrom`
+	Mailfrom Mailfrom `json:mailfrom`
-	Rcptto   Rcptto    `json:rcptto`
+	Rcptto   Rcptto   `json:rcptto`
-	Data     Data      `json:data`
+	Data     Data     `json:data`
-	Content  Content   `json:content`
+	Content  Content  `json:content`
-	Rate     Rate      `json:rate`
+	Rate     Rate     `json:rate`
+}
+
+type Index struct {
+	Host string    `json:"host"`
+	Date time.Time `json:"date"`
+}
+
+func stoi(num string) int {
+	n, err := strconv.Atoi(num)
+	if err != nil {
+		log.Printf("Error converting to int: ", err)
+		return 0
+	}
+
+	return n
+}
+
+func stof(num string) float32 {
+	n, err := strconv.ParseFloat(num, 32)
+	if err != nil {
+		log.Printf("Error converting to int: ", err)
+		return 0.0
+	}
+
+	return float32(n)
 }
 
 func parse(body string, stime time.Time) {
@@ -73,6 +100,7 @@ func parse(body string, stime time.Time) {
 	lines := strings.Split(body, "\n")
 
 	mimp := MongoImp{}
+	index := Index{}
 
 	for ind := range lines {
 		//		fmt.Println(lines[ind])
@@ -80,94 +108,109 @@ func parse(body string, stime time.Time) {
 			h := strings.Split(lines[ind], ":")
 			log.Printf("Host: %s - Date: %s\n", h[3], h[4])
 			mimp.Host = h[3]
+			index.Host = h[3]
 			mimp.Date = stime
+			index.Date = stime
 		} else if strings.HasPrefix(lines[ind], "counters") {
 			k := strings.Split(lines[ind], ";")
-			//			fmt.Printf("%s = %s\n", k[0], k[1])
 			key := strings.Split(k[0], "/")
-			//			fmt.Printf("Keys: %s - %s - %s\n", key[0], key[1], key[2])
+			if opts.Debug {
+				//			fmt.Printf("%s = %s\n", k[0], k[1])
+				//			fmt.Printf("Keys: %s - %s - %s\n", key[0], key[1], key[2])
+			}
 			if key[0] == "counters" {
 				switch key[1] {
 				case "banner":
 					switch key[2] {
 					case "legit":
-						mimp.Banner.Legit = k[1]
+						mimp.Banner.Legit = stoi(k[1])
 					case "spam":
-						mimp.Banner.Spam = k[1]
+						mimp.Banner.Spam = stoi(k[1])
 					case "virus":
-						mimp.Banner.Virus = k[1]
+						mimp.Banner.Virus = stoi(k[1])
-					}
-				case "heloehlo":
-					switch key[2] {
-					case "legit":
-						mimp.Heloehlo.Legit = k[1]
-					case "spam":
-						mimp.Heloehlo.Spam = k[1]
-					case "virus":
-						mimp.Heloehlo.Virus = k[1]
 					}
+					/*
+						case "heloehlo":
+							switch key[2] {
+							case "legit":
+								mimp.Heloehlo.Legit = stoi(k[1])
+							case "spam":
+								mimp.Heloehlo.Spam = stoi(k[1])
+							case "virus":
+								mimp.Heloehlo.Virus = stoi(k[1])
+							}
+					*/
 				case "auth":
 					switch key[2] {
 					case "legit":
-						mimp.Auth.Legit = k[1]
+						mimp.Auth.Legit = stoi(k[1])
 					case "spam":
-						mimp.Auth.Spam = k[1]
+						mimp.Auth.Spam = stoi(k[1])
 					case "virus":
-						mimp.Auth.Virus = k[1]
+						mimp.Auth.Virus = stoi(k[1])
 					}
 				case "mailfrom":
 					switch key[2] {
 					case "legit":
-						mimp.Mailfrom.Legit = k[1]
+						mimp.Mailfrom.Legit = stoi(k[1])
 					case "spam":
-						mimp.Mailfrom.Spam = k[1]
+						mimp.Mailfrom.Spam = stoi(k[1])
 					case "virus":
-						mimp.Mailfrom.Virus = k[1]
+						mimp.Mailfrom.Virus = stoi(k[1])
 					}
 				case "rcptto":
 					switch key[2] {
 					case "legit":
-						mimp.Rcptto.Legit = k[1]
+						mimp.Rcptto.Legit = stoi(k[1])
 					case "spam":
-						mimp.Rcptto.Spam = k[1]
+						mimp.Rcptto.Spam = stoi(k[1])
 					case "virus":
-						mimp.Rcptto.Virus = k[1]
+						mimp.Rcptto.Virus = stoi(k[1])
 					}
 				case "data":
 					switch key[2] {
 					case "legit":
-						mimp.Data.Legit = k[1]
+						mimp.Data.Legit = stoi(k[1])
 					case "spam":
-						mimp.Data.Spam = k[1]
+						mimp.Data.Spam = stoi(k[1])
 					case "virus":
-						mimp.Data.Virus = k[1]
+						mimp.Data.Virus = stoi(k[1])
 					}
 				case "content":
 					switch key[2] {
 					case "legit":
-						mimp.Content.Legit = k[1]
+						mimp.Content.Legit = stoi(k[1])
 					case "spam":
-						mimp.Content.Spam = k[1]
+						mimp.Content.Spam = stoi(k[1])
 					case "virus":
-						mimp.Content.Virus = k[1]
+						mimp.Content.Virus = stoi(k[1])
 					}
 				case "rate":
 					switch key[2] {
 					case "smtpp":
-						mimp.Rate.Smtpp = k[1]
+						mimp.Rate.Smtpp = stof(k[1])
 					case "contentp":
-						mimp.Rate.Contentp = k[1]
+						mimp.Rate.Contentp = stof(k[1])
 					case "totalp":
-						mimp.Rate.Totalp = k[1]
+						mimp.Rate.Totalp = stof(k[1])
 					}
 				}
 			}
 		}
 	}
 
-	err := opts.ll.Insert(mimp)
+	if opts.MongoUri != "" {
-	if err != nil {
+		_, err := opts.ll.Upsert(index, mimp)
-		log.Println("Insert Errro: ", err)
+		if err != nil {
+			log.Println("Insert Errro: ", err)
+		}
+	}
+
+	if opts.Influxdb != "" {
+		writeStats(&mimp)
+	}
+
+	if opts.Debug {
+		fmt.Printf("%+v\n", mimp)
 	}
-	//	fmt.Printf("%+v\n", mimp)
 }