Compare commits
2 commits
Author | SHA1 | Date | |
---|---|---|---|
7b67503f3a | |||
85f88e40c7 |
7 changed files with 166 additions and 22 deletions
7
Docker/Dockerfile
Normal file
7
Docker/Dockerfile
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
FROM scratch
|
||||||
|
|
||||||
|
MAINTAINER Michele Fadda "<mikif70@gmail.com>"
|
||||||
|
|
||||||
|
COPY imp_mongodb-v1.1.0 /bin/imp_mongodb
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/bin/imp_mongodb" ]
|
BIN
Docker/imp_mongodb-v1.1.0
Executable file
BIN
Docker/imp_mongodb-v1.1.0
Executable file
Binary file not shown.
14
Docker/run.sh
Normal file
14
Docker/run.sh
Normal file
|
@ -0,0 +1,14 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
docker run \
|
||||||
|
--rm \
|
||||||
|
-v /opt/impmongo/log:/data \
|
||||||
|
--name impmongo \
|
||||||
|
--log-opt max-size=2m \
|
||||||
|
--log-opt max-file=5 \
|
||||||
|
mikif70/impmongo:1.1.0 \
|
||||||
|
-l /data/llmongo.log \
|
||||||
|
-p /data/llmongo.pid \
|
||||||
|
-m mongodb://10.39.80.189:27017 \
|
||||||
|
-i http://10.39.109.107:8086 \
|
||||||
|
$@
|
3
build.sh
Executable file
3
build.sh
Executable file
|
@ -0,0 +1,3 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo .
|
78
influxdb.go
Normal file
78
influxdb.go
Normal file
|
@ -0,0 +1,78 @@
|
||||||
|
// influxdb
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
// "time"
|
||||||
|
|
||||||
|
influxdb "github.com/influxdata/influxdb/client/v2"
|
||||||
|
)
|
||||||
|
|
||||||
|
func writeStats(mimp *MongoImp) {
|
||||||
|
if opts.Debug {
|
||||||
|
fmt.Printf("writing to influxdb server: %s", opts.Influxdb)
|
||||||
|
}
|
||||||
|
|
||||||
|
c, err := influxdb.NewHTTPClient(influxdb.HTTPConfig{
|
||||||
|
Addr: opts.Influxdb,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error: %+v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer c.Close()
|
||||||
|
|
||||||
|
bp, err := influxdb.NewBatchPoints(influxdb.BatchPointsConfig{
|
||||||
|
Database: "dovecot",
|
||||||
|
Precision: "s",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error: %+v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
tags := map[string]string{"server": mimp.Host}
|
||||||
|
fields := map[string]interface{}{
|
||||||
|
"banner_legit": mimp.Banner.Legit,
|
||||||
|
"banner_spam": mimp.Banner.Spam,
|
||||||
|
"banner_virus": mimp.Banner.Virus,
|
||||||
|
"auth_legit": mimp.Auth.Legit,
|
||||||
|
"auth_spam": mimp.Auth.Spam,
|
||||||
|
"auth_virus": mimp.Auth.Virus,
|
||||||
|
"content_legit": mimp.Content.Legit,
|
||||||
|
"content_spam": mimp.Content.Spam,
|
||||||
|
"content_virus": mimp.Content.Virus,
|
||||||
|
"data_legit": mimp.Data.Legit,
|
||||||
|
"data_spam": mimp.Data.Spam,
|
||||||
|
"data_virus": mimp.Data.Virus,
|
||||||
|
"mailfrom_legit": mimp.Mailfrom.Legit,
|
||||||
|
"mailfrom_spam": mimp.Mailfrom.Spam,
|
||||||
|
"mailfrom_virus": mimp.Mailfrom.Virus,
|
||||||
|
"rcptto_legit": mimp.Rcptto.Legit,
|
||||||
|
"rcptto_spam": mimp.Rcptto.Spam,
|
||||||
|
"rcptto_virus": mimp.Rcptto.Virus,
|
||||||
|
"rate_content": mimp.Rate.Contentp,
|
||||||
|
"rate_smtp": mimp.Rate.Smtpp,
|
||||||
|
"rate_total": mimp.Rate.Totalp,
|
||||||
|
}
|
||||||
|
if opts.Out {
|
||||||
|
pt, err := influxdb.NewPoint("cloudmark_out", tags, fields, mimp.Date)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error: %+v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
bp.AddPoint(pt)
|
||||||
|
} else {
|
||||||
|
pt, err := influxdb.NewPoint("cloudmark", tags, fields, mimp.Date)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error: %+v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
bp.AddPoint(pt)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write the batch
|
||||||
|
c.Write(bp)
|
||||||
|
}
|
74
main.go
74
main.go
|
@ -17,18 +17,20 @@ import (
|
||||||
|
|
||||||
type Options struct {
|
type Options struct {
|
||||||
MongoUri string
|
MongoUri string
|
||||||
|
Influxdb string
|
||||||
mdb *mgo.Session
|
mdb *mgo.Session
|
||||||
ll *mgo.Collection
|
ll *mgo.Collection
|
||||||
LogFile string
|
LogFile string
|
||||||
DateStart string
|
DateStart string
|
||||||
TimeStart time.Time
|
TimeStart time.Time
|
||||||
DateStop time.Duration
|
DateStop time.Duration
|
||||||
|
Out bool
|
||||||
Version bool
|
Version bool
|
||||||
Debug bool
|
Debug bool
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
_VERSION = "v1.0.4"
|
_VERSION = "v1.1.0"
|
||||||
_dashboard = "/dashboard?suser=admin&spass=FreakKitchen&type=counters&view=hour&date="
|
_dashboard = "/dashboard?suser=admin&spass=FreakKitchen&type=counters&view=hour&date="
|
||||||
_domain = ".mail.tiscali.sys"
|
_domain = ".mail.tiscali.sys"
|
||||||
_port = "8082"
|
_port = "8082"
|
||||||
|
@ -37,6 +39,7 @@ const (
|
||||||
|
|
||||||
var (
|
var (
|
||||||
_imp = [...]string{"cmgw-1", "cmgw-2", "cmgw-3", "cmgw-4"}
|
_imp = [...]string{"cmgw-1", "cmgw-2", "cmgw-3", "cmgw-4"}
|
||||||
|
_out = [...]string{"michael", "santino"}
|
||||||
opts = Options{
|
opts = Options{
|
||||||
MongoUri: "mongodb://127.0.0.1:27018",
|
MongoUri: "mongodb://127.0.0.1:27018",
|
||||||
LogFile: "log/imp.log",
|
LogFile: "log/imp.log",
|
||||||
|
@ -53,11 +56,16 @@ func connectMongo() {
|
||||||
log.Println("Mongodb connect Error: ", err.Error())
|
log.Println("Mongodb connect Error: ", err.Error())
|
||||||
os.Exit(-3)
|
os.Exit(-3)
|
||||||
}
|
}
|
||||||
opts.ll = opts.mdb.DB("antispam").C("counters")
|
if opts.Out {
|
||||||
|
opts.ll = opts.mdb.DB("antispam").C("output")
|
||||||
|
} else {
|
||||||
|
opts.ll = opts.mdb.DB("antispam").C("counters")
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func usage() {
|
func usage() {
|
||||||
fmt.Println("Usage: imp_mongo -d <date (yyyy-mm-dd:hh)> -r <date range (HHh)> -m <mongodb uri> -l <logfile> -v -D\n")
|
fmt.Println("Usage: imp_mongo -d <date (yyyy-mm-dd:hh)> -r <date range (HHh)> -m <mongodb uri> -l <logfile> -i <influxdb> -v -D\n")
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -69,10 +77,12 @@ func init() {
|
||||||
|
|
||||||
opts.LogFile = path.Join(current, opts.LogFile)
|
opts.LogFile = path.Join(current, opts.LogFile)
|
||||||
|
|
||||||
flag.StringVar(&opts.MongoUri, "m", opts.MongoUri, "Mongodb")
|
flag.StringVar(&opts.MongoUri, "m", "", "Mongodb")
|
||||||
flag.StringVar(&opts.LogFile, "l", opts.LogFile, "Logs filename")
|
flag.StringVar(&opts.LogFile, "l", opts.LogFile, "Logs filename")
|
||||||
flag.StringVar(&opts.DateStart, "d", opts.DateStart, "From Date <YYYY-MM-DD:HH>")
|
flag.StringVar(&opts.DateStart, "d", opts.DateStart, "From Date <YYYY-MM-DD:HH>")
|
||||||
flag.DurationVar(&opts.DateStop, "r", opts.DateStop, "Duration <HH>h")
|
flag.DurationVar(&opts.DateStop, "r", opts.DateStop, "Duration <HH>h")
|
||||||
|
flag.StringVar(&opts.Influxdb, "i", "", "Influxdb")
|
||||||
|
flag.BoolVar(&opts.Out, "O", false, "Out")
|
||||||
flag.BoolVar(&opts.Debug, "D", false, "Debug")
|
flag.BoolVar(&opts.Debug, "D", false, "Debug")
|
||||||
flag.BoolVar(&opts.Version, "v", false, "Version")
|
flag.BoolVar(&opts.Version, "v", false, "Version")
|
||||||
}
|
}
|
||||||
|
@ -104,6 +114,10 @@ func main() {
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if opts.MongoUri == "" && opts.Influxdb == "" {
|
||||||
|
usage()
|
||||||
|
}
|
||||||
|
|
||||||
fs, err := os.OpenFile(opts.LogFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
|
fs, err := os.OpenFile(opts.LogFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println("Log file error: ", err.Error())
|
fmt.Println("Log file error: ", err.Error())
|
||||||
|
@ -121,27 +135,49 @@ func main() {
|
||||||
fmt.Printf("Start: %+v\n", opts)
|
fmt.Printf("Start: %+v\n", opts)
|
||||||
log.Printf("Start: %+v\n", opts)
|
log.Printf("Start: %+v\n", opts)
|
||||||
|
|
||||||
connectMongo()
|
if opts.MongoUri != "" {
|
||||||
defer opts.mdb.Close()
|
connectMongo()
|
||||||
|
defer opts.mdb.Close()
|
||||||
|
}
|
||||||
|
|
||||||
for h := 0; h < int(opts.DateStop.Hours()); h++ {
|
for h := 0; h < int(opts.DateStop.Hours()); h++ {
|
||||||
stime := opts.TimeStart.Add(time.Hour * time.Duration(h))
|
stime := opts.TimeStart.Add(time.Hour * time.Duration(h))
|
||||||
// fmt.Println("Time: ", stime)
|
// fmt.Println("Time: ", stime)
|
||||||
for ind := range _imp {
|
|
||||||
uri := "http://" + _imp[ind] + _domain + ":" + _port + _dashboard + stime.Format("2006010215") + "00"
|
|
||||||
// fmt.Println(uri)
|
|
||||||
resp, err := http.Get(uri)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
body, err := ioutil.ReadAll(resp.Body)
|
|
||||||
resp.Body.Close()
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
parse(string(body), stime)
|
if opts.Out {
|
||||||
|
for ind := range _out {
|
||||||
|
uri := "http://" + _out[ind] + _domain + ":" + _port + _dashboard + stime.Format("2006010215") + "00"
|
||||||
|
// fmt.Println(uri)
|
||||||
|
resp, err := http.Get(uri)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := ioutil.ReadAll(resp.Body)
|
||||||
|
resp.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
parse(string(body), stime)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for ind := range _imp {
|
||||||
|
uri := "http://" + _imp[ind] + _domain + ":" + _port + _dashboard + stime.Format("2006010215") + "00"
|
||||||
|
// fmt.Println(uri)
|
||||||
|
resp, err := http.Get(uri)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := ioutil.ReadAll(resp.Body)
|
||||||
|
resp.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
parse(string(body), stime)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("Stop: %v\n", time.Since(start))
|
fmt.Printf("Stop: %v\n", time.Since(start))
|
||||||
|
|
12
parser.go
12
parser.go
|
@ -199,9 +199,15 @@ func parse(body string, stime time.Time) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := opts.ll.Upsert(index, mimp)
|
if opts.MongoUri != "" {
|
||||||
if err != nil {
|
_, err := opts.ll.Upsert(index, mimp)
|
||||||
log.Println("Insert Errro: ", err)
|
if err != nil {
|
||||||
|
log.Println("Insert Errro: ", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if opts.Influxdb != "" {
|
||||||
|
writeStats(&mimp)
|
||||||
}
|
}
|
||||||
|
|
||||||
if opts.Debug {
|
if opts.Debug {
|
||||||
|
|
Loading…
Add table
Reference in a new issue