new processor POP3 - usable as input or filter

sinceDB Unit can now handle []byte data
move examples.d to cmd/bitfan
vendoring

api/swagger.json

@@ -8,10 +8,7 @@
         "ContentType": {
           "type": "string"
         },
-        "CreatedAt": {
-          "type": "string",
-          "format": "date-time"
-        },
+        "CreatedAt": {},
         "Name": {
           "type": "string"
         },
@@ -25,10 +22,7 @@
         "Type": {
           "type": "string"
         },
-        "UpdatedAt": {
-          "type": "string",
-          "format": "date-time"
-        },
+        "UpdatedAt": {},
         "Value": {
           "type": "array",
           "items": {
@@ -63,10 +57,7 @@
         "Active": {
           "type": "boolean"
         },
-        "CreatedAt": {
-          "type": "string",
-          "format": "date-time"
-        },
+        "CreatedAt": {},
         "Description": {
           "type": "string"
         },
@@ -76,10 +67,7 @@
         "Playground": {
           "type": "boolean"
         },
-        "UpdatedAt": {
-          "type": "string",
-          "format": "date-time"
-        },
+        "UpdatedAt": {},
         "Webhooks": {
           "type": "array",
           "items": {
@@ -114,8 +102,6 @@
           "x-go-name": "Label"
         },
         "started_at": {
-          "type": "string",
-          "format": "date-time",
           "x-go-name": "StartedAt"
         },
         "uuid": {

cmd/bitfan/commands/plugins.go

@@ -43,6 +43,7 @@ import (
 	null "github.com/vjeantet/bitfan/processors/output-null"
 	rabbitmqoutput "github.com/vjeantet/bitfan/processors/output-rabbitmq"
 	statsd "github.com/vjeantet/bitfan/processors/output-statsd"
+	pop3processor "github.com/vjeantet/bitfan/processors/pop3"
 	route "github.com/vjeantet/bitfan/processors/route"
 	stdout "github.com/vjeantet/bitfan/processors/stdout"
 	websocket "github.com/vjeantet/bitfan/processors/websocket"
@@ -80,6 +81,7 @@ func init() {
 	initPlugin("input", "httpserver", inputhttpserverprocessor.New)
 	initPlugin("input", "event", inputeventprocessor.New)
 	initPlugin("input", "websocket", websocketinput.New)
+	initPlugin("input", "pop3", pop3processor.New)
 
 	initPlugin("filter", "eval", evalprocessor.New)
 	initPlugin("filter", "readfile", file.New)
@@ -107,6 +109,7 @@ func init() {
 	initPlugin("filter", "stdout", stdout.New)
 	initPlugin("filter", "http", httppoller.New)
 	initPlugin("filter", "httpout", httpoutprocessor.New)
+	initPlugin("filter", "pop3", pop3processor.New)
 
 	initPlugin("filter", "use", use.New)
 	initPlugin("filter", "route", route.New)

cmd/bitfan/examples.d/pop3.conf

@@ -0,0 +1,22 @@
+input{
+stdin{}
+}
+
+filter{
+	pop3{
+		host => "pop.free.fr"
+		username => "[email protected]"
+		password => "${BITFAN_POP3PASSWORD}"
+		secure => true
+		delete => false
+		sincedb_path => "/dev/null"
+		strip_attachments => true
+
+	}
+}
+
+output{
+	stdout{
+		codec=>pp
+	}
+}

docs/content/filters/pop3.md

@@ -0,0 +1,99 @@
++++
+title = "pop3"
+description = "Read mail from a pop3 server"
+weight = 10
++++
+
+Used on filter section, you may omit the `interval` option to scan for new emails only when a message comes from input.
+
+{{% processordetails pop3processor %}}
+
+
+## Produced event examples
+{{%expand%}}
+```
+{
+  "delivered-to": {
+    "[email protected]": "",
+  },
+  "cc": {
+    "[email protected]": "",
+  },
+  "from": {
+    "[email protected]": "Valere Jeantet",
+  },
+  "@timestamp":  2018-01-14 00:22:57 Local,
+  "uid":         "0acv1f1810845a5aa63e020051224870",
+  "subject":     "MSGBI",
+  "html":        "<div dir=\"ltr\"><img src=\"cid:160f19327978e9f50f81\" alt=\"Capture d’écran 2018-01-12 à 12.11.52.png\" class=\"\" style=\"max-width: 100%;\"><br></div>\r\n",
+  "return-path": {
+    "[email protected]": "",
+  },
+  "to": {
+    "[email protected]":           "[email protected]",
+    "[email protected]": "Valere Jeantet",
+  },
+  "message": "[image: Capture d’écran 2018-01-12 à 12.11.52.png]\r\n",
+  "headers": {
+    "Return-Path": []string{
+      "<[email protected]>",
+    },
+    "Dkim-Signature": []string{
+      "v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=XXX; h=mime-version:from:date:message-id:subject:to:cc; bh=ODEQpVa/5Qcw==",
+    },
+    "X-Received": []string{
+      "by 10.XXX.55.XX with SMTP id z37mr30058367uad.XXX.1515881XXX; Sat, 13 Jan 2018 14:11:25 -0800 (PST)",
+    },
+    "Content-Type": []string{
+      "multipart/mixed; boundary=\"001a11409cc4725d6c0562afa937\"",
+    },
+    "Received": []string{
+      "from mail-SSS-f17X.google.com (mx25XXX.priv.proxad.net [172.XXX.XXX.95]) by SSSS-g26.priv.proxad.net (Postfix) with ESMTP id 1C055A00634 for <[email protected]>; Sat, 13 Jan 2018 23:11:27 +0100 (CET)",
+      "from mail-ua0-XXXX.google.com ([209.XXX.XXX.179]) by mx1-XXX.XXX.fr (MXproxy) with ESMTPS for [email protected] (version=TLSv1/SSLv3 cipher=AES128-XXX-SHA256 bits=128); Sat, 13 Jan 2018 23:11:28 +0100 (CET)",
+      "by mail-ua0-SSSS.google.com with SMTP id x10so6315721ual.8 for <[email protected]>; Sat, 13 Jan 2018 14:11:26 -0800 (PST)",
+    },
+    "X-Gm-Message-State": []string{
+      "AKwxytfekxmrRWAc21hNb05lq kKc+spWodYAlFaCMPvuFfWOAjybzg==",
+    },
+    "Mime-Version": []string{
+      "1.0",
+    },
+    "From": []string{
+      "Valere Jeantet <[email protected]>",
+    },
+    "Message-Id": []string{
+      "<[email protected]>",
+    },
+    "To": []string{
+      "\"[email protected]\" <[email protected]>, Valere Jeantet <[email protected]>",
+    },
+    "Delivered-To": []string{
+      "[email protected]",
+    },
+    "X-Proxad-Sc": []string{
+      "state=HAM score=0",
+    },
+    "X-Proxad-Cause": []string{
+      "(null)",
+    },
+    "X-Google-Smtp-Source": []string{
+      "ACJfBosF8RqtE9BCxZvUZbC0WnryH5nlfIn12xmVhWoP3KazNLz4RfB82gEdZofHxtKo=",
+    },
+    "Subject": []string{
+      "Message from me",
+    },
+    "X-Google-Dkim-Signature": []string{
+      "v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20561025; h=x-gm-message-state:mime-version:from:date:message-id:subject:to:cc; bh=ODEQpVa/ym6uT9lFT7X pdvw==",
+    },
+    "Date": []string{
+      "Sat, 13 Jan 2018 22:11:13 +0000",
+    },
+    "Cc": []string{
+      "[email protected]",
+    },
+  },
+  "text":   "[image: Capture d’écran 2018-01-12 à 12.11.52.png]\r\n",
+  "sentAt": 2018-01-13 22:11:13 ,
+}
+```
+{{%expand%}}

docs/content/inputs/pop3.md

@@ -0,0 +1,96 @@
++++
+title = "pop3"
+description = "Read mail from a pop3 server"
+weight = 10
++++
+
+{{% processordetails pop3processor %}}
+
+## Produced event examples
+{{%expand%}}
+```
+{
+  "delivered-to": {
+    "[email protected]": "",
+  },
+  "cc": {
+    "[email protected]": "",
+  },
+  "from": {
+    "[email protected]": "Valere Jeantet",
+  },
+  "@timestamp":  2018-01-14 00:22:57 Local,
+  "uid":         "0acv1f1810845a5aa63e020051224870",
+  "subject":     "MSGBI",
+  "html":        "<div dir=\"ltr\"><img src=\"cid:160f19327978e9f50f81\" alt=\"Capture d’écran 2018-01-12 à 12.11.52.png\" class=\"\" style=\"max-width: 100%;\"><br></div>\r\n",
+  "return-path": {
+    "[email protected]": "",
+  },
+  "to": {
+    "[email protected]":           "[email protected]",
+    "[email protected]": "Valere Jeantet",
+  },
+  "message": "[image: Capture d’écran 2018-01-12 à 12.11.52.png]\r\n",
+  "headers": {
+    "Return-Path": []string{
+      "<[email protected]>",
+    },
+    "Dkim-Signature": []string{
+      "v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=XXX; h=mime-version:from:date:message-id:subject:to:cc; bh=ODEQpVa/5Qcw==",
+    },
+    "X-Received": []string{
+      "by 10.XXX.55.XX with SMTP id z37mr30058367uad.XXX.1515881XXX; Sat, 13 Jan 2018 14:11:25 -0800 (PST)",
+    },
+    "Content-Type": []string{
+      "multipart/mixed; boundary=\"001a11409cc4725d6c0562afa937\"",
+    },
+    "Received": []string{
+      "from mail-SSS-f17X.google.com (mx25XXX.priv.proxad.net [172.XXX.XXX.95]) by SSSS-g26.priv.proxad.net (Postfix) with ESMTP id 1C055A00634 for <[email protected]>; Sat, 13 Jan 2018 23:11:27 +0100 (CET)",
+      "from mail-ua0-XXXX.google.com ([209.XXX.XXX.179]) by mx1-XXX.XXX.fr (MXproxy) with ESMTPS for [email protected] (version=TLSv1/SSLv3 cipher=AES128-XXX-SHA256 bits=128); Sat, 13 Jan 2018 23:11:28 +0100 (CET)",
+      "by mail-ua0-SSSS.google.com with SMTP id x10so6315721ual.8 for <[email protected]>; Sat, 13 Jan 2018 14:11:26 -0800 (PST)",
+    },
+    "X-Gm-Message-State": []string{
+      "AKwxytfekxmrRWAc21hNb05lq kKc+spWodYAlFaCMPvuFfWOAjybzg==",
+    },
+    "Mime-Version": []string{
+      "1.0",
+    },
+    "From": []string{
+      "Valere Jeantet <[email protected]>",
+    },
+    "Message-Id": []string{
+      "<[email protected]>",
+    },
+    "To": []string{
+      "\"[email protected]\" <[email protected]>, Valere Jeantet <[email protected]>",
+    },
+    "Delivered-To": []string{
+      "[email protected]",
+    },
+    "X-Proxad-Sc": []string{
+      "state=HAM score=0",
+    },
+    "X-Proxad-Cause": []string{
+      "(null)",
+    },
+    "X-Google-Smtp-Source": []string{
+      "ACJfBosF8RqtE9BCxZvUZbC0WnryH5nlfIn12xmVhWoP3KazNLz4RfB82gEdZofHxtKo=",
+    },
+    "Subject": []string{
+      "Message from me",
+    },
+    "X-Google-Dkim-Signature": []string{
+      "v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20561025; h=x-gm-message-state:mime-version:from:date:message-id:subject:to:cc; bh=ODEQpVa/ym6uT9lFT7X pdvw==",
+    },
+    "Date": []string{
+      "Sat, 13 Jan 2018 22:11:13 +0000",
+    },
+    "Cc": []string{
+      "[email protected]",
+    },
+  },
+  "text":   "[image: Capture d’écran 2018-01-12 à 12.11.52.png]\r\n",
+  "sentAt": 2018-01-13 22:11:13 ,
+}
+```
+{{%expand%}}

docs/content/inputs/websocket.md

@@ -0,0 +1,7 @@
++++
+title = "websocket"
+description = ""
+weight = 10
++++
+
+{{% processordetails websocketinput %}}

docs/content/outputs/websocket.md

@@ -0,0 +1,7 @@
++++
+title = "websocket"
+description = ""
+weight = 10
++++
+
+{{% processordetails websocket %}}

docs/data/processors/geoip.json

@@ -1,5 +1,5 @@
 {
-  "Doc": "",
+  "Doc": "The GeoIP filter adds information about the geographical location of IP addresses,\nbased on data from the Maxmind GeoLite2 databases\n\nThis processor use a GeoLite2 City database. From Maxmind’s description — \"GeoLite2 databases are free IP geolocation databases comparable to, but less accurate than, MaxMind’s GeoIP2 databases\". Please see GeoIP Lite2 license for more details.\nDatabae is not bundled in the processor,  you can download directly from Maxmind’s website and use the\ndatabase option to specify their location. The GeoLite2 databases can be downloaded from https://dev.maxmind.com/geoip/geoip2/geolite2.",
   "DocShort": "Adds geographical information about an IP address",
   "ImportPath": "github.com/vjeantet/bitfan/processors/filter-geoip",
   "Name": "geoip",
@@ -18,7 +18,7 @@
       },
       {
         "Alias": "database",
-        "DefaultValue": null,
+        "DefaultValue": "\"http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz\"",
         "Doc": "Path or URL to the MaxMind GeoIP2 database.\nDefault value is \"http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz\"\nNote that URL can point to gzipped database (*.mmdb.gz) but local path must point to an unzipped file.",
         "ExampleLS": "",
         "Name": "Database",