Merge pull request #61 from istresearch/develop

Release 4.0.12

.gitignore

@@ -35,3 +35,4 @@ dist/
 .envrc
 courier
 .idea
+_storage

CHANGELOG.md

@@ -1,3 +1,142 @@
+v6.4.0
+----------
+ * 6.4.0 Release Candidate
+
+v6.3.5
+----------
+ * up max request size to 1M
+
+v6.3.4
+----------
+ * Include filename when sending WhatsApp attachments
+
+v6.3.3
+----------
+ * Support using namespace from the template translation
+ * Add is_resend to Msg payload to allow for resending messages manually
+
+v6.3.2
+----------
+ * Do not verify the SSL certificate for Bongo Live
+
+v6.3.1
+----------
+ * Update BL to remove UDH parameter and use HTTPS URL
+
+v6.2.2
+----------
+ * Handle whatsapp URNs sent to Twiml handler without prefix
+ * Add support for Zenvia SMS
+
+v6.2.1
+----------
+ * Add support for Zenvia WhatsApp
+
+v6.2.0
+----------
+ * Add handling for button whatsapp message type
+ * Bump CI testing to PG 11 and 12
+ * Add Kaleyra channel type
+ * 6.2.0 RC
+
+v6.1.7
+----------
+ * switch id to bigserial
+
+v6.1.6
+----------
+ * Cache media upload failures localy for 15m
+
+v6.1.5
+----------
+ * include header when sanitizing request/response
+
+v6.1.4
+----------
+ * Cleanup of whatsapp media handling
+ * Detect media type for uploading media
+
+v6.1.3
+----------
+ * Better logging of error cases when uploading WhatsApp media
+
+v6.1.2
+----------
+ * use url.parse to build media URL
+
+v6.1.1
+----------
+ * Add TextIt WhatsApp channel type
+
+v6.1.0
+----------
+ * Check and log errors when building URLs for sending
+
+v6.0.0
+----------
+ * Update README
+
+v5.7.12
+----------
+ * URN channel change only for channels with SEND role
+ * Update to gocommon v1.6.1
+ * Add RocketChat handler
+ * Add discord handler
+
+v5.7.11
+----------
+ * Cache media ids for WhatsApp attachments
+
+v5.7.10
+----------
+ * Support receiving Multipart form data requests for EX channels
+
+v5.7.9
+----------
+ * Update to latest gocommon 1.5.3 and golang 1.15
+ * Add session status from mailroom to MT message sent to external channel API call
+ * Remove incoming message prefix for Play Mobile free accounts
+
+v5.7.8
+----------
+ * deal with empty message in FreshChat incoming requests
+
+v5.7.7
+----------
+ * Update to gocommon v1.5.1
+
+v5.7.6
+----------
+ * Remove dummy values for AWS config values so you can use local file system for testing
+ * Use gsm7, storage, dates and uuids packages from gocommon
+
+v5.7.5
+----------
+ * No longer write contact.is_stopped or is_blocked
+
+v5.7.4
+----------
+ * Support receiving XML for CM channels
+ * Write status on new contacts
+ * Add support for Whatsapp 360dialog
+
+v5.7.3
+----------
+ * Include created_on in msg_event
+ * Include occurred_on when queueing channel events for mailroom
+
+v5.7.2
+----------
+ * Deal with Shaqodoon not properly escaping + in from
+
+v5.7.1
+----------
+ * Add ClickMobile channel type
+
+v5.7.0
+----------
+ * Save the Ad ID for Facebook postback referral 
+
 v5.6.0
 ----------
  * 5.6.0 Candidate Release

README.md

@@ -1,4 +1,8 @@
-# Courier [![Build Status](https://github.com/nyaruka/courier/workflows/CI/badge.svg)] [![codecov](https://codecov.io/gh/nyaruka/courier/branch/master/graph/badge.svg)](https://codecov.io/gh/nyaruka/courier) [![Go Report Card](https://goreportcard.com/badge/github.com/nyaruka/courier)](https://goreportcard.com/report/github.com/nyaruka/courier)
+# Courier
+
+[![Build Status](https://github.com/nyaruka/courier/workflows/CI/badge.svg)](https://github.com/nyaruka/courier/actions?query=workflow%3ACI) 
+[![codecov](https://codecov.io/gh/nyaruka/courier/branch/main/graph/badge.svg)](https://codecov.io/gh/nyaruka/courier)
+[![Go Report Card](https://goreportcard.com/badge/github.com/nyaruka/courier)](https://goreportcard.com/report/github.com/nyaruka/courier)
 
 # About 
 
@@ -9,7 +13,7 @@ Current courier supports over 36 different channel types, ranging for SMS aggreg
 IP channels like Facebook and Telegram messenger. The goal is for Courier to support every popular
 messaging channels and aggregator and we are happy to accept pull requests to help accomplish that.
 
-Courier is currently used to power [RapidPro](https://rapidpro.io) and [TextIt](https://textit.in)
+Courier is currently used to power [RapidPro](https://rapidpro.io) and [TextIt](https://textit.com)
 but the backend is pluggable, so you can add your own backend to read and write messages.
 
 # Deploying
@@ -55,16 +59,10 @@ Recommended settings for error and performance monitoring:
 
 # Development
 
-Install Courier source in your workspace with:
-
-```
-go get github.com/nyaruka/courier
-```
-
-Build Courier with:
+Once you've checked out the code, you can build Courier with:
 
 ```
-go install github.com/nyaruka/courier/cmd/...
+go install github.com/nyaruka/courier/cmd/courier
 ```
 
 This will create a new executable in $GOPATH/bin called `courier`. 
@@ -82,5 +80,5 @@ $ psql -d courier_test -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA publi
 To run all of the tests including benchmarks:
 
 ```
-go test github.com/nyaruka/courier/... -p=1 -bench=.
+go test ./... -p=1 -bench=.
 ```

VERSION

@@ -1 +1 @@
-4.0.11
+4.0.12

backend.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/garyburd/redigo/redis"
+	"github.com/gomodule/redigo/redis"
 	"github.com/nyaruka/gocommon/urns"
 )
 
@@ -68,7 +68,11 @@ type Backend interface {
 
 	// WasMsgSent returns whether the backend thinks the passed in message was already sent. This can be used in cases where
 	// a backend wants to implement a failsafe against double sending messages (say if they were double queued)
-	WasMsgSent(context.Context, Msg) (bool, error)
+	WasMsgSent(context.Context, MsgID) (bool, error)
+
+	// ClearMsgSent clears any internal status that a message was previously sent. This can be used in the case where
+	// a message is being forced in being resent by a user
+	ClearMsgSent(context.Context, MsgID) error
 
 	// IsMsgLoop returns whether the passed in message is part of a message loop, possibly with another bot. Backends should
 	// implement their own logic to implement this.

backends/rapidpro/backend.go

@@ -12,18 +12,15 @@ import (
 	"sync"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3iface"
-	"github.com/garyburd/redigo/redis"
+	"github.com/gomodule/redigo/redis"
 	"github.com/jmoiron/sqlx"
 	"github.com/nyaruka/courier"
 	"github.com/nyaruka/courier/batch"
 	"github.com/nyaruka/courier/chatbase"
 	"github.com/nyaruka/courier/queue"
 	"github.com/nyaruka/courier/utils"
+	"github.com/nyaruka/gocommon/storage"
 	"github.com/nyaruka/gocommon/urns"
 	"github.com/nyaruka/librato"
 	"github.com/pkg/errors"
@@ -81,7 +78,7 @@ func (b *backend) AddURNtoContact(ctx context.Context, c courier.Channel, contac
 	}
 	dbChannel := c.(*DBChannel)
 	dbContact := contact.(*DBContact)
-	_, err = contactURNForURN(tx, dbChannel.OrgID(), dbChannel.ID(), dbContact.ID_, urn, "")
+	_, err = contactURNForURN(tx, dbChannel, dbContact.ID_, urn, "")
 	if err != nil {
 		return urns.NilURN, err
 	}
@@ -186,13 +183,29 @@ var luaSent = redis.NewScript(3,
 `)
 
 // WasMsgSent returns whether the passed in message has already been sent
-func (b *backend) WasMsgSent(ctx context.Context, msg courier.Msg) (bool, error) {
+func (b *backend) WasMsgSent(ctx context.Context, id courier.MsgID) (bool, error) {
 	rc := b.redisPool.Get()
 	defer rc.Close()
 
 	todayKey := fmt.Sprintf(sentSetName, time.Now().UTC().Format("2006_01_02"))
 	yesterdayKey := fmt.Sprintf(sentSetName, time.Now().Add(time.Hour*-24).UTC().Format("2006_01_02"))
-	return redis.Bool(luaSent.Do(rc, todayKey, yesterdayKey, msg.ID().String()))
+	return redis.Bool(luaSent.Do(rc, todayKey, yesterdayKey, id.String()))
+}
+
+var luaClearSent = redis.NewScript(3,
+	`-- KEYS: [TodayKey, YesterdayKey, MsgID]
+	 redis.call("srem", KEYS[1], KEYS[3])
+     redis.call("srem", KEYS[2], KEYS[3])
+`)
+
+func (b *backend) ClearMsgSent(ctx context.Context, id courier.MsgID) error {
+	rc := b.redisPool.Get()
+	defer rc.Close()
+
+	todayKey := fmt.Sprintf(sentSetName, time.Now().UTC().Format("2006_01_02"))
+	yesterdayKey := fmt.Sprintf(sentSetName, time.Now().Add(time.Hour*-24).UTC().Format("2006_01_02"))
+	_, err := luaClearSent.Do(rc, todayKey, yesterdayKey, id.String())
+	return err
 }
 
 var luaMsgLoop = redis.NewScript(3, `-- KEYS: [key, contact_id, text]
@@ -333,16 +346,7 @@ func (b *backend) WriteMsgStatus(ctx context.Context, status courier.MsgStatus)
 
 	// if we have an id and are marking an outgoing msg as errored, then clear our sent flag
 	if status.ID() != courier.NilMsgID && status.Status() == courier.MsgErrored {
-		rc := b.redisPool.Get()
-		defer rc.Close()
-
-		dateKey := fmt.Sprintf(sentSetName, time.Now().UTC().Format("2006_01_02"))
-		prevDateKey := fmt.Sprintf(sentSetName, time.Now().Add(time.Hour*-24).UTC().Format("2006_01_02"))
-
-		// we pipeline the removals because we don't care about the return value
-		rc.Send("srem", dateKey, status.ID().String())
-		rc.Send("srem", prevDateKey, status.ID().String())
-		_, err := rc.Do("")
+		err := b.ClearMsgSent(ctx, status.ID())
 		if err != nil {
 			logrus.WithError(err).WithField("msg", status.ID().String()).Error("error clearing sent flags")
 		}
@@ -679,25 +683,30 @@ func (b *backend) Start() error {
 		queue.StartDethrottler(redisPool, b.stopChan, b.waitGroup, msgQueueName)
 	}
 
-	// create our s3 client
-	s3Session, err := session.NewSession(&aws.Config{
-		Credentials:      credentials.NewStaticCredentials(b.config.AWSAccessKeyID, b.config.AWSSecretAccessKey, ""),
-		Endpoint:         aws.String(b.config.S3Endpoint),
-		Region:           aws.String(b.config.S3Region),
-		DisableSSL:       aws.Bool(b.config.S3DisableSSL),
-		S3ForcePathStyle: aws.Bool(b.config.S3ForcePathStyle),
-	})
-	if err != nil {
-		return err
+	// create our storage (S3 or file system)
+	if b.config.AWSAccessKeyID != "" {
+		s3Client, err := storage.NewS3Client(&storage.S3Options{
+			AWSAccessKeyID:     b.config.AWSAccessKeyID,
+			AWSSecretAccessKey: b.config.AWSSecretAccessKey,
+			Endpoint:           b.config.S3Endpoint,
+			Region:             b.config.S3Region,
+			DisableSSL:         b.config.S3DisableSSL,
+			ForcePathStyle:     b.config.S3ForcePathStyle,
+		})
+		if err != nil {
+			return err
+		}
+		b.storage = storage.NewS3(s3Client, b.config.S3MediaBucket)
+	} else {
+		b.storage = storage.NewFS("_storage")
 	}
-	b.s3Client = s3.New(s3Session)
 
-	// test out our S3 credentials
-	err = utils.TestS3(b.s3Client, b.config.S3MediaBucket)
+	// test our storage
+	err = b.storage.Test()
 	if err != nil {
-		log.WithError(err).Error("s3 bucket not reachable")
+		log.WithError(err).Error(b.storage.Name() + " storage not available")
 	} else {
-		log.Info("s3 bucket ok")
+		log.Info(b.storage.Name() + " storage ok")
 	}
 
 	// make sure our spool dirs are writable
@@ -883,7 +892,7 @@ type backend struct {
 
 	db        *sqlx.DB
 	redisPool *redis.Pool
-	s3Client  s3iface.S3API
+	storage   storage.Storage
 	awsCreds  *credentials.Credentials
 
 	popScript *redis.Script