Merge branch 'master' into swarmy

.travis.yml

@@ -9,6 +9,7 @@ env:
     - SSL_MODE_DISABLE=true
   global:
     - GO111MODULE=on
+    - RECAPTCHA_KEY=cantbeblankortestsfail
     # decodes into SENGRID_API_KEY environment variable
     - secure: GXKJ+1Wa+pqUm1PKHJMbDrejSfIsqhsZ6l88i0paTvVou9T/mv03mx9hzrSLsYXFFVjinfbKysxMtEorTFDl0YBSNxuPc7eaiJinnoptubBh++bvEzF/A8wmWHgIBxroFIrK4SNsnreCXPqT1YBQZWi3JenMkzS68j1oa/uPa5ODZ3rjx1Wmu6hSANSZPqzOoGO6lnRP8G30oiFzbawEnB/52iBcHhPtffUihLFnn3k1wXyL8fpi4JYuyNQe84Br+w91KSS2nMmG+RVvJWumHNGXFyjEVv0n7HbirjCBI1iZo50bTdFtujkGSOCoHqM1hwa2yLWISipkb19Ls4eeWcWZBFpcdBhIIeHEmz9iCXxn0ksIGzuw00Xof/HTcWUpnzzQFq4E8iVE4tmMrFbYEcD5vcGB3S7bURjTD38uZ3/7I4Oyuo33/WQGdDNIzugU/dycGkSNzwTeIB3yJfpr/wbC6lU5RnADh5Ej5hGreAfkng/iuC9T36Fnn9u4fTocj++smOxZSjbbKsyekNNdadYmU0sfd6Ka9mFcv8H6sDs7zwqyZLTDu6P57CN5VFz4Yjc1emupFX5NFWor5jull9cr+ilOmfnBSCQPV0ArKrpKjrL9K9NJAhRKOqxmpLKx3pozGnrQMFKM0xsSuTcOCdyTdZF8JEQESeENHzwyoB4=
 install:

api/middleware/jwt.go

@@ -48,16 +48,21 @@ func JwtConfigGenerate(jwtKey, realmName string, db *gorm.DB, l *zap.SugaredLogg
 					return "", false
 				}
 			}
+			// email enabled implies they have verified their email
+			if !usr.EmailEnabled {
+				return "", false
+			}
 			lAuth.Info("successful login", "username", usr.UserName)
 			return usr.UserName, true
 		},
 		Authorizator: func(userId string, c *gin.Context) bool {
 			// as a final security step, ensure that we can find the user in our database
 			userManager := models.NewUserManager(db)
-			if _, err := userManager.FindByUserName(userId); err != nil {
+			usr, err := userManager.FindByUserName(userId)
+			if err != nil {
 				return false
 			}
-			return true
+			return usr.EmailEnabled && usr.AccountEnabled
 		},
 		Unauthorized: func(c *gin.Context, code int, message string) {
 			l.Error("invalid login detected")

api/v2/api.go

@@ -9,22 +9,21 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/streadway/amqp"
-
+	"github.com/RTradeLtd/ChainRider-Go/dash"
+	"github.com/RTradeLtd/Temporal/queue"
 	"github.com/RTradeLtd/Temporal/rtfscluster"
 	pbLens "github.com/RTradeLtd/grpc/lensv2"
 	pbOrch "github.com/RTradeLtd/grpc/nexus"
 	pbSigner "github.com/RTradeLtd/grpc/pay"
-	pbBchWallet "github.com/gcash/bchwallet/rpc/walletrpc"
-
 	"github.com/RTradeLtd/kaas/v2"
-	"go.uber.org/zap"
-
-	"github.com/RTradeLtd/ChainRider-Go/dash"
-	"github.com/RTradeLtd/Temporal/queue"
 	"github.com/RTradeLtd/rtfs/v2"
-
-	limit "github.com/aviddiviner/gin-limit"
+	recaptcha "github.com/ezzarghili/recaptcha-go"
+	pbBchWallet "github.com/gcash/bchwallet/rpc/walletrpc"
+	"github.com/streadway/amqp"
+	"github.com/ulule/limiter/v3"
+	mgin "github.com/ulule/limiter/v3/drivers/middleware/gin"
+	"github.com/ulule/limiter/v3/drivers/store/memory"
+	"go.uber.org/zap"
 
 	"github.com/RTradeLtd/config/v2"
 	stats "github.com/semihalev/gin-stats"
@@ -62,8 +61,10 @@ type API struct {
 	dc          *dash.Client
 	queues      queues
 	service     string
-	cmcAPIKey   string
 	version     string
+
+	captcha        recaptcha.ReCAPTCHA
+	captchaEnabled bool
 }
 
 // Initialize is used ot initialize our API service. debug = true is useful
@@ -83,6 +84,8 @@ func Initialize(
 		err    error
 		router = gin.Default()
 	)
+	// if we dont set this, rate limiting wont work properly
+	router.ForwardedByClientIP = true
 	// update dev mode
 	dev = opts.DevMode
 	l = l.Named("api")
@@ -108,7 +111,14 @@ func Initialize(
 		return nil, err
 	}
 	api.version = version
-
+	if api.getCaptchaKey() != "" {
+		captcha, err := recaptcha.NewReCAPTCHA(api.getCaptchaKey(), recaptcha.V3, time.Second*20)
+		if err != nil {
+			return nil, err
+		}
+		api.captcha = captcha
+		api.captchaEnabled = true
+	}
 	// init routes
 	if err = api.setupRoutes(opts.DebugLogging); err != nil {
 		return nil, err
@@ -124,7 +134,6 @@ func new(cfg *config.TemporalConfig, router *gin.Engine, l *zap.SugaredLogger, c
 		dbm *database.Manager
 		err error
 	)
-
 	// set up database manager
 	dbm, err = database.New(cfg, database.Options{LogMode: debug})
 	if err != nil {
@@ -379,6 +388,18 @@ func (api *API) setupRoutes(debug bool) error {
 			return err
 		}
 	}
+	// make sure we dont throttle dev too much
+	var rateLimit string
+	if dev {
+		rateLimit = "100000-H"
+	} else {
+		rateLimit = fmt.Sprintf("%v-H", connLimit)
+	}
+	rate, err := limiter.NewRateFromFormatted(rateLimit)
+	if err != nil {
+		return err
+	}
+
 	// ensure we have valid cors configuration, otherwise default to allow all
 	var allowedOrigins []string
 	if len(api.cfg.API.Connection.CORS.AllowedOrigins) > 0 {
@@ -392,7 +413,7 @@ func (api *API) setupRoutes(debug bool) error {
 		// greater than what can be configured with HTTP Headers
 		xssMdlwr.RemoveXss(),
 		// rate limiting
-		limit.MaxAllowed(connLimit),
+		mgin.NewMiddleware(limiter.New(memory.NewStore(), rate)),
 		// security middleware
 		middleware.NewSecWare(dev),
 		// request id middleware
@@ -650,6 +671,13 @@ func (api *API) setupRoutes(debug bool) error {
 		ens.POST("/claim", api.ClaimENSName)
 		ens.POST("/update", api.UpdateContentHash)
 	}
+	if api.captchaEnabled {
+		recap := v2.Group("/captcha")
+		{
+			recap.POST("/verify", api.verifyCaptcha)
+		}
+
+	}
 
 	// swarm routes
 	swarm := v2.Group("/swarm")

api/v2/api_test.go

@@ -256,7 +256,8 @@ func Test_API_Setup(t *testing.T) {
 		args     args
 		wantCode int
 	}{
-		{"Login-testuser2", args{"POST", "/v2/auth/login", "testuser2", "password123!@#$%^&&**(!@#!", ""}, 200},
+		// this test should fail due to invalid email
+		{"Login-testuser2", args{"POST", "/v2/auth/login", "testuser2", "password123!@#$%^&&**(!@#!", ""}, 401},
 		{"Login-testuser", args{"POST", "/v2/auth/login", "testuser", "admin", ""}, 200},
 		// tests login via the email instead of using
 		{"Login-TestUser-Email", args{"POST", "/v2/auth/login", "[email protected]", "admin", ""}, 200},
@@ -272,7 +273,7 @@ func Test_API_Setup(t *testing.T) {
 			)
 			api.r.ServeHTTP(testRecorder, req)
 			if testRecorder.Code != tt.wantCode {
-				t.Fatalf("bad http status code from %s", tt.args.call)
+				t.Fatalf("bad http status code from %s. got %v, want %v", tt.args.call, testRecorder.Code, tt.wantCode)
 			}
 			bodBytes, err := ioutil.ReadAll(testRecorder.Result().Body)
 			if err != nil {

api/v2/routes_account.go

@@ -328,6 +328,9 @@ func (api *API) upgradeAccount(c *gin.Context) {
 		api.LogError(c, err, eh.UserSearchError)(http.StatusBadRequest)
 		return
 	}
+	if usages.Tier == models.Unverified {
+		Fail(c, errors.New("unverified account upgrade process must be done via email verification"))
+	}
 	// prevent people from repeatedly calling this granting perpetual credits
 	if usages.Tier != models.Free {
 		Fail(c, errors.New("user account is already upgrade"))

api/v2/routes_ens.go

@@ -24,7 +24,7 @@ func (api *API) ClaimENSName(c *gin.Context) {
 		return
 	}
 	// prevent processing if account is free tier
-	if usage.Tier == models.Free {
+	if usage.Tier == models.Free || usage.Tier == models.Unverified {
 		Fail(c, errors.New("free accounts not eligible for ens claim"), http.StatusBadRequest)
 		return
 	}
@@ -74,7 +74,7 @@ func (api *API) UpdateContentHash(c *gin.Context) {
 		return
 	}
 	// prevent processing if account is free tier
-	if usage.Tier == models.Free {
+	if usage.Tier == models.Free || usage.Tier == models.Unverified {
 		Fail(c, errors.New("free accounts not eligible for ens claim"), http.StatusBadRequest)
 		return
 	}

api/v2/routes_frontend.go

@@ -35,7 +35,7 @@ func (api *API) calculatePinCost(c *gin.Context) {
 		return
 	}
 	// calculate pin cost
-	totalCost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+	totalCost, _, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
 	if err != nil {
 		api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
 		Fail(c, err)

api/v2/routes_ipns.go

@@ -167,14 +167,8 @@ func (api *API) pinIPNSHash(c *gin.Context) {
 		Respond(c, http.StatusBadRequest, gin.H{"response": alreadyUploadedMessage})
 		return
 	}
-	// get size of object
-	stats, err := api.ipfs.Stat(hash)
-	if err != nil {
-		api.LogError(c, err, eh.IPFSObjectStatError)(http.StatusBadRequest)
-		return
-	}
 	// get the cost of this object
-	cost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+	cost, size, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
 	if err != nil {
 		api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
 		return
@@ -184,7 +178,7 @@ func (api *API) pinIPNSHash(c *gin.Context) {
 		api.LogError(c, err, eh.InvalidBalanceError)(http.StatusPaymentRequired)
 		return
 	}
-	if err := api.usage.UpdateDataUsage(username, uint64(stats.CumulativeSize)); err != nil {
+	if err := api.usage.UpdateDataUsage(username, uint64(size)); err != nil {
 		api.LogError(c, err, eh.CantUploadError)(http.StatusBadRequest)
 		api.refundUserCredits(username, "pin", cost)
 		return
@@ -196,13 +190,13 @@ func (api *API) pinIPNSHash(c *gin.Context) {
 		UserName:         username,
 		HoldTimeInMonths: holdTimeInt,
 		CreditCost:       cost,
-		Size:             int64(stats.CumulativeSize),
+		Size:             int64(size),
 	}
 	// send message for processing
 	if err = api.queues.cluster.PublishMessage(qp); err != nil {
 		api.LogError(c, err, eh.QueuePublishError)(http.StatusBadRequest)
 		api.refundUserCredits(username, "pin", cost)
-		api.usage.ReduceDataUsage(username, uint64(stats.CumulativeSize))
+		api.usage.ReduceDataUsage(username, uint64(size))
 		return
 	}
 	// log and return

api/v2/routes_ipns_test.go

@@ -188,6 +188,12 @@ func Test_API_Routes_IPNS_Pin(t *testing.T) {
 			fakeManager.ResolveReturnsOnCall(0, validResolveResult, nil)
 			fakeManager.StatReturnsOnCall(0, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
 			fakeManager.StatReturnsOnCall(1, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
+			fakeManager.StatReturnsOnCall(2, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
+			fakeManager.StatReturnsOnCall(3, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
+			fakeManager.RefsReturnsOnCall(0, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
+			fakeManager.RefsReturnsOnCall(1, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
+			fakeManager.RefsReturnsOnCall(2, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
+			fakeManager.RefsReturnsOnCall(3, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
 			var apiResp apiResponse
 			urlValues := url.Values{}
 			urlValues.Add("hold_time", tt.args.holdTime)

api/v2/routes_rtfs.go

@@ -54,14 +54,8 @@ func (api *API) pinHashLocally(c *gin.Context) {
 		Respond(c, http.StatusBadRequest, gin.H{"response": alreadyUploadedMessage})
 		return
 	}
-	// get object size
-	stats, err := api.ipfs.Stat(hash)
-	if err != nil {
-		api.LogError(c, err, eh.IPFSObjectStatError)(http.StatusBadRequest)
-		return
-	}
 	// determine cost of upload
-	cost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+	cost, size, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
 	if err != nil {
 		api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
 		return
@@ -72,7 +66,7 @@ func (api *API) pinHashLocally(c *gin.Context) {
 		return
 	}
 	// update their data usage
-	if err := api.usage.UpdateDataUsage(username, uint64(stats.CumulativeSize)); err != nil {
+	if err := api.usage.UpdateDataUsage(username, uint64(size)); err != nil {
 		api.LogError(c, err, eh.CantUploadError)(http.StatusBadRequest)
 		api.refundUserCredits(username, "pin", cost)
 		return
@@ -83,15 +77,15 @@ func (api *API) pinHashLocally(c *gin.Context) {
 		NetworkName:      "public",
 		UserName:         username,
 		HoldTimeInMonths: holdTimeInt,
-		Size:             int64(stats.CumulativeSize),
+		Size:             size,
 		CreditCost:       cost,
 		FileName:         c.PostForm("file_name"),
 	}
 	// sent pin message
 	if err = api.queues.cluster.PublishMessage(qp); err != nil {
 		api.LogError(c, err, eh.QueuePublishError)(http.StatusBadRequest)
 		api.refundUserCredits(username, "pin", cost)
-		api.usage.ReduceDataUsage(username, uint64(stats.CumulativeSize))
+		api.usage.ReduceDataUsage(username, uint64(size))
 		return
 	}
 	// log success and return
@@ -197,7 +191,7 @@ func (api *API) addFile(c *gin.Context) {
 		// if the user is within the free tier, then we throttle on-demand encryption
 		// free accounts are limited to a file upload size of 275MB when performing
 		// on-demand encryption. Non free accounts do not have this limit
-		if userUsage.Tier == models.Free {
+		if userUsage.Tier == models.Free || userUsage.Tier == models.Unverified {
 			megabytesUint := datasize.MB.Bytes()
 			maxSize := megabytesUint * 275
 			if fileHandler.Size > int64(maxSize) {
@@ -381,7 +375,7 @@ func (api *API) extendPin(c *gin.Context) {
 		return
 	}
 	// calculate cost of hold time extension
-	cost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+	cost, _, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
 	if err != nil {
 		api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
 		return

api/v2/routes_utils.go

@@ -17,6 +17,7 @@ import (
 	pb "github.com/RTradeLtd/grpc/krab"
 	"github.com/RTradeLtd/rtfs/v2"
 	"github.com/RTradeLtd/rtfs/v2/beam"
+	"github.com/ezzarghili/recaptcha-go"
 	"github.com/gin-gonic/gin"
 	gocid "github.com/ipfs/go-cid"
 )
@@ -343,7 +344,7 @@ func (api *API) handleUserCreate(c *gin.Context, forms map[string]string, create
 			"<br>",
 			"<br>",
 			"Lastly let's talk about emails! We try our best to not spam your inbox, so we limit emails to a few things: payment notifications, pin expiration warnings, password/username retrieval and processing failures.\n",
-			"But before we do this, you must validate your email. Note that email validation is not required unless you want these notifications\n",
+			"But before we do this, you must validate your email. Additionally before validating your email, you are in the 'unverified' tier which is limited to 100MB of data consumption. Email verification is now mandatory\n",
 			"To validate your email, just click the following "+link+"\n",
 			"<br>",
 			"<br>",
@@ -388,3 +389,15 @@ func (api *API) handleUserCreate(c *gin.Context, forms map[string]string, create
 	},
 	})
 }
+
+func (api *API) verifyCaptcha(c *gin.Context) {
+	err := api.captcha.VerifyWithOptions(
+		c.PostForm("g-recaptcha-response"),
+		// require a threshold of 0.8, default is 0.5
+		recaptcha.VerifyOption{Threshold: 0.8},
+	)
+	if err != nil {
+		Fail(c, errors.New("captcha validation failed"))
+	}
+	Respond(c, http.StatusOK, gin.H{"response": "captcha validation succeeded"})
+}