diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..42757e6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof +nohup.out diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6f5e289 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015-present Cloud + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..f7033d1 --- /dev/null +++ b/README.md @@ -0,0 +1,102 @@ +# Cloud Upload + +Upload files to multiple cloud storage in parallel. + +### Download + +#### Binary + +Download from (releases)[https://github.com/txthinking/cloudupload/releases] page. + +#### Source + +``` +$ go get github.com/txthinking/cloudupload/cli/cloudupload +``` + +### Run + +``` +NAME: + Cloud Upload - Upload files to multiple cloud storage in parallel + +USAGE: + main [global options] command [command options] [arguments...] + +VERSION: + 20181008 + +AUTHOR: + Cloud + +COMMANDS: + help, h Shows a list of commands or help for one command + +GLOBAL OPTIONS: + --debug, -d Enable debug, more logs + --debugListen value, -l value Listen address for debug (default: "127.0.0.1:6060") + --listen value Listen address + --domain value If domain is specified, 80 and 443 ports will be used. Listen address is no longer needed + --maxBodySize value Max size of http body, M (default: 0) + --timeout value Read timeout, write timeout x2, idle timeout x20, s (default: 0) + --origin value Allow origins for CORS, can repeat more times. like https://google.com, suggest add https://google.com/ too + --enableLocal Enable local store + --localStorage value Local directory path + --enableGoogle Enable google store, first needs $ gcloud auth application-default login + --googleBucket value Google bucket name + --enableAliyun Enable aliyun OSS + --aliyunAccessKeyID value Aliyun access key id + --aliyunAccessKeySecret value Aliyun access key secret + --aliyunEndpoint value Aliyun endpoint, like: https://oss-cn-shanghai.aliyuncs.com + --aliyunBucket value Aliyun bucket name + --enableTencent Enable Tencent + --tencentSecretId value Tencent secret id + --tencentSecretKey value Tencent secret key + --tencentHost value domain + --help, -h show help + --version, -v print the version +``` + +### Upload + +#### Request + +* Method: `POST` +* Header: + * `Accept`, one of: + * `application/json` + * `text/plain` + * `Content-Type`, one of: + * `application/octet-stream` + * `application/base64` + * `multipart/form-data...`, field name: `file` + * `X-File-Name`: + * Full file name with suffix, Only required when `Content-Type` is `application/octet-stream` or `application/base64`. +* Body, one of: + * Binary file content when `Content-Type` is `application/octet-stream`. + * Base64 encoded file content when `Content-Type` is `application/octet-stream`. + * Multipart form data when `Content-Type` is `multipart/form-data...`. + +#### Response + +* Status Code: 200 + * Content-Type, one of: + * `application/json` + * `text/plain; charset=utf-8` + * Body: + * File path + * `{ "file": "file path" }` +* Status Code: !200 + * Content-Type: + * `text/plain; charset=utf-8` + * Body: + * Error message + +### Example + +``` +$ curl -H 'Content-Type: application/octet-stream' -H 'X-File-Name: Angry.png' --data-binary @Angry.png https://yourdomain.com +vbpovzsdzbxu/Angry.png +$ curl -F 'file=@Angry.png' https://yourdomain.com +vbpovzsdzbxu/Angry.png +``` diff --git a/aliyun.go b/aliyun.go new file mode 100644 index 0000000..3ebedde --- /dev/null +++ b/aliyun.go @@ -0,0 +1,29 @@ +package cloudupload + +import ( + "io" + + "github.com/aliyun/aliyun-oss-go-sdk/oss" +) + +type Aliyun struct { + AccessKeyID string + AccessKeySecret string + Endpoint string + Bucket string +} + +func (a *Aliyun) Save(name string, r io.Reader) error { + client, err := oss.New(a.Endpoint, a.AccessKeyID, a.AccessKeySecret) + if err != nil { + return err + } + bucket, err := client.Bucket(a.Bucket) + if err != nil { + return err + } + if err := bucket.PutObject(name, r); err != nil { + return err + } + return nil +} diff --git a/cli/cloudupload/main.go b/cli/cloudupload/main.go new file mode 100644 index 0000000..2565822 --- /dev/null +++ b/cli/cloudupload/main.go @@ -0,0 +1,278 @@ +package main + +import ( + "crypto/tls" + "log" + "net/http" + "os" + "time" + + "github.com/codegangsta/cli" + "github.com/codegangsta/negroni" + "github.com/didip/tollbooth" + "github.com/didip/tollbooth/limiter" + "github.com/gorilla/mux" + "github.com/pilu/xrequestid" + "github.com/rs/cors" + "github.com/txthinking/cloudupload" + "github.com/unrolled/secure" + "golang.org/x/crypto/acme/autocert" +) + +var debug bool +var debugListen string + +func main() { + app := cli.NewApp() + app.Name = "Cloud Upload" + app.Version = "20181008" + app.Usage = "Upload files to multiple cloud storage in parallel" + app.Author = "Cloud" + app.Email = "cloud@txthinking.com" + app.Flags = []cli.Flag{ + cli.BoolFlag{ + Name: "debug, d", + Usage: "Enable debug, more logs", + Destination: &debug, + }, + cli.StringFlag{ + Name: "debugListen, l", + Usage: "Listen address for debug", + Value: "127.0.0.1:6060", + Destination: &debugListen, + }, + cli.StringFlag{ + Name: "listen", + Usage: "Listen address", + }, + cli.StringFlag{ + Name: "domain", + Usage: "If domain is specified, 80 and 443 ports will be used. Listen address is no longer needed", + }, + cli.Int64Flag{ + Name: "maxBodySize", + Usage: "Max size of http body, M", + }, + cli.Int64Flag{ + Name: "timeout", + Usage: "Read timeout, write timeout x2, idle timeout x20, s", + }, + cli.StringSliceFlag{ + Name: "origin", + Usage: "Allow origins for CORS, can repeat more times. like https://google.com, suggest add https://google.com/ too", + }, + cli.BoolFlag{ + Name: "enableLocal", + Usage: "Enable local store", + }, + cli.StringFlag{ + Name: "localStorage", + Value: "", + Usage: "Local directory path", + }, + cli.BoolFlag{ + Name: "enableGoogle", + Usage: "Enable google store, first needs $ gcloud auth application-default login", + }, + cli.StringFlag{ + Name: "googleBucket", + Value: "", + Usage: "Google bucket name", + }, + cli.BoolFlag{ + Name: "enableAliyun", + Usage: "Enable aliyun OSS", + }, + cli.StringFlag{ + Name: "aliyunAccessKeyID", + Value: "", + Usage: "Aliyun access key id", + }, + cli.StringFlag{ + Name: "aliyunAccessKeySecret", + Value: "", + Usage: "Aliyun access key secret", + }, + cli.StringFlag{ + Name: "aliyunEndpoint", + Value: "", + Usage: "Aliyun endpoint, like: https://oss-cn-shanghai.aliyuncs.com", + }, + cli.StringFlag{ + Name: "aliyunBucket", + Value: "", + Usage: "Aliyun bucket name", + }, + cli.BoolFlag{ + Name: "enableTencent", + Usage: "Enable Tencent", + }, + cli.StringFlag{ + Name: "tencentSecretId", + Value: "", + Usage: "Tencent secret id", + }, + cli.StringFlag{ + Name: "tencentSecretKey", + Value: "", + Usage: "Tencent secret key", + }, + cli.StringFlag{ + Name: "tencentHost", + Value: "", + Usage: "domain", + }, + } + app.Action = func(c *cli.Context) error { + ss := make([]cloudupload.Storer, 0, 1) + if c.Bool("enableLocal") { + local := &cloudupload.Local{ + StoragePath: c.String("localStorage"), + } + ss = append(ss, local) + } + if c.Bool("enableGoogle") { + google := &cloudupload.Google{ + Bucket: c.String("googleBucket"), + } + ss = append(ss, google) + } + if c.Bool("enableAliyun") { + aliyun := &cloudupload.Aliyun{ + AccessKeyID: c.String("aliyunAccessKeyID"), + AccessKeySecret: c.String("aliyunAccessKeySecret"), + Endpoint: c.String("aliyunEndpoint"), + Bucket: c.String("aliyunBucket"), + } + ss = append(ss, aliyun) + } + if c.Bool("enableTencent") { + tencent := &cloudupload.Tencent{ + SecretId: c.String("tencentSecretId"), + SecretKey: c.String("tencentSecretKey"), + Host: c.String("tencentHost"), + } + ss = append(ss, tencent) + } + if debug { + go func() { + log.Println(http.ListenAndServe(debugListen, nil)) + }() + } + return runHTTPServer(c.String("listen"), c.String("domain"), c.StringSlice("origin"), ss, c.Int64("maxBodySize")*1024*1024, c.Int64("timeout")) + } + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} + +func runHTTPServer(address, domain string, origins []string, stores []cloudupload.Storer, maxBodySize, timeout int64) error { + up := &cloudupload.Upload{Stores: stores} + r := mux.NewRouter() + r.Methods("POST").Path("/").Handler(up) + r.Methods("OPTIONS").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + }) + + n := negroni.New() + n.Use(negroni.NewRecovery()) + if debug { + n.Use(negroni.NewLogger()) + } + if domain != "" { + n.Use(negroni.HandlerFunc(secure.New(secure.Options{ + AllowedHosts: []string{domain}, + SSLRedirect: false, + SSLHost: domain, + SSLProxyHeaders: map[string]string{"X-Forwarded-Proto": "https"}, + STSSeconds: 315360000, + STSIncludeSubdomains: true, + STSPreload: true, + FrameDeny: true, + CustomFrameOptionsValue: "SAMEORIGIN", + ContentTypeNosniff: true, + BrowserXssFilter: true, + ContentSecurityPolicy: "default-src 'self'", + }).HandlerFuncWithNext)) + } + if len(origins) != 0 { + n.Use(cors.New(cors.Options{ + AllowedOrigins: origins, + AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, + AllowedHeaders: []string{"Accept", "Content-Type", "X-File-Name", "X-Request-With"}, + MaxAge: 60 * 60 * 24 * 30, + ExposedHeaders: []string{"Content-Type", "X-Request-Id"}, + AllowCredentials: true, + OptionsPassthrough: true, + })) + } + + n.UseFunc(func(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { + w.Header().Set("Server", "github.com/txthinking/cloudupload") + next(w, r) + }) + + n.UseFunc(func(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { + r.Body = http.MaxBytesReader(w, r.Body, maxBodySize) + next(w, r) + }) + + n.Use(xrequestid.New(16)) + + for _, store := range stores { + if local, ok := store.(*cloudupload.Local); ok { + n.Use(negroni.NewStatic(http.Dir(local.StoragePath))) + break + } + } + + lmt := tollbooth.NewLimiter(30, &limiter.ExpirableOptions{DefaultExpirationTTL: time.Hour}) + if domain == "" { + lmt.SetIPLookups([]string{"X-Forwarded-For", "X-Real-IP", "RemoteAddr"}) + } else { + lmt.SetIPLookups([]string{"RemoteAddr", "X-Forwarded-For", "X-Real-IP"}) + } + n.UseFunc(func(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { + if r.Method == "POST" { + httpError := tollbooth.LimitByRequest(lmt, w, r) + if httpError != nil { + w.Header().Add("Content-Type", lmt.GetMessageContentType()) + w.WriteHeader(httpError.StatusCode) + w.Write([]byte(httpError.Message)) + return + } + } + next(w, r) + }) + + n.UseHandler(r) + + if domain == "" { + s := &http.Server{ + Addr: address, + ReadTimeout: time.Duration(timeout) * time.Second, + WriteTimeout: time.Duration(timeout) * 2 * time.Second, + IdleTimeout: time.Duration(timeout) * 20 * time.Second, + MaxHeaderBytes: 1 << 20, + Handler: n, + } + return s.ListenAndServe() + } + m := autocert.Manager{ + Cache: autocert.DirCache(".letsencrypt"), + Prompt: autocert.AcceptTOS, + HostPolicy: autocert.HostWhitelist(domain), + Email: "cloud@txthinking.com", + } + go http.ListenAndServe(":80", m.HTTPHandler(nil)) + ss := &http.Server{ + Addr: ":443", + ReadTimeout: time.Duration(timeout) * time.Second, + WriteTimeout: time.Duration(timeout) * 2 * time.Second, + IdleTimeout: time.Duration(timeout) * 20 * time.Second, + MaxHeaderBytes: 1 << 20, + Handler: n, + TLSConfig: &tls.Config{GetCertificate: m.GetCertificate}, + } + return ss.ListenAndServeTLS("", "") +} diff --git a/google.go b/google.go new file mode 100644 index 0000000..8e6c149 --- /dev/null +++ b/google.go @@ -0,0 +1,32 @@ +package cloudupload + +import ( + "io" + + "cloud.google.com/go/storage" + "golang.org/x/net/context" +) + +type Google struct { + Bucket string +} + +func (g *Google) Save(name string, r io.Reader) error { + ctx := context.Background() + //ctx, cancel := context.WithTimeout(ctx, time.Second*60*5) + //defer cancel() + client, err := storage.NewClient(ctx) + if err != nil { + return err + } + bkt := client.Bucket(g.Bucket) + obj := bkt.Object(name) + w := obj.NewWriter(ctx) + if _, err := io.Copy(w, r); err != nil { + return err + } + if err := w.Close(); err != nil { + return err + } + return nil +} diff --git a/local.go b/local.go new file mode 100644 index 0000000..0b10f41 --- /dev/null +++ b/local.go @@ -0,0 +1,25 @@ +package cloudupload + +import ( + "io" + "os" +) + +type Local struct { + StoragePath string +} + +func (l *Local) Save(name string, r io.Reader) error { + var f *os.File + f, err := os.OpenFile(l.StoragePath+"/"+name, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644) + if err != nil { + return err + } + if _, err := io.Copy(f, r); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + return nil +} diff --git a/short.go b/short.go new file mode 100644 index 0000000..0b95727 --- /dev/null +++ b/short.go @@ -0,0 +1,37 @@ +package cloudupload + +import ( + "math" + "strings" +) + +const ( + SYMBOLS = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + BASE = int64(len(SYMBOLS)) +) + +// encodes a number into our *base* representation +func ToShort(number int64) string { + rest := number % BASE + // strings are a bit weird in go... + result := string(SYMBOLS[rest]) + if number-rest != 0 { + newnumber := (number - rest) / BASE + result = ToShort(newnumber) + result + } + return result +} + +// Decodes a string given in our encoding and returns the decimal +// integer. +func FromShort(input string) int64 { + const floatbase = float64(BASE) + l := len(input) + var sum int = 0 + for index := l - 1; index > -1; index -= 1 { + current := string(input[index]) + pos := strings.Index(SYMBOLS, current) + sum = sum + (pos * int(math.Pow(floatbase, float64((l-index-1))))) + } + return int64(sum) +} diff --git a/store.go b/store.go new file mode 100644 index 0000000..6a0d4b4 --- /dev/null +++ b/store.go @@ -0,0 +1,7 @@ +package cloudupload + +import "io" + +type Storer interface { + Save(name string, r io.Reader) error +} diff --git a/tencent.go b/tencent.go new file mode 100644 index 0000000..3d3c878 --- /dev/null +++ b/tencent.go @@ -0,0 +1,84 @@ +package cloudupload + +import ( + "bytes" + "crypto/hmac" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "strconv" + "strings" + "time" + + "github.com/txthinking/ant" +) + +type Tencent struct { + SecretId string + SecretKey string + Host string +} + +func (t *Tencent) Save(name string, r io.Reader) error { + data, err := ioutil.ReadAll(r) + if err != nil { + return err + } + c := &http.Client{ + Timeout: 10 * time.Second, + } + ss := strings.Split(name, "/") + rq, err := http.NewRequest("PUT", "https://"+t.Host+"/"+ss[0]+"/"+ant.URIEscape(ss[1]), bytes.NewReader(data)) + if err != nil { + return err + } + a, err := t.Authorization(name, len(data)) + if err != nil { + return err + } + rq.Header.Set("Authorization", a) + rq.Header.Set("Content-Length", strconv.FormatInt(int64(len(data)), 10)) + res, err := c.Do(rq) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return errors.New(res.Status) + } + return nil +} + +func (t *Tencent) Authorization(name string, length int) (string, error) { + ts0 := time.Now().Unix() + ts1 := time.Now().Unix() + 60*60 + ts := strconv.FormatInt(ts0, 10) + ";" + strconv.FormatInt(ts1, 10) + s := "" + s += "q-sign-algorithm=sha1" + s += "&q-ak=" + t.SecretId + s += "&q-sign-time=" + ts + s += "&q-key-time=" + ts + s += "&q-header-list=" + "content-length;host" + s += "&q-url-param-list=" + "" + + mac := hmac.New(sha1.New, []byte(t.SecretKey)) + if _, err := mac.Write([]byte(ts)); err != nil { + return "", err + } + signKey := hex.EncodeToString(mac.Sum(nil)) + httpString := fmt.Sprintf("put\n/%s\n%s\ncontent-length=%d&host=%s\n", name, "", length, t.Host) + stringToSign := fmt.Sprintf("sha1\n%s\n%s\n", ts, ant.SHA1(httpString)) + mac = hmac.New(sha1.New, []byte(signKey)) + if _, err := mac.Write([]byte(stringToSign)); err != nil { + return "", err + } + signature := hex.EncodeToString(mac.Sum(nil)) + + s += "&q-signature=" + signature + + return s, nil +} diff --git a/upload.go b/upload.go new file mode 100644 index 0000000..ef7f67b --- /dev/null +++ b/upload.go @@ -0,0 +1,132 @@ +package cloudupload + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "io" + "io/ioutil" + "net/http" + "strconv" + "strings" + + "github.com/juju/ratelimit" + uuid "github.com/satori/go.uuid" + "github.com/txthinking/ant" +) + +type Upload struct { + URL string + Stores []Storer + Rate int64 +} + +func (u *Upload) ServeHTTP(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + uid, err := uuid.NewV4() + if err != nil { + http.Error(w, err.Error(), 500) + return + } + id := strings.Replace(uid.String(), "-", "", -1) + i := binary.BigEndian.Uint64([]byte(id)) + id = strconv.FormatUint(i, 36) + + var name string + var b []byte + if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") { + f, fh, err := r.FormFile("file") + if err != nil { + http.Error(w, err.Error(), 400) + return + } + defer f.Close() + b, err = ioutil.ReadAll(f) + if err != nil { + http.Error(w, err.Error(), 400) + return + } + name = fh.Filename + } + if !strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") { + var src io.Reader = r.Body + if u.Rate != 0 { + bucket := ratelimit.NewBucketWithRate(float64(u.Rate), u.Rate) + src = ratelimit.Reader(r.Body, bucket) + } + b, err = ioutil.ReadAll(src) + if err != nil { + http.Error(w, err.Error(), 400) + return + } + if r.Header.Get("Content-Type") == "application/base64" { + b, err = base64.StdEncoding.DecodeString(string(b)) + if err != nil { + http.Error(w, err.Error(), 400) + return + } + } + } + if name == "" { + name = Name(r) + } + + e := make(chan error) + for _, store := range u.Stores { + go func(store Storer) { + e <- store.Save(id+"/"+name, bytes.NewReader(b)) + }(store) + } + done := make(chan error) + var times int + go func() { + var isDone bool + for { + err := <-e + if err != nil { + if !isDone { + done <- err + isDone = true + } + } + times++ + if times == len(u.Stores) { + close(e) + break + } + } + if !isDone { + done <- nil + } + }() + + err = <-done + if err != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(err.Error())) + return + } + if r.Header.Get("Accept") == "application/json" { + ant.JSON(w, map[string]string{ + "file": u.URL + id + "/" + ant.URIEscape(name), + }) + return + } + w.Write([]byte(u.URL + id + "/" + ant.URIEscape(name))) +} + +func Name(r *http.Request) string { + name := r.Header.Get("X-File-Name") + if name != "" { + s, err := ant.URIUnescape(name) + if err != nil { + name = "" + } else { + name = s + } + } + if name == "" { + name = "NoName" + } + return name +} diff --git a/uuid_test.go b/uuid_test.go new file mode 100644 index 0000000..41ad517 --- /dev/null +++ b/uuid_test.go @@ -0,0 +1,22 @@ +package cloudupload + +import ( + "encoding/binary" + "log" + "strconv" + "strings" + "testing" + + uuid "github.com/satori/go.uuid" +) + +func TestUUID(t *testing.T) { + id := strings.Replace(uuid.NewV4().String(), "-", "", -1) + log.Println(id) + i := binary.BigEndian.Uint64([]byte(id)) + log.Println(i) + s := strconv.FormatUint(i, 36) + log.Println(s) + s = ToShort(int64(i)) + log.Println(s) +}