diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9409776 --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +# Created by .ignore support plugin (hsz.mobi) +### Go template +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + +.gitignore diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 7d3b5fe..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "qless-core"] - path = qless-core - url = https://github.com/seomoz/qless-core diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9be7adf --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +PROJ=$(shell realpath $$PWD/../../../..) +ENV=env GOPATH=$(PROJ) +CLI=$(ENV) easyjson -all + +JSON_SRC_FILES=\ + structs.go + +JSON_OUT_FILES:=$(JSON_SRC_FILES:%.go=%_easyjson.go) + +json: $(JSON_OUT_FILES) + +%_easyjson.go: %.go + $(CLI) $^ + diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/client.go b/client.go old mode 100644 new mode 100755 index f1094c3..9bece1c --- a/client.go +++ b/client.go @@ -1,4 +1,4 @@ -package goqless +package qless import ( "encoding/json" @@ -19,38 +19,21 @@ type Client struct { port string events *Events - lua *Lua + lua *redis.Script } -func NewClient(host, port string) *Client { - return &Client{host: host, port: port} -} - -func Dial(host, port string) (*Client, error) { - c := NewClient(host, port) - - conn, err := redis.Dial("tcp", fmt.Sprintf("%s:%s", host, port)) - if err != nil { - return nil, err - } - - c.lua = NewLua(conn) - dir, err := GetCurrentDir() - if err != nil { - println(err.Error()) - conn.Close() - return nil, err - } - //println(dir + "/qless-core") - err = c.lua.LoadScripts(dir + "/qless-core") // make get from lib path +func Dial(host, port string, db int) (*Client, error) { + conn, err := redis.Dial("tcp", fmt.Sprintf("%s:%s", host, port), redis.DialDatabase(db)) if err != nil { - println(err.Error()) - conn.Close() return nil, err } - c.conn = conn - return c, nil + return &Client{ + host: host, + port: port, + lua: redis.NewScript(0, qlessLua), + conn: conn, + }, nil } func (c *Client) Close() { @@ -65,8 +48,8 @@ func (c *Client) Events() *Events { return c.events } -func (c *Client) Do(name string, keysAndArgs ...interface{}) (interface{}, error) { - return c.lua.Do(name, keysAndArgs...) +func (c *Client) Do(args ...interface{}) (interface{}, error) { + return c.lua.Do(c.conn, args...) } func (c *Client) Queue(name string) *Queue { @@ -76,12 +59,12 @@ func (c *Client) Queue(name string) *Queue { } func (c *Client) Queues(name string) ([]*Queue, error) { - args := []interface{}{0, "queues", timestamp()} + args := []interface{}{"queues", timestamp()} if name != "" { args = append(args, name) } - byts, err := redis.Bytes(c.Do("qless", args...)) + byts, err := redis.Bytes(c.Do(args...)) if err != nil { return nil, err } @@ -105,17 +88,17 @@ func (c *Client) Queues(name string) ([]*Queue, error) { // Track the jid func (c *Client) Track(jid string) (bool, error) { - return Bool(c.Do("qless", 0, "track", timestamp(), "track", jid, "")) + return Bool(c.Do("track", timestamp(), "track", jid, "")) } // Untrack the jid func (c *Client) Untrack(jid string) (bool, error) { - return Bool(c.Do("qless", 0, "track", timestamp(), 0, "untrack", jid)) + return Bool(c.Do("track", timestamp(), 0, "untrack", jid)) } // Returns all the tracked jobs func (c *Client) Tracked() (string, error) { - return redis.String(c.Do("qless", 0, "track", timestamp())) + return redis.String(c.Do("track", timestamp())) } func (c *Client) Get(jid string) (interface{}, error) { @@ -127,22 +110,22 @@ func (c *Client) Get(jid string) (interface{}, error) { return job, err } -func (c *Client) GetJob(jid string) (*Job, error) { - byts, err := redis.Bytes(c.Do("qless", 0, "get", timestamp(), jid)) +func (c *Client) GetJob(jid string) (Job, error) { + byts, err := redis.Bytes(c.Do("get", timestamp(), jid)) if err != nil { return nil, err } - job := NewJob(c) - err = json.Unmarshal(byts, job) + var d jobData + err = json.Unmarshal(byts, d) if err != nil { return nil, err } - return job, err + return &job{&d, c}, err } func (c *Client) GetRecurringJob(jid string) (*RecurringJob, error) { - byts, err := redis.Bytes(c.Do("qless", 0, "recur", timestamp(), "get", jid)) + byts, err := redis.Bytes(c.Do("recur", timestamp(), "get", jid)) if err != nil { return nil, err } @@ -156,7 +139,7 @@ func (c *Client) GetRecurringJob(jid string) (*RecurringJob, error) { } func (c *Client) Completed(start, count int) ([]string, error) { - reply, err := redis.Values(c.Do("qless", 0, "jobs", timestamp(), "complete")) + reply, err := redis.Values(c.Do("jobs", timestamp(), "complete")) if err != nil { return nil, err } @@ -170,7 +153,7 @@ func (c *Client) Completed(start, count int) ([]string, error) { } func (c *Client) Tagged(tag string, start, count int) (*TaggedReply, error) { - byts, err := redis.Bytes(c.Do("qless", 0, "tag", timestamp(), "get", tag, start, count)) + byts, err := redis.Bytes(c.Do("tag", timestamp(), "get", tag, start, count)) if err != nil { return nil, err } @@ -181,7 +164,7 @@ func (c *Client) Tagged(tag string, start, count int) (*TaggedReply, error) { } func (c *Client) GetConfig(option string) (string, error) { - interf, err := c.Do("qless", 0, "config.get", timestamp(), option) + interf, err := c.Do("config.get", timestamp(), option) if err != nil { return "", err } @@ -207,12 +190,12 @@ func (c *Client) GetConfig(option string) (string, error) { } func (c *Client) SetConfig(option string, value interface{}) { - intf, err := c.Do("qless", 0, "config.set", timestamp(), option, value) + intf, err := c.Do("config.set", timestamp(), option, value) if err != nil { fmt.Println("setconfig, c.Do fail. interface:", intf, " err:", err) } } func (c *Client) UnsetConfig(option string) { - c.Do("qless", 0, "config.unset", timestamp(), option) + c.Do("config.unset", timestamp(), option) } diff --git a/client_test.go b/client_test.go new file mode 100644 index 0000000..dec2d81 --- /dev/null +++ b/client_test.go @@ -0,0 +1,16 @@ +package qless + +import ( + "testing" + "github.com/stretchr/testify/assert" + "fmt" +) + +func TestDial(t *testing.T) { + c := newClient() + c.SetConfig("test", "hello world") + v, _ := c.GetConfig("test") + fmt.Println(v) + assert.NotNil(t, c) + c.conn.Do("FLUSHDB") +} diff --git a/events.go b/events.go old mode 100644 new mode 100755 index dbe11e4..d08b6a2 --- a/events.go +++ b/events.go @@ -1,4 +1,4 @@ -package goqless +package qless import ( "fmt" diff --git a/example/example_goqless.go b/example/example_goqless.go deleted file mode 100644 index d539c89..0000000 --- a/example/example_goqless.go +++ /dev/null @@ -1,189 +0,0 @@ -package main - -import ( - "github.com/zimulala/goqless" - "log" - "strconv" - "time" -) - -const ( - jobs_count = 20 - queues_capacity = 5 - priority = 10 - //If you don't want a job to be run right away but some time in the future, you can specify a delay - delay = -1 - retries = 3 - interval = 10 - test_move = 1 - test_fail = 2 - test_retry = 3 - original_queue = "original_queue" -) - -var clientCh = make(chan *goqless.Client) - -type dataWorker struct { -} - -func main() { - log.SetFlags(log.Lshortfile | log.Ltime) - - client, err := goqless.Dial("127.0.0.1", "6379") - if err != nil { - log.Println("Dial err:", err) - } - defer client.Close() - - originalQueue := client.Queue(original_queue) - err = initOriginalQueue(originalQueue) - if err != nil { - log.Println("initQueue failed, err:", err) - } - batchJobs(originalQueue) - - select {} -} - -func initOriginalQueue(queue *goqless.Queue) (err error) { - - var klass string - depends := []string{} - - for i := 0; i < jobs_count; i++ { - if i%2 == 0 { - klass = "klass.Add" - } else { - klass = "klass.Sub" - } - tags := []string{"e", "__call", "_haha", strconv.Itoa(i)} - queue.Put("", klass, `"dataJson"`, delay, i, tags, retries, depends) - } - - return -} - -func batchJobs(queue *goqless.Queue) (err error) { - - queueCount := int(jobs_count / queues_capacity) - // go goqless.ChannelDialClient("127.0.0.1", "6379", clientCh) - - for i := 0; i < queueCount; i++ { - handlePerQueue(queue, i) - } - - return -} - -func handlePerQueue(queue *goqless.Queue, num int) { - client, err := goqless.Dial("127.0.0.1", "6379") - if err != nil { - log.Println("Dial err:", err) - } - defer client.Close() - - queueStr := "queue_name_" + strconv.Itoa(num) - readyQueue := client.Queue(queueStr) - for i := 0; i < queues_capacity; i++ { - jobs, err := queue.Pop(1) - if err != nil { - log.Println("pop failed, count:", len(jobs), " err:", err) - return - } else { - // log.Printf("jobs :%+v", jobs[0].History) - } - jobPut, err := readyQueue.Put(queueStr+"lll_"+strconv.Itoa(i), jobs[0].Klass, "***dataJsondddd", delay, - jobs[0].Priority, jobs[0].Tags, jobs[0].Retries, jobs[0].Dependents) - if err != nil { - log.Println("put failed, err:", err) - continue - } - //testJobMoveFailRetry(i, jobs[0]) - jobs[0].Complete() - //log.Printf("---jobPut:%+v, queue:%+v", jobs[0], jobs[0].Queue) - log.Println("==jobPut : ", jobPut, " jobsPut.Priority:", jobs[0].Priority, "\n") - } - - startWorkers(queueStr) -} - -func testJobMoveFailRetry(QueueNo int, job *goqless.Job) { - - //Move to self.queue - if test_move == QueueNo { - //return jid - moveStr, err := job.Move("queue_name_1") - if err != nil { - log.Println("Move failed. err:", err) - } - log.Printf("moveStr:%+v", moveStr) - } - //Fail - // if test_fail == QueueNo { - // failRet, err := job.Fail("typ", "message") - // if err != nil { - // log.Println("Fail failed. err:", err) - // } - // log.Println("failRet:", failRet) - // } - //Retry - // if test_retry == QueueNo { - // delay := 0 - // retryStr, err := job.Retry(delay) - // if err != nil { - // log.Println("Retry failed. err:", err) - // } - // log.Println("retryStr:", retryStr, " jid:", job.Jid) - // log.Printf("===jobPut:%+v", job) - // } -} - -func startWorkers(queueStr string) (err error) { - log.Println("worker queues", queueStr) - - startWorker := func(queueStr string) { - client, err := goqless.Dial("127.0.0.1", "6379") - if err != nil { - log.Println("Dial err:", err) - } - defer client.Close() - - worker_x, err := initWorker(client, queueStr) - if err != nil { - log.Println("initWorker failed, err:", err) - } - log.Printf("qStr:%v, cli:%+v", queueStr, client) - // client.SetConfig("heartbeat", 120) - worker_x.Start() - } - - go startWorker(queueStr) - go startWorker(queueStr) - - time.Sleep(10 * time.Second) - - return -} - -func initWorker(cli *goqless.Client, queue string) (worker *goqless.Worker, err error) { - - worker = goqless.NewWorker(cli, queue, interval) - log.Printf("worker: %+v, p:%p", worker, worker) - dataW := &dataWorker{} - worker.AddService("klass", dataW) - - return -} - -func (dataW *dataWorker) Add(job *goqless.Job) (err error) { - job.Data = map[string]interface{}{"id": "Add"} - time.Sleep(190 * time.Second) - job.Data = map[string]interface{}{"id": "sleepEndAdd"} - - return -} - -func (dataW *dataWorker) Sub(job *goqless.Job) (err error) { - job.Data = map[string]interface{}{"id": "Sub"} - return -} diff --git a/examples_test.go b/examples_test.go deleted file mode 100644 index 0e80de8..0000000 --- a/examples_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package goqless - -// import ( -// "fmt" -// // "github.com/garyburd/redigo/redis" -// // "time" -// ) - -// func ExampleGoqless_1() { -// c, err := Dial("", "6379") -// if err != nil { -// panic(err) -// } -// defer c.Close() - -// jid := "f5b66400bf027c191ddb80a85785b03eb9765456" -// c.Track(jid) -// q := c.Queue("goqless_testing_queue") - -// putreply, err := q.Put(jid, "dunno", `{"hey": "there"}`, -1, -1, []string{}, -1, []string{}) -// fmt.Println("Put:", putreply, err) -// //fmt.Println(q.Recur(jid, "dunno", `{"hey": "there"}`, 5, 0, 0, []string{}, 1)) - -// jobs, err := q.Pop(1) -// fmt.Printf("Pop: %s %s %v\n", jobs[0].Queue, jobs[0].Data, err) -// fmt.Print("Complete: ") -// fmt.Println(jobs[0].Complete()) - -// //wg.Wait() - -// // Output: -// // Put: f5b66400bf027c191ddb80a85785b03eb9765456 -// // Pop: goqless_testing_queue map[hey:there] -// // Complete: complete -// } - -// func ExampleGoqless_2() { -// c, err := Dial("", "6379") -// if err != nil { -// panic(err) -// } -// defer c.Close() - -// jid := "f5b66400bf027c191ddb80a85785b03eb9765457" -// c.Track(jid) -// q := c.Queue("goqless_testing_queue") - -// data := struct { -// Str string -// }{ -// "a string", -// } - -// putreply, err := q.Put(jid, "dunno", data, -1, -1, []string{}, -1, []string{}) -// fmt.Println("Put:", putreply, err) -// //fmt.Println(q.Recur(jid, "dunno", `{"hey": "there"}`, 5, 0, 0, []string{}, 1)) - -// jobs, err := q.Pop(1) -// fmt.Printf("Pop: %s %v %v\n", jobs[0].Queue, jobs[0].Data, err) -// fmt.Print("Fail: ") -// fmt.Println(jobs[0].Fail("justbecause", "i said so")) - -// // Output: -// // Put: f5b66400bf027c191ddb80a85785b03eb9765457 -// // Pop: goqless_testing_queue map[Str:a string] -// // Fail: true -// } diff --git a/goqless.go b/goqless.go index 79fdd02..66dfa08 100644 --- a/goqless.go +++ b/goqless.go @@ -1,5 +1,5 @@ // reference: https://github.com/seomoz/qless-py -package goqless +package qless import ( "bytes" @@ -10,9 +10,7 @@ import ( "github.com/garyburd/redigo/redis" mrand "math/rand" "os" - "bitbucket.org/kardianos/osext" "strconv" - "strings" "time" "unicode" "unicode/utf8" @@ -45,18 +43,6 @@ func init() { workerNameStr = fmt.Sprintf("%s-%d", hn, os.Getpid()) } -func GetCurrentDir() (string, error) { - dir, err := osext.Executable() - if err != nil { - return "", err - } - - dir = string(dir[:len(dir)-1]) - pos := strings.LastIndex(dir, "/") - dir = string(dir[:pos]) - return dir, nil -} - func (s *StringSlice) UnmarshalJSON(data []byte) error { // because tables and arrays are equal in LUA, // an empty array would be presented as "{}". @@ -116,13 +102,6 @@ func ucfirst(s string) string { // marshals a value. if the value happens to be // a string or []byte, just return it. func marshal(i interface{}) []byte { - //switch v := i.(type) { - //case []byte: - // return v - //case string: - // return []byte(v) - //} - byts, err := json.Marshal(i) if err != nil { return nil diff --git a/goqless_test.go b/goqless_test.go deleted file mode 100644 index 7fa1fb7..0000000 --- a/goqless_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package goqless - -import ( - "fmt" - "github.com/garyburd/redigo/redis" - "sync" - "testing" - "time" -) - -// func TestRandom(t *testing.T) { -// c, err := Dial("", "6379") -// if err != nil { -// panic(err) -// } -// defer c.Close() - -// // fmt.Println(c.Completed(0, 9999)) - -// tagged, err := c.Tagged("__callback", 0, 0) -// if err != nil { -// fmt.Println(err) -// } - -// if err == nil { -// job := NewJob(c) -// for n, j := range tagged.Jobs { -// fmt.Println(n, j) -// job.Jid = j -// fmt.Println(job.Untag("__callback")) -// } -// } -// } - -func TestGoqless(t *testing.T) { - var wg sync.WaitGroup - - c, err := Dial("", "6379") - if err != nil { - panic(err) - } - defer c.Close() - - e := c.Events() - ch, err := e.Listen() - if err != nil { - panic(err) - } - - go func() { - wg.Add(1) - defer wg.Done() - - for { - if val, ok := <-ch; ok { - switch v := (val).(type) { - case redis.Message: - fmt.Printf("WATCH: %s: message: %s\n", v.Channel, v.Data) - case redis.Subscription: - fmt.Printf("WATCH: %s: %s %d\n", v.Channel, v.Kind, v.Count) - case error: - return - } - } else { - return - } - } - }() - - jid := generateJID() - c.Track(jid) - q := c.Queue("goqless_testing_queue") - - putreply, err := q.Put(jid, "dunno", `{"hey": "there"}`, -1, -1, []string{"__callback"}, -1, []string{}) - fmt.Println("Put:", putreply, err) - //fmt.Println(q.Recur(jid, "dunno", `{"hey": "there"}`, 5, 0, 0, []string{}, 1)) - - for { - jobs, err := q.Pop(1) - if err != nil { - panic(err) - } - - if len(jobs) > 0 { - jobs[0].Data = map[string]interface{}{"idid": "id"} - fmt.Println(jobs[0].Complete()) - } - time.Sleep(3 * time.Second) - - fmt.Println(c.GetJob(jid)) - } - - wg.Wait() -} diff --git a/job.go b/job.go old mode 100644 new mode 100755 index d9c970f..36b1202 --- a/job.go +++ b/job.go @@ -1,10 +1,9 @@ -package goqless +package qless import ( "github.com/garyburd/redigo/redis" "reflect" "strings" - "time" ) var ( @@ -19,130 +18,158 @@ type History struct { Worker string } -type Job struct { - Jid string - Klass string - State string - Queue string - Worker string - Tracked bool - Priority int - Expires int64 - Retries int - Remaining int - Data interface{} - Tags StringSlice - History []History - Failure interface{} - Dependents StringSlice - Dependencies interface{} +type Job interface { + JID() string + Class() string + State() string + Queue() string + Worker() string + Tracked() bool + Priority() int + Expires() int64 + Retries() int + Remaining() int + Data() interface{} + Tags() []string + History() []History + Failure() interface{} + Dependents() []string + Dependencies() interface{} - cli *Client + // operations + Fail(typ, message string) (bool, error) + CompleteWithNoData() (string, error) + HeartbeatWithNoData() (bool, error) } -func NewJob(cli *Client) *Job { - return &Job{ - Expires: time.Now().Add(time.Hour * 60).UTC().Unix(), // hour from now - Dependents: nil, // []interface{}{}, - Tracked: false, - Tags: nil, - Jid: generateJID(), - Retries: 5, - Data: nil, - Queue: "mock_queue", - State: "running", - Remaining: 5, - Failure: nil, - History: nil, // []interface{}{}, - Dependencies: nil, - Klass: "Job", - Priority: 0, - Worker: "mock_worker", - cli: cli, - } +//easyjson:json +type job struct { + data *jobData + c *Client } -func (j *Job) Client() *Client { - return j.cli +func (j *job) JID() string { + return j.data.Jid } - -func (j *Job) SetClient(cli *Client) { - j.cli = cli +func (j *job) Class() string { + return j.data.Klass +} +func (j *job) State() string { + return j.data.State +} +func (j *job) Queue() string { + return j.data.Queue +} +func (j *job) Worker() string { + return j.data.Worker +} +func (j *job) Tracked() bool { + return j.data.Tracked +} +func (j *job) Priority() int { + return j.data.Priority +} +func (j *job) Expires() int64 { + return j.data.Expires +} +func (j *job) Retries() int { + return j.data.Retries +} +func (j *job) Remaining() int { + return j.data.Remaining +} +func (j *job) Data() interface{} { + return j.data.Data +} +func (j *job) Tags() []string { + return j.data.Tags +} +func (j *job) History() []History { + return j.data.History +} +func (j *job) Failure() interface{} { + return j.data.Failure +} +func (j *job) Dependents() []string { + return j.data.Dependents +} +func (j *job) Dependencies() interface{} { + return j.data.Dependencies } // Move this from it's current queue into another -func (j *Job) Move(queueName string) (string, error) { - return redis.String(j.cli.Do("qless", 0, "put", timestamp(), queueName, j.Jid, j.Klass, marshal(j.Data), 0)) +func (j *job) Move(queueName string) (string, error) { + return redis.String(j.c.Do("put", timestamp(), queueName, j.data.Jid, j.data.Klass, marshal(j.data.Data), 0)) } // Fail this job // return success, error -func (j *Job) Fail(typ, message string) (bool, error) { - return Bool(j.cli.Do("qless", 0, "fail", timestamp(), j.Jid, j.Worker, typ, message, marshal(j.Data))) +func (j *job) Fail(typ, message string) (bool, error) { + return Bool(j.c.Do("fail", timestamp(), j.data.Jid, j.data.Worker, typ, message, marshal(j.data.Data))) } // Heartbeats this job // return success, error -func (j *Job) Heartbeat() (bool, error) { - return Bool(j.cli.Do("qless", 0, "heartbeat", timestamp(), j.Jid, j.Worker, marshal(j.Data))) +func (j *job) Heartbeat() (bool, error) { + return Bool(j.c.Do("heartbeat", timestamp(), j.data.Jid, j.data.Worker, marshal(j.data.Data))) } // Completes this job // returns state, error -func (j *Job) Complete() (string, error) { - return redis.String(j.cli.Do("qless", 0, "complete", timestamp(), j.Jid, j.Worker, j.Queue, marshal(j.Data))) +func (j *job) Complete() (string, error) { + return redis.String(j.c.Do("complete", timestamp(), j.data.Jid, j.data.Worker, j.data.Queue, marshal(j.data.Data))) } //for big job, save memory in redis -func (j *Job) CompleteWithNoData() (string, error) { - return redis.String(j.cli.Do("qless", 0, "complete", timestamp(), j.Jid, j.Worker, j.Queue, finishBytes)) +func (j *job) CompleteWithNoData() (string, error) { + return redis.String(j.c.Do("complete", timestamp(), j.data.Jid, j.data.Worker, j.data.Queue, finishBytes)) } -func (j *Job) HeartbeatWithNoData() (bool, error) { - return Bool(j.cli.Do("qless", 0, "heartbeat", timestamp(), j.Jid, j.Worker)) +func (j *job) HeartbeatWithNoData() (bool, error) { + return Bool(j.c.Do("heartbeat", timestamp(), j.data.Jid, j.data.Worker)) } // Cancels this job -func (j *Job) Cancel() { - j.cli.Do("qless", 0, "cancel", timestamp(), j.Jid) +func (j *job) Cancel() { + j.c.Do("cancel", timestamp(), j.data.Jid) } // Track this job -func (j *Job) Track() (bool, error) { - return Bool(j.cli.Do("qless", 0, "track", timestamp(), "track", j.Jid)) +func (j *job) Track() (bool, error) { + return Bool(j.c.Do("track", timestamp(), "track", j.data.Jid)) } // Untrack this job -func (j *Job) Untrack() (bool, error) { - return Bool(j.cli.Do("qless", 0, "track", timestamp(), "untrack", j.Jid)) +func (j *job) Untrack() (bool, error) { + return Bool(j.c.Do("track", timestamp(), "untrack", j.data.Jid)) } -func (j *Job) Tag(tags ...interface{}) (string, error) { - args := []interface{}{0, "tag", timestamp(), "add", j.Jid} +func (j *job) Tag(tags ...interface{}) (string, error) { + args := []interface{}{"tag", timestamp(), "add", j.data.Jid} args = append(args, tags...) - return redis.String(j.cli.Do("qless", args...)) + return redis.String(j.c.Do(args...)) } -func (j *Job) Untag(tags ...interface{}) (string, error) { - args := []interface{}{0, "tag", timestamp(), "remove", j.Jid} +func (j *job) Untag(tags ...interface{}) (string, error) { + args := []interface{}{"tag", timestamp(), "remove", j.data.Jid} args = append(args, tags...) - return redis.String(j.cli.Do("qless", args...)) + return redis.String(j.c.Do(args...)) } -func (j *Job) Retry(delay int) (int, error) { - return redis.Int(j.cli.Do("qless", 0, "retry", timestamp(), j.Jid, j.Queue, j.Worker, delay)) +func (j *job) Retry(delay int) (int, error) { + return redis.Int(j.c.Do("retry", timestamp(), j.data.Jid, j.data.Queue, j.data.Worker, delay)) } -func (j *Job) Depend(jids ...interface{}) (string, error) { - args := []interface{}{0, "depends", timestamp(), j.Jid, "on"} +func (j *job) Depend(jids ...interface{}) (string, error) { + args := []interface{}{"depends", timestamp(), j.data.Jid, "on"} args = append(args, jids...) - return redis.String(j.cli.Do("qless", args...)) + return redis.String(j.c.Do(args...)) } -func (j *Job) Undepend(jids ...interface{}) (string, error) { - args := []interface{}{0, "depends", timestamp(), j.Jid, "off"} +func (j *job) Undepend(jids ...interface{}) (string, error) { + args := []interface{}{"depends", timestamp(), j.data.Jid, "off"} args = append(args, jids...) - return redis.String(j.cli.Do("qless", args...)) + return redis.String(j.c.Do(args...)) } type RecurringJob struct { @@ -171,7 +198,7 @@ func NewRecurringJob(cli *Client) *RecurringJob { // data interface{} // klass string func (r *RecurringJob) Update(opts map[string]interface{}) { - args := []interface{}{0, "recur", timestamp(), "update", r.Jid} + args := []interface{}{"recur", timestamp(), "update", r.Jid} vOf := reflect.ValueOf(r).Elem() for key, value := range opts { @@ -187,21 +214,21 @@ func (r *RecurringJob) Update(opts map[string]interface{}) { } } - r.cli.Do("qless", args...) + r.cli.Do(args...) } func (r *RecurringJob) Cancel() { - r.cli.Do("qless", 0, "recur", timestamp(), "off", r.Jid) + r.cli.Do("recur", timestamp(), "off", r.Jid) } func (r *RecurringJob) Tag(tags ...interface{}) { - args := []interface{}{0, "recur", timestamp(), "tag", r.Jid} + args := []interface{}{"recur", timestamp(), "tag", r.Jid} args = append(args, tags...) - r.cli.Do("qless", args...) + r.cli.Do(args...) } func (r *RecurringJob) Untag(tags ...interface{}) { - args := []interface{}{0, "recur", timestamp(), "untag", r.Jid} + args := []interface{}{"recur", timestamp(), "untag", r.Jid} args = append(args, tags...) - r.cli.Do("qless", args...) + r.cli.Do(args...) } diff --git a/lua.go b/lua.go deleted file mode 100644 index 8340af3..0000000 --- a/lua.go +++ /dev/null @@ -1,55 +0,0 @@ -package goqless - -import ( - "fmt" - "github.com/garyburd/redigo/redis" - "io/ioutil" - // "log" - "os" - "path/filepath" - "strings" -) - -type Lua struct { - conn redis.Conn - f map[string]*redis.Script -} - -func NewLua(c redis.Conn) *Lua { - l := &Lua{c, make(map[string]*redis.Script)} - return l -} - -// loads all the given scripts from the path -func (l *Lua) LoadScripts(path string) error { - err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { - if strings.HasSuffix(info.Name(), ".lua") { - src, err := ioutil.ReadFile(path) - if err != nil { - return err - } - script := redis.NewScript(-1, string(src)) - err = script.Load(l.conn) - if err != nil { - return err - } - - l.f[info.Name()[:len(info.Name())-4]] = script - - // log.Println("Loaded: ", info.Name()[:len(info.Name())-4]) - } - - return nil - }) - - return err -} - -// calls a script with the arguments -func (l *Lua) Do(name string, keysAndArgs ...interface{}) (interface{}, error) { - if fn, ok := l.f[name]; ok { - return fn.Do(l.conn, keysAndArgs...) - } - - return nil, fmt.Errorf("no script named %s found", name) -} diff --git a/qless.lua b/qless.lua new file mode 100644 index 0000000..9e5d2ce --- /dev/null +++ b/qless.lua @@ -0,0 +1,2518 @@ +-- Current SHA: 9a235bcc4091f798e6a64503969900f1415df8a4 +-- This is a generated file +local Qless = { + ns = 'ql:' +} + +local QlessQueue = { + ns = Qless.ns .. 'q:' +} +QlessQueue.__index = QlessQueue + +local QlessWorker = { + ns = Qless.ns .. 'w:' +} +QlessWorker.__index = QlessWorker + +local QlessJob = { + ns = Qless.ns .. 'j:' +} +QlessJob.__index = QlessJob + +local QlessRecurringJob = {} +QlessRecurringJob.__index = QlessRecurringJob + +local QlessResource = { + ns = Qless.ns .. 'rs:' +} +QlessResource.__index = QlessResource; + +Qless.config = {} + +function table.extend(self, other) + for i, v in ipairs(other) do + table.insert(self, v) + end +end + +function Qless.publish(channel, message) + redis.call('publish', Qless.ns .. channel, message) +end + +function Qless.job(jid) + assert(jid, 'Job(): no jid provided') + local job = {} + setmetatable(job, QlessJob) + job.jid = jid + return job +end + +function Qless.recurring(jid) + assert(jid, 'Recurring(): no jid provided') + local job = {} + setmetatable(job, QlessRecurringJob) + job.jid = jid + return job +end + +function Qless.resource(rid) + assert(rid, 'Resource(): no rid provided') + local res = {} + setmetatable(res, QlessResource) + res.rid = rid + return res +end + +function Qless.failed(group, start, limit) + start = assert(tonumber(start or 0), + 'Failed(): Arg "start" is not a number: ' .. (start or 'nil')) + limit = assert(tonumber(limit or 25), + 'Failed(): Arg "limit" is not a number: ' .. (limit or 'nil')) + + if group then + return { + total = redis.call('llen', 'ql:f:' .. group), + jobs = redis.call('lrange', 'ql:f:' .. group, start, start + limit - 1) + } + else + local response = {} + local groups = redis.call('smembers', 'ql:failures') + for index, group in ipairs(groups) do + response[group] = redis.call('llen', 'ql:f:' .. group) + end + return response + end +end + +function Qless.jobs(now, state, ...) + assert(state, 'Jobs(): Arg "state" missing') + if state == 'complete' then + local offset = assert(tonumber(arg[1] or 0), + 'Jobs(): Arg "offset" not a number: ' .. tostring(arg[1])) + local count = assert(tonumber(arg[2] or 25), + 'Jobs(): Arg "count" not a number: ' .. tostring(arg[2])) + return redis.call('zrevrange', 'ql:completed', offset, + offset + count - 1) + else + local name = assert(arg[1], 'Jobs(): Arg "queue" missing') + local offset = assert(tonumber(arg[2] or 0), + 'Jobs(): Arg "offset" not a number: ' .. tostring(arg[2])) + local count = assert(tonumber(arg[3] or 25), + 'Jobs(): Arg "count" not a number: ' .. tostring(arg[3])) + + local queue = Qless.queue(name) + if state == 'running' then + return queue.locks.peek(now, offset, count) + elseif state == 'stalled' then + return queue.locks.expired(now, offset, count) + elseif state == 'scheduled' then + queue:check_scheduled(now, queue.scheduled.length()) + return queue.scheduled.peek(now, offset, count) + elseif state == 'depends' then + return queue.depends.peek(now, offset, count) + elseif state == 'recurring' then + return queue.recurring.peek(math.huge, offset, count) + else + error('Jobs(): Unknown type "' .. state .. '"') + end + end +end + +function Qless.track(now, command, jid) + if command ~= nil then + assert(jid, 'Track(): Arg "jid" missing') + assert(Qless.job(jid):exists(), 'Track(): Job does not exist') + if string.lower(command) == 'track' then + Qless.publish('track', jid) + return redis.call('zadd', 'ql:tracked', now, jid) + elseif string.lower(command) == 'untrack' then + Qless.publish('untrack', jid) + return redis.call('zrem', 'ql:tracked', jid) + else + error('Track(): Unknown action "' .. command .. '"') + end + else + local response = { + jobs = {}, + expired = {} + } + local jids = redis.call('zrange', 'ql:tracked', 0, -1) + for index, jid in ipairs(jids) do + local data = Qless.job(jid):data() + if data then + table.insert(response.jobs, data) + else + table.insert(response.expired, jid) + end + end + return response + end +end + +function Qless.tag(now, command, ...) + assert(command, + 'Tag(): Arg "command" must be "add", "remove", "get" or "top"') + + if command == 'add' then + local jid = assert(arg[1], 'Tag(): Arg "jid" missing') + local tags = redis.call('hget', QlessJob.ns .. jid, 'tags') + if tags then + tags = cjson.decode(tags) + local _tags = {} + for i,v in ipairs(tags) do _tags[v] = true end + + for i=2,#arg do + local tag = arg[i] + if _tags[tag] == nil then + _tags[tag] = true + table.insert(tags, tag) + end + redis.call('zadd', 'ql:t:' .. tag, now, jid) + redis.call('zincrby', 'ql:tags', 1, tag) + end + + tags = cjson.encode(tags) + redis.call('hset', QlessJob.ns .. jid, 'tags', tags) + return tags + else + error('Tag(): Job ' .. jid .. ' does not exist') + end + elseif command == 'remove' then + local jid = assert(arg[1], 'Tag(): Arg "jid" missing') + local tags = redis.call('hget', QlessJob.ns .. jid, 'tags') + if tags then + tags = cjson.decode(tags) + local _tags = {} + for i,v in ipairs(tags) do _tags[v] = true end + + for i=2,#arg do + local tag = arg[i] + _tags[tag] = nil + redis.call('zrem', 'ql:t:' .. tag, jid) + redis.call('zincrby', 'ql:tags', -1, tag) + end + + local results = {} + for i,tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end + + tags = cjson.encode(results) + redis.call('hset', QlessJob.ns .. jid, 'tags', tags) + return results + else + error('Tag(): Job ' .. jid .. ' does not exist') + end + elseif command == 'get' then + local tag = assert(arg[1], 'Tag(): Arg "tag" missing') + local offset = assert(tonumber(arg[2] or 0), + 'Tag(): Arg "offset" not a number: ' .. tostring(arg[2])) + local count = assert(tonumber(arg[3] or 25), + 'Tag(): Arg "count" not a number: ' .. tostring(arg[3])) + return { + total = redis.call('zcard', 'ql:t:' .. tag), + jobs = redis.call('zrange', 'ql:t:' .. tag, offset, offset + count - 1) + } + elseif command == 'top' then + local offset = assert(tonumber(arg[1] or 0) , 'Tag(): Arg "offset" not a number: ' .. tostring(arg[1])) + local count = assert(tonumber(arg[2] or 25), 'Tag(): Arg "count" not a number: ' .. tostring(arg[2])) + return redis.call('zrevrangebyscore', 'ql:tags', '+inf', 2, 'limit', offset, count) + else + error('Tag(): First argument must be "add", "remove" or "get"') + end +end + +function Qless.cancel(now, ...) + local dependents = {} + for _, jid in ipairs(arg) do + dependents[jid] = redis.call( + 'smembers', QlessJob.ns .. jid .. '-dependents') or {} + end + + for i, jid in ipairs(arg) do + for j, dep in ipairs(dependents[jid]) do + if dependents[dep] == nil then + error('Cancel(): ' .. jid .. ' is a dependency of ' .. dep .. + ' but is not mentioned to be canceled') + end + end + end + + local cancelled_jids = {} + + for _, jid in ipairs(arg) do + local real_jid, state, queue, failure, worker = unpack(redis.call( + 'hmget', QlessJob.ns .. jid, 'jid', 'state', 'queue', 'failure', 'worker')) + + if state ~= false and state ~= 'complete' then + table.insert(cancelled_jids, jid) + + local encoded = cjson.encode({ + jid = jid, + worker = worker, + event = 'canceled', + queue = queue + }) + Qless.publish('log', encoded) + + if worker and (worker ~= '') then + redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid) + Qless.publish('w:' .. worker, encoded) + end + + if queue then + local queue = Qless.queue(queue) + queue.work.remove(jid) + queue.locks.remove(jid) + queue.scheduled.remove(jid) + queue.depends.remove(jid) + end + + Qless.job(jid):release_resources(now) + + for i, j in ipairs(redis.call( + 'smembers', QlessJob.ns .. jid .. '-dependencies')) do + redis.call('srem', QlessJob.ns .. j .. '-dependents', jid) + end + + redis.call('del', QlessJob.ns .. jid .. '-dependencies') + + if state == 'failed' then + failure = cjson.decode(failure) + redis.call('lrem', 'ql:f:' .. failure.group, 0, jid) + if redis.call('llen', 'ql:f:' .. failure.group) == 0 then + redis.call('srem', 'ql:failures', failure.group) + end + local bin = failure.when - (failure.when % 86400) + local failed = redis.call( + 'hget', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed') + redis.call('hset', + 'ql:s:stats:' .. bin .. ':' .. queue, 'failed', failed - 1) + end + + local tags = cjson.decode( + redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}') + for i, tag in ipairs(tags) do + redis.call('zrem', 'ql:t:' .. tag, jid) + redis.call('zincrby', 'ql:tags', -1, tag) + end + + if redis.call('zscore', 'ql:tracked', jid) ~= false then + Qless.publish('canceled', jid) + end + + redis.call('del', QlessJob.ns .. jid) + redis.call('del', QlessJob.ns .. jid .. '-history') + end + end + + return cancelled_jids +end + +local Set = {} + +function Set.new (t) + local set = {} + for _, l in ipairs(t) do set[l] = true end + return set +end + +function Set.union (a,b) + local res = Set.new{} + for k in pairs(a) do res[k] = true end + for k in pairs(b) do res[k] = true end + return res +end + +function Set.intersection (a,b) + local res = Set.new{} + for k in pairs(a) do + res[k] = b[k] + end + return res +end + +function Set.diff(a,b) + local res = Set.new{} + for k in pairs(a) do + if not b[k] then res[k] = true end + end + + return res +end + +Qless.config.defaults = { + ['application'] = 'qless', + ['heartbeat'] = 60, + ['grace-period'] = 10, + ['stats-history'] = 30, + ['histogram-history'] = 7, + ['jobs-history-count'] = 50000, + ['jobs-history'] = 604800 +} + +Qless.config.get = function(key, default) + if key then + return redis.call('hget', 'ql:config', key) or + Qless.config.defaults[key] or default + else + local reply = redis.call('hgetall', 'ql:config') + for i = 1, #reply, 2 do + Qless.config.defaults[reply[i]] = reply[i + 1] + end + return Qless.config.defaults + end +end + +Qless.config.set = function(option, value) + assert(option, 'config.set(): Arg "option" missing') + assert(value , 'config.set(): Arg "value" missing') + Qless.publish('log', cjson.encode({ + event = 'config_set', + option = option, + value = value + })) + + redis.call('hset', 'ql:config', option, value) +end + +Qless.config.unset = function(option) + assert(option, 'config.unset(): Arg "option" missing') + Qless.publish('log', cjson.encode({ + event = 'config_unset', + option = option + })) + + redis.call('hdel', 'ql:config', option) +end + +function QlessJob:data(...) + local job = redis.call( + 'hmget', QlessJob.ns .. self.jid, 'jid', 'klass', 'state', 'queue', + 'worker', 'priority', 'expires', 'retries', 'remaining', 'data', + 'tags', 'failure', 'resources', 'result_data', 'throttle_interval') + + if not job[1] then + return nil + end + + local data = { + jid = job[1], + klass = job[2], + state = job[3], + queue = job[4], + worker = job[5] or '', + tracked = redis.call( + 'zscore', 'ql:tracked', self.jid) ~= false, + priority = tonumber(job[6]), + expires = tonumber(job[7]) or 0, + retries = tonumber(job[8]), + remaining = math.floor(tonumber(job[9])), + data = job[10], + tags = cjson.decode(job[11]), + history = self:history(), + failure = cjson.decode(job[12] or '{}'), + resources = cjson.decode(job[13] or '[]'), + result_data = cjson.decode(job[14] or '{}'), + interval = tonumber(job[15]) or 0, + dependents = redis.call( + 'smembers', QlessJob.ns .. self.jid .. '-dependents'), + dependencies = redis.call( + 'smembers', QlessJob.ns .. self.jid .. '-dependencies') + } + + if #arg > 0 then + local response = {} + for index, key in ipairs(arg) do + table.insert(response, data[key]) + end + return response + else + return data + end +end + +function QlessJob:complete(now, worker, queue, data, ...) + assert(worker, 'Complete(): Arg "worker" missing') + assert(queue , 'Complete(): Arg "queue" missing') + data = assert(cjson.decode(data), + 'Complete(): Arg "data" missing or not JSON: ' .. tostring(data)) + + local options = {} + for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end + + local nextq = options['next'] + local result_data = options['result_data'] + local delay = assert(tonumber(options['delay'] or 0)) + local depends = assert(cjson.decode(options['depends'] or '[]'), + 'Complete(): Arg "depends" not JSON: ' .. tostring(options['depends'])) + + if options['delay'] and nextq == nil then + error('Complete(): "delay" cannot be used without a "next".') + end + + if options['depends'] and nextq == nil then + error('Complete(): "depends" cannot be used without a "next".') + end + + local bin = now - (now % 86400) + + local lastworker, state, priority, retries, interval = unpack( + redis.call('hmget', QlessJob.ns .. self.jid, 'worker', 'state', + 'priority', 'retries', 'throttle_interval')) + + if lastworker == false then + error('Complete(): Job does not exist') + elseif (state ~= 'running') then + error('Complete(): Job is not currently running: ' .. state) + elseif lastworker ~= worker then + error('Complete(): Job has been handed out to another worker: ' .. + tostring(lastworker)) + end + + local next_run = 0 + if interval then + interval = tonumber(interval) + if interval > 0 then + next_run = now + interval + else + next_run = -1 + end + end + + self:history(now, 'done') + + if data then + redis.call('hset', QlessJob.ns .. self.jid, 'data', cjson.encode(data)) + end + + if result_data then + redis.call('hset', QlessJob.ns .. self.jid, 'result_data', result_data) + end + + local queue_obj = Qless.queue(queue) + queue_obj.work.remove(self.jid) + queue_obj.locks.remove(self.jid) + queue_obj.scheduled.remove(self.jid) + + self:release_resources(now) + + local time = tonumber( + redis.call('hget', QlessJob.ns .. self.jid, 'time') or now) + local waiting = now - time + Qless.queue(queue):stat(now, 'run', waiting) + redis.call('hset', QlessJob.ns .. self.jid, + 'time', string.format("%.20f", now)) + + redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid) + + if redis.call('zscore', 'ql:tracked', self.jid) ~= false then + Qless.publish('completed', self.jid) + end + + if nextq then + queue_obj = Qless.queue(nextq) + Qless.publish('log', cjson.encode({ + jid = self.jid, + event = 'advanced', + queue = queue, + to = nextq + })) + + self:history(now, 'put', {q = nextq}) + + if redis.call('zscore', 'ql:queues', nextq) == false then + redis.call('zadd', 'ql:queues', now, nextq) + end + + redis.call('hmset', QlessJob.ns .. self.jid, + 'state', 'waiting', + 'worker', '', + 'failure', '{}', + 'queue', nextq, + 'expires', 0, + 'remaining', tonumber(retries)) + + if (delay > 0) and (#depends == 0) then + queue_obj.scheduled.add(now + delay, self.jid) + return 'scheduled' + else + local count = 0 + for i, j in ipairs(depends) do + local state = redis.call('hget', QlessJob.ns .. j, 'state') + if (state and state ~= 'complete') then + count = count + 1 + redis.call( + 'sadd', QlessJob.ns .. j .. '-dependents',self.jid) + redis.call( + 'sadd', QlessJob.ns .. self.jid .. '-dependencies', j) + end + end + if count > 0 then + queue_obj.depends.add(now, self.jid) + redis.call('hset', QlessJob.ns .. self.jid, 'state', 'depends') + if delay > 0 then + queue_obj.depends.add(now, self.jid) + redis.call('hset', QlessJob.ns .. self.jid, 'scheduled', now + delay) + end + return 'depends' + else + if self:acquire_resources(now) then + queue_obj.work.add(now, priority, self.jid) + end + return 'waiting' + end + end + else + Qless.publish('log', cjson.encode({ + jid = self.jid, + event = 'completed', + queue = queue + })) + + redis.call('hmset', QlessJob.ns .. self.jid, + 'state', 'complete', + 'worker', '', + 'failure', '{}', + 'queue', '', + 'expires', 0, + 'remaining', tonumber(retries), + 'throttle_next_run', next_run + ) + + local count = Qless.config.get('jobs-history-count') + local time = Qless.config.get('jobs-history') + + count = tonumber(count or 50000) + time = tonumber(time or 7 * 24 * 60 * 60) + + redis.call('zadd', 'ql:completed', now, self.jid) + + local jids = redis.call('zrangebyscore', 'ql:completed', 0, now - time) + for index, jid in ipairs(jids) do + local tags = cjson.decode( + redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}') + for i, tag in ipairs(tags) do + redis.call('zrem', 'ql:t:' .. tag, jid) + redis.call('zincrby', 'ql:tags', -1, tag) + end + redis.call('del', QlessJob.ns .. jid) + redis.call('del', QlessJob.ns .. jid .. '-history') + end + redis.call('zremrangebyscore', 'ql:completed', 0, now - time) + + jids = redis.call('zrange', 'ql:completed', 0, (-1-count)) + for index, jid in ipairs(jids) do + local tags = cjson.decode( + redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}') + for i, tag in ipairs(tags) do + redis.call('zrem', 'ql:t:' .. tag, jid) + redis.call('zincrby', 'ql:tags', -1, tag) + end + redis.call('del', QlessJob.ns .. jid) + redis.call('del', QlessJob.ns .. jid .. '-history') + end + redis.call('zremrangebyrank', 'ql:completed', 0, (-1-count)) + + for i, j in ipairs(redis.call( + 'smembers', QlessJob.ns .. self.jid .. '-dependents')) do + redis.call('srem', QlessJob.ns .. j .. '-dependencies', self.jid) + if redis.call( + 'scard', QlessJob.ns .. j .. '-dependencies') == 0 then + local q, p, scheduled = unpack( + redis.call('hmget', QlessJob.ns .. j, 'queue', 'priority', 'scheduled')) + if q then + local queue = Qless.queue(q) + queue.depends.remove(j) + if scheduled then + queue.scheduled.add(scheduled, j) + redis.call('hset', QlessJob.ns .. j, 'state', 'scheduled') + redis.call('hdel', QlessJob.ns .. j, 'scheduled') + else + if Qless.job(j):acquire_resources(now) then + queue.work.add(now, p, j) + end + redis.call('hset', QlessJob.ns .. j, 'state', 'waiting') + end + end + end + end + + redis.call('del', QlessJob.ns .. self.jid .. '-dependents') + + return 'complete' + end +end + +function QlessJob:fail(now, worker, group, message, data) + local worker = assert(worker , 'Fail(): Arg "worker" missing') + local group = assert(group , 'Fail(): Arg "group" missing') + local message = assert(message , 'Fail(): Arg "message" missing') + + local bin = now - (now % 86400) + + if data then + data = cjson.decode(data) + end + + local queue, state, oldworker = unpack(redis.call( + 'hmget', QlessJob.ns .. self.jid, 'queue', 'state', 'worker')) + + if not state then + error('Fail(): Job does not exist') + elseif state ~= 'running' then + error('Fail(): Job not currently running: ' .. state) + elseif worker ~= oldworker then + error('Fail(): Job running with another worker: ' .. oldworker) + end + + Qless.publish('log', cjson.encode({ + jid = self.jid, + event = 'failed', + worker = worker, + group = group, + message = message + })) + + if redis.call('zscore', 'ql:tracked', self.jid) ~= false then + Qless.publish('failed', self.jid) + end + + redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid) + + self:history(now, 'failed', {worker = worker, group = group}) + + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failures', 1) + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed' , 1) + + local queue_obj = Qless.queue(queue) + queue_obj.work.remove(self.jid) + queue_obj.locks.remove(self.jid) + queue_obj.scheduled.remove(self.jid) + + self:release_resources(now) + + if data then + redis.call('hset', QlessJob.ns .. self.jid, 'data', cjson.encode(data)) + end + + redis.call('hmset', QlessJob.ns .. self.jid, + 'state', 'failed', + 'worker', '', + 'expires', '', + 'failure', cjson.encode({ + ['group'] = group, + ['message'] = message, + ['when'] = math.floor(now), + ['worker'] = worker + })) + + redis.call('sadd', 'ql:failures', group) + redis.call('lpush', 'ql:f:' .. group, self.jid) + + + return self.jid +end + +function QlessJob:retry(now, queue, worker, delay, group, message) + assert(queue , 'Retry(): Arg "queue" missing') + assert(worker, 'Retry(): Arg "worker" missing') + delay = assert(tonumber(delay or 0), + 'Retry(): Arg "delay" not a number: ' .. tostring(delay)) + + local oldqueue, state, retries, oldworker, priority, failure = unpack( + redis.call('hmget', QlessJob.ns .. self.jid, 'queue', 'state', + 'retries', 'worker', 'priority', 'failure')) + + if oldworker == false then + error('Retry(): Job does not exist') + elseif state ~= 'running' then + error('Retry(): Job is not currently running: ' .. state) + elseif oldworker ~= worker then + error('Retry(): Job has been given to another worker: ' .. oldworker) + end + + local remaining = tonumber(redis.call( + 'hincrby', QlessJob.ns .. self.jid, 'remaining', -1)) + redis.call('hdel', QlessJob.ns .. self.jid, 'grace') + + Qless.queue(oldqueue).locks.remove(self.jid) + self:release_resources(now) + + redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid) + + if remaining < 0 then + local group = group or 'failed-retries-' .. queue + self:history(now, 'failed', {['group'] = group}) + + redis.call('hmset', QlessJob.ns .. self.jid, 'state', 'failed', + 'worker', '', + 'expires', '') + if group ~= nil and message ~= nil then + redis.call('hset', QlessJob.ns .. self.jid, + 'failure', cjson.encode({ + ['group'] = group, + ['message'] = message, + ['when'] = math.floor(now), + ['worker'] = worker + }) + ) + else + redis.call('hset', QlessJob.ns .. self.jid, + 'failure', cjson.encode({ + ['group'] = group, + ['message'] = + 'Job exhausted retries in queue "' .. oldqueue .. '"', + ['when'] = now, + ['worker'] = unpack(self:data('worker')) + })) + end + + redis.call('sadd', 'ql:failures', group) + redis.call('lpush', 'ql:f:' .. group, self.jid) + local bin = now - (now % 86400) + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failures', 1) + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed' , 1) + else + local queue_obj = Qless.queue(queue) + if delay > 0 then + queue_obj.scheduled.add(now + delay, self.jid) + redis.call('hset', QlessJob.ns .. self.jid, 'state', 'scheduled') + else + if self:acquire_resources(now) then + queue_obj.work.add(now, priority, self.jid) + end + redis.call('hset', QlessJob.ns .. self.jid, 'state', 'waiting') + end + + if group ~= nil and message ~= nil then + redis.call('hset', QlessJob.ns .. self.jid, + 'failure', cjson.encode({ + ['group'] = group, + ['message'] = message, + ['when'] = math.floor(now), + ['worker'] = worker + }) + ) + end + end + + return math.floor(remaining) +end + +function QlessJob:depends(now, command, ...) + assert(command, 'Depends(): Arg "command" missing') + local state = redis.call('hget', QlessJob.ns .. self.jid, 'state') + if state ~= 'depends' then + error('Depends(): Job ' .. self.jid .. + ' not in the depends state: ' .. tostring(state)) + end + + if command == 'on' then + for i, j in ipairs(arg) do + local state = redis.call('hget', QlessJob.ns .. j, 'state') + if (state and state ~= 'complete') then + redis.call( + 'sadd', QlessJob.ns .. j .. '-dependents' , self.jid) + redis.call( + 'sadd', QlessJob.ns .. self.jid .. '-dependencies', j) + end + end + return true + elseif command == 'off' then + if arg[1] == 'all' then + for i, j in ipairs(redis.call( + 'smembers', QlessJob.ns .. self.jid .. '-dependencies')) do + redis.call('srem', QlessJob.ns .. j .. '-dependents', self.jid) + end + redis.call('del', QlessJob.ns .. self.jid .. '-dependencies') + local q, p = unpack(redis.call( + 'hmget', QlessJob.ns .. self.jid, 'queue', 'priority')) + if q then + local queue_obj = Qless.queue(q) + queue_obj.depends.remove(self.jid) + if self:acquire_resources(now) then + queue_obj.work.add(now, p, self.jid) + end + redis.call('hset', QlessJob.ns .. self.jid, 'state', 'waiting') + end + else + for i, j in ipairs(arg) do + redis.call('srem', QlessJob.ns .. j .. '-dependents', self.jid) + redis.call( + 'srem', QlessJob.ns .. self.jid .. '-dependencies', j) + if redis.call('scard', + QlessJob.ns .. self.jid .. '-dependencies') == 0 then + local q, p = unpack(redis.call( + 'hmget', QlessJob.ns .. self.jid, 'queue', 'priority')) + if q then + local queue_obj = Qless.queue(q) + queue_obj.depends.remove(self.jid) + if self:acquire_resources(now) then + queue_obj.work.add(now, p, self.jid) + end + redis.call('hset', + QlessJob.ns .. self.jid, 'state', 'waiting') + end + end + end + end + return true + else + error('Depends(): Argument "command" must be "on" or "off"') + end +end + +function QlessJob:heartbeat(now, worker, data) + assert(worker, 'Heatbeat(): Arg "worker" missing') + + local queue = redis.call('hget', QlessJob.ns .. self.jid, 'queue') or '' + local expires = now + tonumber( + Qless.config.get(queue .. '-heartbeat') or + Qless.config.get('heartbeat', 60)) + + if data then + data = cjson.decode(data) + end + + local job_worker, state = unpack( + redis.call('hmget', QlessJob.ns .. self.jid, 'worker', 'state')) + if job_worker == false then + error('Heartbeat(): Job does not exist') + elseif state ~= 'running' then + error('Heartbeat(): Job not currently running: ' .. state) + elseif job_worker ~= worker or #job_worker == 0 then + error('Heartbeat(): Job given out to another worker: ' .. job_worker) + else + if data then + redis.call('hmset', QlessJob.ns .. self.jid, 'expires', + expires, 'worker', worker, 'data', cjson.encode(data)) + else + redis.call('hmset', QlessJob.ns .. self.jid, + 'expires', expires, 'worker', worker) + end + + redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, self.jid) + + local queue = Qless.queue( + redis.call('hget', QlessJob.ns .. self.jid, 'queue')) + queue.locks.add(expires, self.jid) + return expires + end +end + +function QlessJob:priority(priority) + priority = assert(tonumber(priority), + 'Priority(): Arg "priority" missing or not a number: ' .. + tostring(priority)) + + local queue = redis.call('hget', QlessJob.ns .. self.jid, 'queue') + + if queue == nil then + error('Priority(): Job ' .. self.jid .. ' does not exist') + elseif queue == '' then + redis.call('hset', QlessJob.ns .. self.jid, 'priority', priority) + return priority + else + local queue_obj = Qless.queue(queue) + if queue_obj.work.score(self.jid) then + queue_obj.work.add(0, priority, self.jid) + end + redis.call('hset', QlessJob.ns .. self.jid, 'priority', priority) + return priority + end +end + +function QlessJob:update(data) + local tmp = {} + for k, v in pairs(data) do + table.insert(tmp, k) + table.insert(tmp, v) + end + redis.call('hmset', QlessJob.ns .. self.jid, unpack(tmp)) +end + +function QlessJob:timeout(now) + local queue_name, state, worker = unpack(redis.call('hmget', + QlessJob.ns .. self.jid, 'queue', 'state', 'worker')) + if queue_name == nil then + error('Timeout(): Job does not exist') + elseif state ~= 'running' then + error('Timeout(): Job ' .. self.jid .. ' not running') + else + self:history(now, 'timed-out') + local queue = Qless.queue(queue_name) + queue.locks.remove(self.jid) + queue.work.add(now, math.huge, self.jid) + redis.call('hmset', QlessJob.ns .. self.jid, + 'state', 'stalled', 'expires', 0) + local encoded = cjson.encode({ + jid = self.jid, + event = 'lock_lost', + worker = worker + }) + Qless.publish('w:' .. worker, encoded) + Qless.publish('log', encoded) + return queue_name + end +end + +function QlessJob:exists() + return redis.call('exists', QlessJob.ns .. self.jid) == 1 +end + +function QlessJob:history(now, what, item) + local history = redis.call('hget', QlessJob.ns .. self.jid, 'history') + if history then + history = cjson.decode(history) + for i, value in ipairs(history) do + redis.call('rpush', QlessJob.ns .. self.jid .. '-history', + cjson.encode({math.floor(value.put), 'put', {q = value.q}})) + + if value.popped then + redis.call('rpush', QlessJob.ns .. self.jid .. '-history', + cjson.encode({math.floor(value.popped), 'popped', + {worker = value.worker}})) + end + + if value.failed then + redis.call('rpush', QlessJob.ns .. self.jid .. '-history', + cjson.encode( + {math.floor(value.failed), 'failed', nil})) + end + + if value.done then + redis.call('rpush', QlessJob.ns .. self.jid .. '-history', + cjson.encode( + {math.floor(value.done), 'done', nil})) + end + end + redis.call('hdel', QlessJob.ns .. self.jid, 'history') + end + + if what == nil then + local response = {} + for i, value in ipairs(redis.call('lrange', + QlessJob.ns .. self.jid .. '-history', 0, -1)) do + value = cjson.decode(value) + local dict = value[3] or {} + dict['when'] = value[1] + dict['what'] = value[2] + table.insert(response, dict) + end + return response + else + local count = tonumber(Qless.config.get('max-job-history', 100)) + if count > 0 then + local obj = redis.call('lpop', QlessJob.ns .. self.jid .. '-history') + redis.call('ltrim', QlessJob.ns .. self.jid .. '-history', -count + 2, -1) + if obj ~= nil then + redis.call('lpush', QlessJob.ns .. self.jid .. '-history', obj) + end + end + return redis.call('rpush', QlessJob.ns .. self.jid .. '-history', + cjson.encode({math.floor(now), what, item})) + end +end + +function QlessJob:release_resources(now) + local resources = redis.call('hget', QlessJob.ns .. self.jid, 'resources') + resources = cjson.decode(resources or '[]') + for _, res in ipairs(resources) do + Qless.resource(res):release(now, self.jid) + end +end + +function QlessJob:acquire_resources(now) + local resources, priority = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'resources', 'priority')) + resources = cjson.decode(resources or '[]') + if (#resources == 0) then + return true + end + + local acquired_all = true + + for _, res in ipairs(resources) do + local ok, res = pcall(function() return Qless.resource(res):acquire(now, priority, self.jid) end) + if not ok then + self:set_failed(now, 'system:fatal', res.msg) + return false + end + acquired_all = acquired_all and res + end + + return acquired_all +end + +function QlessJob:set_failed(now, group, message, worker, release_work, release_resources) + local group = assert(group, 'Fail(): Arg "group" missing') + local message = assert(message, 'Fail(): Arg "message" missing') + local worker = worker or 'none' + local release_work = release_work or true + local release_resources = release_resources or false + + local bin = now - (now % 86400) + + local queue = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'queue')) + + Qless.publish('log', cjson.encode({ + jid = self.jid, + event = 'failed', + worker = worker, + group = group, + message = message + })) + + if redis.call('zscore', 'ql:tracked', self.jid) ~= false then + Qless.publish('failed', self.jid) + end + + self:history(now, 'failed', {group = group}) + + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failures', 1) + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed' , 1) + + if release_work then + local queue_obj = Qless.queue(queue) + queue_obj.work.remove(self.jid) + queue_obj.locks.remove(self.jid) + queue_obj.scheduled.remove(self.jid) + end + + if release_resources then + self:release_resources(now) + end + + redis.call('hmset', QlessJob.ns .. self.jid, + 'state', 'failed', + 'worker', '', + 'expires', '', + 'failure', cjson.encode({ + ['group'] = group, + ['message'] = message, + ['when'] = math.floor(now) + })) + + redis.call('sadd', 'ql:failures', group) + redis.call('lpush', 'ql:f:' .. group, self.jid) + + + return self.jid +end +function Qless.queue(name) + assert(name, 'Queue(): no queue name provided') + local queue = {} + setmetatable(queue, QlessQueue) + queue.name = name + + queue.work = { + peek = function(count) + if count == 0 then + return {} + end + local jids = {} + for index, jid in ipairs(redis.call( + 'zrevrange', queue:prefix('work'), 0, count - 1)) do + table.insert(jids, jid) + end + return jids + end, remove = function(...) + if #arg > 0 then + return redis.call('zrem', queue:prefix('work'), unpack(arg)) + end + end, add = function(now, priority, jid) + return redis.call('zadd', + queue:prefix('work'), priority - (now / 10000000000), jid) + end, score = function(jid) + return redis.call('zscore', queue:prefix('work'), jid) + end, length = function() + return redis.call('zcard', queue:prefix('work')) + end + } + + queue.locks = { + expired = function(now, offset, count) + return redis.call('zrangebyscore', + queue:prefix('locks'), -math.huge, now, 'LIMIT', offset, count) + end, peek = function(now, offset, count) + return redis.call('zrangebyscore', queue:prefix('locks'), + now, math.huge, 'LIMIT', offset, count) + end, add = function(expires, jid) + redis.call('zadd', queue:prefix('locks'), expires, jid) + end, remove = function(...) + if #arg > 0 then + return redis.call('zrem', queue:prefix('locks'), unpack(arg)) + end + end, running = function(now) + return redis.call('zcount', queue:prefix('locks'), now, math.huge) + end, length = function(now) + if now then + return redis.call('zcount', queue:prefix('locks'), 0, now) + else + return redis.call('zcard', queue:prefix('locks')) + end + end, job_time_left = function(now, jid) + return tonumber(redis.call('zscore', queue:prefix('locks'), jid) or 0) - now + end + } + + queue.depends = { + peek = function(now, offset, count) + return redis.call('zrange', + queue:prefix('depends'), offset, offset + count - 1) + end, add = function(now, jid) + redis.call('zadd', queue:prefix('depends'), now, jid) + end, remove = function(...) + if #arg > 0 then + return redis.call('zrem', queue:prefix('depends'), unpack(arg)) + end + end, length = function() + return redis.call('zcard', queue:prefix('depends')) + end + } + + queue.scheduled = { + peek = function(now, offset, count) + return redis.call('zrange', + queue:prefix('scheduled'), offset, offset + count - 1) + end, ready = function(now, offset, count) + return redis.call('zrangebyscore', + queue:prefix('scheduled'), 0, now, 'LIMIT', offset, count) + end, add = function(when, jid) + redis.call('zadd', queue:prefix('scheduled'), when, jid) + end, remove = function(...) + if #arg > 0 then + return redis.call('zrem', queue:prefix('scheduled'), unpack(arg)) + end + end, length = function() + return redis.call('zcard', queue:prefix('scheduled')) + end + } + + queue.recurring = { + peek = function(now, offset, count) + return redis.call('zrangebyscore', queue:prefix('recur'), + 0, now, 'LIMIT', offset, count) + end, ready = function(now, offset, count) + end, add = function(when, jid) + redis.call('zadd', queue:prefix('recur'), when, jid) + end, remove = function(...) + if #arg > 0 then + return redis.call('zrem', queue:prefix('recur'), unpack(arg)) + end + end, update = function(increment, jid) + redis.call('zincrby', queue:prefix('recur'), increment, jid) + end, score = function(jid) + return redis.call('zscore', queue:prefix('recur'), jid) + end, length = function() + return redis.call('zcard', queue:prefix('recur')) + end + } + return queue +end + +function QlessQueue:prefix(group) + if group then + return QlessQueue.ns..self.name..'-'..group + else + return QlessQueue.ns..self.name + end +end + +function QlessQueue:stats(now, date) + date = assert(tonumber(date), + 'Stats(): Arg "date" missing or not a number: '.. (date or 'nil')) + + local bin = date - (date % 86400) + + local histokeys = { + 's0','s1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11','s12','s13','s14','s15','s16','s17','s18','s19','s20','s21','s22','s23','s24','s25','s26','s27','s28','s29','s30','s31','s32','s33','s34','s35','s36','s37','s38','s39','s40','s41','s42','s43','s44','s45','s46','s47','s48','s49','s50','s51','s52','s53','s54','s55','s56','s57','s58','s59', + 'm1','m2','m3','m4','m5','m6','m7','m8','m9','m10','m11','m12','m13','m14','m15','m16','m17','m18','m19','m20','m21','m22','m23','m24','m25','m26','m27','m28','m29','m30','m31','m32','m33','m34','m35','m36','m37','m38','m39','m40','m41','m42','m43','m44','m45','m46','m47','m48','m49','m50','m51','m52','m53','m54','m55','m56','m57','m58','m59', + 'h1','h2','h3','h4','h5','h6','h7','h8','h9','h10','h11','h12','h13','h14','h15','h16','h17','h18','h19','h20','h21','h22','h23', + 'd1','d2','d3','d4','d5','d6' + } + + local mkstats = function(name, bin, queue) + local results = {} + + local key = 'ql:s:' .. name .. ':' .. bin .. ':' .. queue + local count, mean, vk = unpack(redis.call('hmget', key, 'total', 'mean', 'vk')) + + count = tonumber(count) or 0 + mean = tonumber(mean) or 0 + vk = tonumber(vk) + + results.count = count or 0 + results.mean = mean or 0 + results.histogram = {} + + if not count then + results.std = 0 + else + if count > 1 then + results.std = math.sqrt(vk / (count - 1)) + else + results.std = 0 + end + end + + local histogram = redis.call('hmget', key, unpack(histokeys)) + for i=1,#histokeys do + table.insert(results.histogram, tonumber(histogram[i]) or 0) + end + return results + end + + local retries, failed, failures = unpack(redis.call('hmget', 'ql:s:stats:' .. bin .. ':' .. self.name, 'retries', 'failed', 'failures')) + return { + retries = tonumber(retries or 0), + failed = tonumber(failed or 0), + failures = tonumber(failures or 0), + wait = mkstats('wait', bin, self.name), + run = mkstats('run' , bin, self.name) + } +end + +function QlessQueue:peek(now, count) + count = assert(tonumber(count), + 'Peek(): Arg "count" missing or not a number: ' .. tostring(count)) + + local jids = self.locks.expired(now, 0, count) + + self:check_recurring(now, count - #jids) + + self:check_scheduled(now, count - #jids) + + table.extend(jids, self.work.peek(count - #jids)) + + return jids +end + +function QlessQueue:paused() + return redis.call('sismember', 'ql:paused_queues', self.name) == 1 +end + +function QlessQueue.pause(now, ...) + redis.call('sadd', 'ql:paused_queues', unpack(arg)) +end + +function QlessQueue.unpause(...) + redis.call('srem', 'ql:paused_queues', unpack(arg)) +end + +function QlessQueue:pop(now, worker, count) + assert(worker, 'Pop(): Arg "worker" missing') + count = assert(tonumber(count), + 'Pop(): Arg "count" missing or not a number: ' .. tostring(count)) + + local expires = now + tonumber( + Qless.config.get(self.name .. '-heartbeat') or + Qless.config.get('heartbeat', 60)) + + if self:paused() then + return {} + end + + redis.call('zadd', 'ql:workers', now, worker) + + local max_concurrency = tonumber( + Qless.config.get(self.name .. '-max-concurrency', 0)) + + if max_concurrency > 0 then + local allowed = math.max(0, max_concurrency - self.locks.running(now)) + count = math.min(allowed, count) + if count == 0 then + return {} + end + end + + local jids = self:invalidate_locks(now, count) + + self:check_recurring(now, count - #jids) + + self:check_scheduled(now, count - #jids) + + table.extend(jids, self.work.peek(count - #jids)) + + local state + for index, jid in ipairs(jids) do + local job = Qless.job(jid) + state = unpack(job:data('state')) + job:history(now, 'popped', {worker = worker}) + + local time = tonumber( + redis.call('hget', QlessJob.ns .. jid, 'time') or now) + local waiting = now - time + self:stat(now, 'wait', waiting) + redis.call('hset', QlessJob.ns .. jid, + 'time', string.format("%.20f", now)) + + redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, jid) + + job:update({ + worker = worker, + expires = expires, + state = 'running' + }) + + self.locks.add(expires, jid) + + local tracked = redis.call('zscore', 'ql:tracked', jid) ~= false + if tracked then + Qless.publish('popped', jid) + end + end + + self.work.remove(unpack(jids)) + + return jids +end + +function QlessQueue:stat(now, stat, val) + local bin = now - (now % 86400) + local key = 'ql:s:' .. stat .. ':' .. bin .. ':' .. self.name + + local count, mean, vk = unpack( + redis.call('hmget', key, 'total', 'mean', 'vk')) + + count = count or 0 + if count == 0 then + mean = val + vk = 0 + count = 1 + else + count = count + 1 + local oldmean = mean + mean = mean + (val - mean) / count + vk = vk + (val - mean) * (val - oldmean) + end + + val = math.floor(val) + if val < 60 then -- seconds + redis.call('hincrby', key, 's' .. val, 1) + elseif val < 3600 then -- minutes + redis.call('hincrby', key, 'm' .. math.floor(val / 60), 1) + elseif val < 86400 then -- hours + redis.call('hincrby', key, 'h' .. math.floor(val / 3600), 1) + else -- days + redis.call('hincrby', key, 'd' .. math.floor(val / 86400), 1) + end + redis.call('hmset', key, 'total', count, 'mean', mean, 'vk', vk) +end + +function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...) + assert(jid , 'Put(): Arg "jid" missing') + assert(klass, 'Put(): Arg "klass" missing') + local data = assert(cjson.decode(raw_data), + 'Put(): Arg "data" missing or not JSON: ' .. tostring(raw_data)) + delay = assert(tonumber(delay), + 'Put(): Arg "delay" not a number: ' .. tostring(delay)) + + if #arg % 2 == 1 then + error('Odd number of additional args: ' .. tostring(arg)) + end + local options = {} + for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end + + local job = Qless.job(jid) + local priority, tags, oldqueue, state, failure, retries, oldworker, interval, next_run, old_resources = + unpack(redis.call('hmget', QlessJob.ns .. jid, 'priority', 'tags', + 'queue', 'state', 'failure', 'retries', 'worker', 'throttle_interval', 'throttle_next_run', 'resources')) + + next_run = next_run or now + + local replace = assert(tonumber(options['replace'] or 1) , + 'Put(): Arg "replace" not a number: ' .. tostring(options['replace'])) + + if replace == 0 and state == 'running' then + local time_left = self.locks.job_time_left(now, jid) + if time_left > 0 then + return time_left + end + end + + if tags then + Qless.tag(now, 'remove', jid, unpack(cjson.decode(tags))) + end + + retries = assert(tonumber(options['retries'] or retries or 5) , + 'Put(): Arg "retries" not a number: ' .. tostring(options['retries'])) + tags = assert(cjson.decode(options['tags'] or tags or '[]' ), + 'Put(): Arg "tags" not JSON' .. tostring(options['tags'])) + priority = assert(tonumber(options['priority'] or priority or 0), + 'Put(): Arg "priority" not a number' .. tostring(options['priority'])) + local depends = assert(cjson.decode(options['depends'] or '[]') , + 'Put(): Arg "depends" not JSON: ' .. tostring(options['depends'])) + + local resources = assert(cjson.decode(options['resources'] or '[]'), + 'Put(): Arg "resources" not JSON array: ' .. tostring(options['resources'])) + assert(#resources == 0 or QlessResource.all_exist(resources), 'Put(): invalid resources requested') + + if old_resources then + old_resources = Set.new(cjson.decode(old_resources)) + local removed_resources = Set.diff(old_resources, Set.new(resources)) + for k in pairs(removed_resources) do + Qless.resource(k):release(now, jid) + end + end + + local interval = assert(tonumber(options['interval'] or interval or 0), + 'Put(): Arg "interval" not a number: ' .. tostring(options['interval'])) + + if interval > 0 then + local minimum_delay = next_run - now + if minimum_delay < 0 then + minimum_delay = 0 + elseif minimum_delay > delay then + delay = minimum_delay + end + else + next_run = 0 + end + + if #depends > 0 then + local new = {} + for _, d in ipairs(depends) do new[d] = 1 end + + local original = redis.call( + 'smembers', QlessJob.ns .. jid .. '-dependencies') + for _, dep in pairs(original) do + if new[dep] == nil then + redis.call('srem', QlessJob.ns .. dep .. '-dependents' , jid) + redis.call('srem', QlessJob.ns .. jid .. '-dependencies', dep) + end + end + end + + Qless.publish('log', cjson.encode({ + jid = jid, + event = 'put', + queue = self.name + })) + + job:history(now, 'put', {q = self.name}) + + if oldqueue then + local queue_obj = Qless.queue(oldqueue) + queue_obj.work.remove(jid) + queue_obj.locks.remove(jid) + queue_obj.depends.remove(jid) + queue_obj.scheduled.remove(jid) + end + + if oldworker and oldworker ~= '' then + redis.call('zrem', 'ql:w:' .. oldworker .. ':jobs', jid) + if oldworker ~= worker then + local encoded = cjson.encode({ + jid = jid, + event = 'lock_lost', + worker = oldworker + }) + Qless.publish('w:' .. oldworker, encoded) + Qless.publish('log', encoded) + end + end + + if state == 'complete' then + redis.call('zrem', 'ql:completed', jid) + end + + for i, tag in ipairs(tags) do + redis.call('zadd', 'ql:t:' .. tag, now, jid) + redis.call('zincrby', 'ql:tags', 1, tag) + end + + if state == 'failed' then + failure = cjson.decode(failure) + redis.call('lrem', 'ql:f:' .. failure.group, 0, jid) + if redis.call('llen', 'ql:f:' .. failure.group) == 0 then + redis.call('srem', 'ql:failures', failure.group) + end + local bin = failure.when - (failure.when % 86400) + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. self.name, 'failed' , -1) + end + + redis.call('hmset', QlessJob.ns .. jid, + 'jid' , jid, + 'klass' , klass, + 'data' , raw_data, + 'priority' , priority, + 'tags' , cjson.encode(tags), + 'resources', cjson.encode(resources), + 'state' , ((delay > 0) and 'scheduled') or 'waiting', + 'worker' , '', + 'expires' , 0, + 'queue' , self.name, + 'retries' , retries, + 'remaining', retries, + 'throttle_interval', interval, + 'throttle_next_run', next_run, + 'time' , string.format("%.20f", now), + 'result_data', '{}') + + for i, j in ipairs(depends) do + local state = redis.call('hget', QlessJob.ns .. j, 'state') + if (state and state ~= 'complete') then + redis.call('sadd', QlessJob.ns .. j .. '-dependents' , jid) + redis.call('sadd', QlessJob.ns .. jid .. '-dependencies', j) + end + end + + if delay > 0 then + if redis.call('scard', QlessJob.ns .. jid .. '-dependencies') > 0 then + self.depends.add(now, jid) + redis.call('hmset', QlessJob.ns .. jid, + 'state', 'depends', + 'scheduled', now + delay) + else + self.scheduled.add(now + delay, jid) + end + else + if redis.call('scard', QlessJob.ns .. jid .. '-dependencies') > 0 then + self.depends.add(now, jid) + redis.call('hset', QlessJob.ns .. jid, 'state', 'depends') + elseif #resources > 0 then + if Qless.job(jid):acquire_resources(now) then + self.work.add(now, priority, jid) + end + else + self.work.add(now, priority, jid) + end + end + + if redis.call('zscore', 'ql:queues', self.name) == false then + redis.call('zadd', 'ql:queues', now, self.name) + end + + if redis.call('zscore', 'ql:tracked', jid) ~= false then + Qless.publish('put', jid) + end + + return jid +end + +function QlessQueue:unfail(now, group, count) + assert(group, 'Unfail(): Arg "group" missing') + count = assert(tonumber(count or 25), + 'Unfail(): Arg "count" not a number: ' .. tostring(count)) + + local jids = redis.call('lrange', 'ql:f:' .. group, -count, -1) + + local toinsert = {} + for index, jid in ipairs(jids) do + local job = Qless.job(jid) + local data = job:data() + job:history(now, 'put', {q = self.name}) + redis.call('hmset', QlessJob.ns .. data.jid, + 'state' , 'waiting', + 'worker' , '', + 'expires' , 0, + 'queue' , self.name, + 'remaining', data.retries or 5) + + if #data['resources'] then + if job:acquire_resources(now) then + self.work.add(now, data.priority, data.jid) + end + else + self.work.add(now, data.priority, data.jid) + end + end + + redis.call('ltrim', 'ql:f:' .. group, 0, -count - 1) + if (redis.call('llen', 'ql:f:' .. group) == 0) then + redis.call('srem', 'ql:failures', group) + end + + return #jids +end + +function QlessQueue:recur(now, jid, klass, raw_data, spec, ...) + assert(jid , 'RecurringJob On(): Arg "jid" missing') + assert(klass, 'RecurringJob On(): Arg "klass" missing') + assert(spec , 'RecurringJob On(): Arg "spec" missing') + local data = assert(cjson.decode(raw_data), + 'RecurringJob On(): Arg "data" not JSON: ' .. tostring(raw_data)) + + if spec == 'interval' then + local interval = assert(tonumber(arg[1]), + 'Recur(): Arg "interval" not a number: ' .. tostring(arg[1])) + local offset = assert(tonumber(arg[2]), + 'Recur(): Arg "offset" not a number: ' .. tostring(arg[2])) + if interval <= 0 then + error('Recur(): Arg "interval" must be greater than or equal to 0') + end + + if #arg % 2 == 1 then + error('Odd number of additional args: ' .. tostring(arg)) + end + + local options = {} + for i = 3, #arg, 2 do options[arg[i]] = arg[i + 1] end + options.tags = assert(cjson.decode(options.tags or '{}'), + 'Recur(): Arg "tags" must be JSON string array: ' .. tostring( + options.tags)) + options.priority = assert(tonumber(options.priority or 0), + 'Recur(): Arg "priority" not a number: ' .. tostring( + options.priority)) + options.retries = assert(tonumber(options.retries or 0), + 'Recur(): Arg "retries" not a number: ' .. tostring( + options.retries)) + options.backlog = assert(tonumber(options.backlog or 0), + 'Recur(): Arg "backlog" not a number: ' .. tostring( + options.backlog)) + options.resources = assert(cjson.decode(options['resources'] or '[]'), + 'Recur(): Arg "resources" not JSON array: ' .. tostring(options['resources'])) + + local count, old_queue = unpack(redis.call('hmget', 'ql:r:' .. jid, 'count', 'queue')) + count = count or 0 + + if old_queue then + Qless.queue(old_queue).recurring.remove(jid) + end + + redis.call('hmset', 'ql:r:' .. jid, + 'jid' , jid, + 'klass' , klass, + 'data' , raw_data, + 'priority' , options.priority, + 'tags' , cjson.encode(options.tags or {}), + 'state' , 'recur', + 'queue' , self.name, + 'type' , 'interval', + 'count' , count, + 'interval' , interval, + 'retries' , options.retries, + 'backlog' , options.backlog, + 'resources', cjson.encode(options.resources)) + self.recurring.add(now + offset, jid) + + if redis.call('zscore', 'ql:queues', self.name) == false then + redis.call('zadd', 'ql:queues', now, self.name) + end + + return jid + else + error('Recur(): schedule type "' .. tostring(spec) .. '" unknown') + end +end + +function QlessQueue:length() + return self.locks.length() + self.work.length() + self.scheduled.length() +end + +function QlessQueue:check_recurring(now, count) + local moved = 0 + local r = self.recurring.peek(now, 0, count) + for index, jid in ipairs(r) do + local klass, data, priority, tags, retries, interval, backlog, resources = unpack( + redis.call('hmget', 'ql:r:' .. jid, 'klass', 'data', 'priority', + 'tags', 'retries', 'interval', 'backlog', 'resources')) + local _tags = cjson.decode(tags) + local resources = cjson.decode(resources or '[]') + local score = math.floor(tonumber(self.recurring.score(jid))) + interval = tonumber(interval) + + backlog = tonumber(backlog or 0) + if backlog ~= 0 then + local num = ((now - score) / interval) + if num > backlog then + score = score + ( + math.ceil(num - backlog) * interval + ) + end + end + + while (score <= now) and (moved < count) do + local count = redis.call('hincrby', 'ql:r:' .. jid, 'count', 1) + moved = moved + 1 + + local child_jid = jid .. '-' .. count + + for i, tag in ipairs(_tags) do + redis.call('zadd', 'ql:t:' .. tag, now, child_jid) + redis.call('zincrby', 'ql:tags', 1, tag) + end + + redis.call('hmset', QlessJob.ns .. child_jid, + 'jid' , child_jid, + 'klass' , klass, + 'data' , data, + 'priority' , priority, + 'tags' , tags, + 'state' , 'waiting', + 'worker' , '', + 'expires' , 0, + 'queue' , self.name, + 'retries' , retries, + 'remaining' , retries, + 'resources' , cjson.encode(resources), + 'throttle_interval', 0, + 'time' , string.format("%.20f", score)) + + local job = Qless.job(child_jid) + job:history(score, 'put', {q = self.name}) + + + local add_job = true + if #resources then + add_job = job:acquire_resources(score) + end + if add_job then + self.work.add(score, priority, child_jid) + end + + score = score + interval + self.recurring.add(score, jid) + end + end +end + +function QlessQueue:check_scheduled(now, count) + local scheduled = self.scheduled.ready(now, 0, count) + for index, jid in ipairs(scheduled) do + local priority = tonumber( + redis.call('hget', QlessJob.ns .. jid, 'priority') or 0) + if Qless.job(jid):acquire_resources(now) then + self.work.add(now, priority, jid) + end + self.scheduled.remove(jid) + + redis.call('hset', QlessJob.ns .. jid, 'state', 'waiting') + end +end + +function QlessQueue:invalidate_locks(now, count) + local jids = {} + for index, jid in ipairs(self.locks.expired(now, 0, count)) do + local worker, failure = unpack( + redis.call('hmget', QlessJob.ns .. jid, 'worker', 'failure')) + redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid) + + local grace_period = tonumber(Qless.config.get('grace-period')) + + local courtesy_sent = tonumber( + redis.call('hget', QlessJob.ns .. jid, 'grace') or 0) + + local send_message = (courtesy_sent ~= 1) + local invalidate = not send_message + + if grace_period <= 0 then + send_message = true + invalidate = true + end + + if send_message then + if redis.call('zscore', 'ql:tracked', jid) ~= false then + Qless.publish('stalled', jid) + end + Qless.job(jid):history(now, 'timed-out') + redis.call('hset', QlessJob.ns .. jid, 'grace', 1) + + local encoded = cjson.encode({ + jid = jid, + event = 'lock_lost', + worker = worker + }) + Qless.publish('w:' .. worker, encoded) + Qless.publish('log', encoded) + self.locks.add(now + grace_period, jid) + + local bin = now - (now % 86400) + redis.call('hincrby', + 'ql:s:stats:' .. bin .. ':' .. self.name, 'retries', 1) + end + + if invalidate then + redis.call('hdel', QlessJob.ns .. jid, 'grace', 0) + + local remaining = tonumber(redis.call( + 'hincrby', QlessJob.ns .. jid, 'remaining', -1)) + + if remaining < 0 then + Qless.job(jid):release_resources(now) + + self.work.remove(jid) + self.locks.remove(jid) + self.scheduled.remove(jid) + + local group = 'failed-retries-' .. Qless.job(jid):data()['queue'] + local job = Qless.job(jid) + job:history(now, 'failed', {group = group}) + redis.call('hmset', QlessJob.ns .. jid, 'state', 'failed', + 'worker', '', + 'expires', '') + redis.call('hset', QlessJob.ns .. jid, + 'failure', cjson.encode({ + ['group'] = group, + ['message'] = + 'Job exhausted retries in queue "' .. self.name .. '"', + ['when'] = now, + ['worker'] = unpack(job:data('worker')) + })) + + redis.call('sadd', 'ql:failures', group) + redis.call('lpush', 'ql:f:' .. group, jid) + + if redis.call('zscore', 'ql:tracked', jid) ~= false then + Qless.publish('failed', jid) + end + Qless.publish('log', cjson.encode({ + jid = jid, + event = 'failed', + group = group, + worker = worker, + message = + 'Job exhausted retries in queue "' .. self.name .. '"' + })) + + local bin = now - (now % 86400) + redis.call('hincrby', + 'ql:s:stats:' .. bin .. ':' .. self.name, 'failures', 1) + redis.call('hincrby', + 'ql:s:stats:' .. bin .. ':' .. self.name, 'failed' , 1) + else + table.insert(jids, jid) + end + end + end + + return jids +end + +function QlessQueue.deregister(...) + redis.call('zrem', Qless.ns .. 'queues', unpack(arg)) +end + +function QlessQueue.counts(now, name) + if name then + local queue = Qless.queue(name) + local stalled = queue.locks.length(now) + queue:check_scheduled(now, queue.scheduled.length()) + return { + name = name, + waiting = queue.work.length(), + stalled = stalled, + running = queue.locks.length() - stalled, + scheduled = queue.scheduled.length(), + depends = queue.depends.length(), + recurring = queue.recurring.length(), + paused = queue:paused() + } + else + local queues = redis.call('zrange', 'ql:queues', 0, -1) + local response = {} + for index, qname in ipairs(queues) do + table.insert(response, QlessQueue.counts(now, qname)) + end + return response + end +end +function QlessRecurringJob:data() + local job = redis.call( + 'hmget', 'ql:r:' .. self.jid, 'jid', 'klass', 'state', 'queue', + 'priority', 'interval', 'retries', 'count', 'data', 'tags', 'backlog') + + if not job[1] then + return nil + end + + return { + jid = job[1], + klass = job[2], + state = job[3], + queue = job[4], + priority = tonumber(job[5]), + interval = tonumber(job[6]), + retries = tonumber(job[7]), + count = tonumber(job[8]), + data = job[9], + tags = cjson.decode(job[10]), + backlog = tonumber(job[11] or 0) + } +end + +function QlessRecurringJob:update(now, ...) + local options = {} + if redis.call('exists', 'ql:r:' .. self.jid) ~= 0 then + for i = 1, #arg, 2 do + local key = arg[i] + local value = arg[i+1] + assert(value, 'No value provided for ' .. tostring(key)) + if key == 'priority' or key == 'interval' or key == 'retries' then + value = assert(tonumber(value), 'Recur(): Arg "' .. key .. '" must be a number: ' .. tostring(value)) + if key == 'interval' then + local queue, interval = unpack(redis.call('hmget', 'ql:r:' .. self.jid, 'queue', 'interval')) + Qless.queue(queue).recurring.update( + value - tonumber(interval), self.jid) + end + redis.call('hset', 'ql:r:' .. self.jid, key, value) + elseif key == 'data' then + assert(cjson.decode(value), 'Recur(): Arg "data" is not JSON-encoded: ' .. tostring(value)) + redis.call('hset', 'ql:r:' .. self.jid, 'data', value) + elseif key == 'klass' then + redis.call('hset', 'ql:r:' .. self.jid, 'klass', value) + elseif key == 'queue' then + local queue_obj = Qless.queue( + redis.call('hget', 'ql:r:' .. self.jid, 'queue')) + local score = queue_obj.recurring.score(self.jid) + queue_obj.recurring.remove(self.jid) + Qless.queue(value).recurring.add(score, self.jid) + redis.call('hset', 'ql:r:' .. self.jid, 'queue', value) + if redis.call('zscore', 'ql:queues', value) == false then + redis.call('zadd', 'ql:queues', now, value) + end + elseif key == 'backlog' then + value = assert(tonumber(value), + 'Recur(): Arg "backlog" not a number: ' .. tostring(value)) + redis.call('hset', 'ql:r:' .. self.jid, 'backlog', value) + else + error('Recur(): Unrecognized option "' .. key .. '"') + end + end + return true + else + error('Recur(): No recurring job ' .. self.jid) + end +end + +function QlessRecurringJob:tag(...) + local tags = redis.call('hget', 'ql:r:' .. self.jid, 'tags') + if tags then + tags = cjson.decode(tags) + local _tags = {} + for i,v in ipairs(tags) do _tags[v] = true end + + for i=1,#arg do if _tags[arg[i]] == nil then table.insert(tags, arg[i]) end end + + tags = cjson.encode(tags) + redis.call('hset', 'ql:r:' .. self.jid, 'tags', tags) + return tags + else + error('Tag(): Job ' .. self.jid .. ' does not exist') + end +end + +function QlessRecurringJob:untag(...) + local tags = redis.call('hget', 'ql:r:' .. self.jid, 'tags') + if tags then + tags = cjson.decode(tags) + local _tags = {} + for i,v in ipairs(tags) do _tags[v] = true end + for i = 1,#arg do _tags[arg[i]] = nil end + local results = {} + for i, tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end + tags = cjson.encode(results) + redis.call('hset', 'ql:r:' .. self.jid, 'tags', tags) + return tags + else + error('Untag(): Job ' .. self.jid .. ' does not exist') + end +end + +function QlessRecurringJob:unrecur() + local queue = redis.call('hget', 'ql:r:' .. self.jid, 'queue') + if queue then + Qless.queue(queue).recurring.remove(self.jid) + redis.call('del', 'ql:r:' .. self.jid) + return true + else + return true + end +end +function QlessWorker.deregister(...) + redis.call('zrem', 'ql:workers', unpack(arg)) +end + +function QlessWorker.counts(now, worker) + local interval = tonumber(Qless.config.get('max-worker-age', 86400)) + + local workers = redis.call('zrangebyscore', 'ql:workers', 0, now - interval) + for index, worker in ipairs(workers) do + redis.call('del', 'ql:w:' .. worker .. ':jobs') + end + + redis.call('zremrangebyscore', 'ql:workers', 0, now - interval) + + if worker then + return { + jobs = redis.call('zrevrangebyscore', 'ql:w:' .. worker .. ':jobs', now + 8640000, now), + stalled = redis.call('zrevrangebyscore', 'ql:w:' .. worker .. ':jobs', now, 0) + } + else + local response = {} + local workers = redis.call('zrevrange', 'ql:workers', 0, -1) + for index, worker in ipairs(workers) do + table.insert(response, { + name = worker, + jobs = redis.call('zcount', 'ql:w:' .. worker .. ':jobs', now, now + 8640000), + stalled = redis.call('zcount', 'ql:w:' .. worker .. ':jobs', 0, now) + }) + end + return response + end +end + +function QlessResource:data(...) + local res = redis.call( + 'hmget', QlessResource.ns .. self.rid, 'rid', 'max') + + if not res[1] then + return nil + end + + local data = { + rid = res[1], + max = tonumber(res[2] or 0), + pending = self:pending(), + locks = self:locks(), + } + + return data +end + +function QlessResource:get(...) + local res = redis.call( + 'hmget', QlessResource.ns .. self.rid, 'rid', 'max') + + if not res[1] then + return nil + end + + return tonumber(res[2] or 0) +end + +function QlessResource:set(now, max) + local max = assert(tonumber(max), 'Set(): Arg "max" not a number: ' .. tostring(max)) + + local current_max = self:get() + if current_max == nil then + current_max = max + end + + local keyLocks = self:prefix('locks') + local current_locks = redis.pcall('scard', keyLocks) + local confirm_limit = math.max(current_max,current_locks) + local max_change = max - confirm_limit + local keyPending = self:prefix('pending') + + redis.call('hmset', QlessResource.ns .. self.rid, 'rid', self.rid, 'max', max); + + if max_change > 0 then + local jids = redis.call('zrevrange', keyPending, 0, max_change - 1, 'withscores') + local jid_count = #jids + if jid_count == 0 then + return self.rid + end + + for i = 1, jid_count, 2 do + + local newJid = jids[i] + local score = jids[i + 1] + + if Qless.job(newJid):acquire_resources(now) then + local data = Qless.job(newJid):data() + local queue = Qless.queue(data['queue']) + queue.work.add(score, 0, newJid) + end + end + end + + return self.rid +end + +function QlessResource:unset() + return redis.call('del', QlessResource.ns .. self.rid); +end + +function QlessResource:prefix(group) + if group then + return QlessResource.ns..self.rid..'-'..group + end + + return QlessResource.ns..self.rid +end + +function QlessResource:acquire(now, priority, jid) + local keyLocks = self:prefix('locks') + local max = self:get() + if max == nil then + error({code=1, msg='Acquire(): resource ' .. self.rid .. ' does not exist'}) + end + + if type(jid) ~= 'string' then + error({code=2, msg='Acquire(): invalid jid; expected string, got \'' .. type(jid) .. '\''}) + end + + + if redis.call('sismember', self:prefix('locks'), jid) == 1 then + return true + end + + local remaining = max - redis.pcall('scard', keyLocks) + + if remaining > 0 then + redis.call('sadd', keyLocks, jid) + redis.call('zrem', self:prefix('pending'), jid) + + return true + end + + if redis.call('zscore', self:prefix('pending'), jid) == false then + redis.call('zadd', self:prefix('pending'), priority - (now / 10000000000), jid) + end + + return false +end + +function QlessResource:release(now, jid) + local keyLocks = self:prefix('locks') + local keyPending = self:prefix('pending') + + redis.call('srem', keyLocks, jid) + redis.call('zrem', keyPending, jid) + + local jids = redis.call('zrevrange', keyPending, 0, 0, 'withscores') + if #jids == 0 then + return false + end + + local newJid = jids[1] + local score = jids[2] + + if Qless.job(newJid):acquire_resources(now) then + local data = Qless.job(newJid):data() + local queue = Qless.queue(data['queue']) + queue.work.add(score, 0, newJid) + end + + return newJid +end + +function QlessResource:locks() + return redis.call('smembers', self:prefix('locks')) +end + +function QlessResource:lock_count() + return redis.call('scard', self:prefix('locks')) +end + +function QlessResource:pending() + return redis.call('zrevrange', self:prefix('pending'), 0, -1) +end + +function QlessResource:pending_count() + return redis.call('zcard', self:prefix('pending')) +end + +function QlessResource:exists() + return redis.call('exists', self:prefix()) == 1 +end + +function QlessResource.all_exist(resources) + for _, res in ipairs(resources) do + if redis.call('exists', QlessResource.ns .. res) == 0 then + return false + end + end + return true +end + +function QlessResource.pending_counts(now) + local search = QlessResource.ns..'*-pending' + local reply = redis.call('keys', search) + local response = {} + for index, rname in ipairs(reply) do + local count = redis.call('zcard', rname) + local resStat = {name = rname, count = count} + table.insert(response,resStat) + end + return response +end + +function QlessResource.locks_counts(now) + local search = QlessResource.ns..'*-locks' + local reply = redis.call('keys', search) + local response = {} + for index, rname in ipairs(reply) do + local count = redis.call('scard', rname) + local resStat = {name = rname, count = count} + table.insert(response,resStat) + end + return response +end------------------------------------------------------------------------------- +local QlessAPI = {} + +local function tonil(value) + if (value == '') then return nil else return value end +end + +function QlessAPI.get(now, jid) + local data = Qless.job(jid):data() + if not data then + return nil + end + return cjson.encode(data) +end + +function QlessAPI.multiget(now, ...) + local results = {} + for i, jid in ipairs(arg) do + table.insert(results, Qless.job(jid):data()) + end + return cjson.encode(results) +end + +QlessAPI['config.get'] = function(now, key) + key = tonil(key) + if not key then + return cjson.encode(Qless.config.get(key)) + else + return Qless.config.get(key) + end +end + +QlessAPI['config.set'] = function(now, key, value) + key = tonil(key) + return Qless.config.set(key, value) +end + +QlessAPI['config.unset'] = function(now, key) + key = tonil(key) + return Qless.config.unset(key) +end + +QlessAPI.queues = function(now, queue) + return cjson.encode(QlessQueue.counts(now, queue)) +end + +QlessAPI.complete = function(now, jid, worker, queue, data, ...) + data = tonil(data) + return Qless.job(jid):complete(now, worker, queue, data, unpack(arg)) +end + +QlessAPI.failed = function(now, group, start, limit) + return cjson.encode(Qless.failed(group, start, limit)) +end + +QlessAPI.fail = function(now, jid, worker, group, message, data) + data = tonil(data) + return Qless.job(jid):fail(now, worker, group, message, data) +end + +QlessAPI.jobs = function(now, state, ...) + return Qless.jobs(now, state, unpack(arg)) +end + +QlessAPI.retry = function(now, jid, queue, worker, delay, group, message) + return Qless.job(jid):retry(now, queue, worker, delay, group, message) +end + +QlessAPI.depends = function(now, jid, command, ...) + return Qless.job(jid):depends(now, command, unpack(arg)) +end + +QlessAPI.heartbeat = function(now, jid, worker, data) + data = tonil(data) + return Qless.job(jid):heartbeat(now, worker, data) +end + +QlessAPI.workers = function(now, worker) + return cjson.encode(QlessWorker.counts(now, worker)) +end + +QlessAPI.track = function(now, command, jid) + return cjson.encode(Qless.track(now, command, jid)) +end + +QlessAPI.tag = function(now, command, ...) + return cjson.encode(Qless.tag(now, command, unpack(arg))) +end + +QlessAPI.stats = function(now, queue, date) + return cjson.encode(Qless.queue(queue):stats(now, date)) +end + +QlessAPI.priority = function(now, jid, priority) + return Qless.job(jid):priority(priority) +end + +QlessAPI.log = function(now, jid, message, data) + assert(jid, "Log(): Argument 'jid' missing") + assert(message, "Log(): Argument 'message' missing") + if data then + data = assert(cjson.decode(data), + "Log(): Argument 'data' not cjson: " .. tostring(data)) + end + + local job = Qless.job(jid) + assert(job:exists(), 'Log(): Job ' .. jid .. ' does not exist') + job:history(now, message, data) +end + +QlessAPI.peek = function(now, queue, count) + local jids = Qless.queue(queue):peek(now, count) + local response = {} + for i, jid in ipairs(jids) do + table.insert(response, Qless.job(jid):data()) + end + return cjson.encode(response) +end + +QlessAPI.pop = function(now, queue, worker, count) + local jids = Qless.queue(queue):pop(now, worker, count) + local response = {} + for i, jid in ipairs(jids) do + table.insert(response, Qless.job(jid):data()) + end + return cjson.encode(response) +end + +QlessAPI.pause = function(now, ...) + return QlessQueue.pause(now, unpack(arg)) +end + +QlessAPI.unpause = function(now, ...) + return QlessQueue.unpause(unpack(arg)) +end + +QlessAPI.paused = function(now, queue) + return Qless.queue(queue):paused() +end + +QlessAPI.cancel = function(now, ...) + return Qless.cancel(now, unpack(arg)) +end + +QlessAPI.timeout = function(now, ...) + for _, jid in ipairs(arg) do + Qless.job(jid):timeout(now) + end +end + +QlessAPI.put = function(now, me, queue, jid, klass, data, delay, ...) + data = tonil(data) + return Qless.queue(queue):put(now, me, jid, klass, data, delay, unpack(arg)) +end + +QlessAPI.requeue = function(now, me, queue, jid, ...) + local job = Qless.job(jid) + assert(job:exists(), 'Requeue(): Job ' .. jid .. ' does not exist') + return QlessAPI.put(now, me, queue, jid, unpack(arg)) +end + +QlessAPI.unfail = function(now, queue, group, count) + return Qless.queue(queue):unfail(now, group, count) +end + +QlessAPI.recur = function(now, queue, jid, klass, data, spec, ...) + data = tonil(data) + return Qless.queue(queue):recur(now, jid, klass, data, spec, unpack(arg)) +end + +QlessAPI.unrecur = function(now, jid) + return Qless.recurring(jid):unrecur() +end + +QlessAPI['recur.get'] = function(now, jid) + local data = Qless.recurring(jid):data() + if not data then + return nil + end + return cjson.encode(data) +end + +QlessAPI['recur.update'] = function(now, jid, ...) + return Qless.recurring(jid):update(now, unpack(arg)) +end + +QlessAPI['recur.tag'] = function(now, jid, ...) + return Qless.recurring(jid):tag(unpack(arg)) +end + +QlessAPI['recur.untag'] = function(now, jid, ...) + return Qless.recurring(jid):untag(unpack(arg)) +end + +QlessAPI.length = function(now, queue) + return Qless.queue(queue):length() +end + +QlessAPI['worker.deregister'] = function(now, ...) + return QlessWorker.deregister(unpack(arg)) +end + +QlessAPI['queue.forget'] = function(now, ...) + QlessQueue.deregister(unpack(arg)) +end + +QlessAPI['resource.set'] = function(now, rid, max) + return Qless.resource(rid):set(now, max) +end + +QlessAPI['resource.get'] = function(now, rid) + return Qless.resource(rid):get() +end + +QlessAPI['resource.data'] = function(now, rid) + local data = Qless.resource(rid):data() + if not data then + return nil + end + + return cjson.encode(data) +end + +QlessAPI['resource.exists'] = function(now, rid) + return Qless.resource(rid):exists() +end + +QlessAPI['resource.unset'] = function(now, rid) + return Qless.resource(rid):unset() +end + +QlessAPI['resource.locks'] = function(now, rid) + local data = Qless.resource(rid):locks() + if not data then + return nil + end + + return cjson.encode(data) +end + +QlessAPI['resource.lock_count'] = function(now, rid) + return Qless.resource(rid):lock_count() +end + +QlessAPI['resource.pending'] = function(now, rid) + local data = Qless.resource(rid):pending() + if not data then + return nil + end + + return cjson.encode(data) + +end + +QlessAPI['resource.pending_count'] = function(now, rid) + return Qless.resource(rid):pending_count() +end + +QlessAPI['resource.stats_pending'] = function(now) + return cjson.encode(QlessResource.pending_counts(now)) +end + +QlessAPI['resource.stats_locks'] = function(now) + return cjson.encode(QlessResource.locks_counts(now)) +end + + +if #KEYS > 0 then error('No Keys should be provided') end + +local command_name = assert(table.remove(ARGV, 1), 'Must provide a command') +local command = assert( + QlessAPI[command_name], 'Unknown command ' .. command_name) + +local now = tonumber(table.remove(ARGV, 1)) +local now = assert( + now, 'Arg "now" missing or not a number: ' .. (now or 'nil')) + +return command(now, unpack(ARGV)) \ No newline at end of file diff --git a/qless_lua.go b/qless_lua.go new file mode 100644 index 0000000..ccbfd2e --- /dev/null +++ b/qless_lua.go @@ -0,0 +1,2520 @@ +package qless + +const qlessLua = `-- Current SHA: 9a235bcc4091f798e6a64503969900f1415df8a4 +-- This is a generated file +local Qless = { + ns = 'ql:' +} + +local QlessQueue = { + ns = Qless.ns .. 'q:' +} +QlessQueue.__index = QlessQueue + +local QlessWorker = { + ns = Qless.ns .. 'w:' +} +QlessWorker.__index = QlessWorker + +local QlessJob = { + ns = Qless.ns .. 'j:' +} +QlessJob.__index = QlessJob + +local QlessRecurringJob = {} +QlessRecurringJob.__index = QlessRecurringJob + +local QlessResource = { + ns = Qless.ns .. 'rs:' +} +QlessResource.__index = QlessResource; + +Qless.config = {} + +function table.extend(self, other) + for i, v in ipairs(other) do + table.insert(self, v) + end +end + +function Qless.publish(channel, message) + redis.call('publish', Qless.ns .. channel, message) +end + +function Qless.job(jid) + assert(jid, 'Job(): no jid provided') + local job = {} + setmetatable(job, QlessJob) + job.jid = jid + return job +end + +function Qless.recurring(jid) + assert(jid, 'Recurring(): no jid provided') + local job = {} + setmetatable(job, QlessRecurringJob) + job.jid = jid + return job +end + +function Qless.resource(rid) + assert(rid, 'Resource(): no rid provided') + local res = {} + setmetatable(res, QlessResource) + res.rid = rid + return res +end + +function Qless.failed(group, start, limit) + start = assert(tonumber(start or 0), + 'Failed(): Arg "start" is not a number: ' .. (start or 'nil')) + limit = assert(tonumber(limit or 25), + 'Failed(): Arg "limit" is not a number: ' .. (limit or 'nil')) + + if group then + return { + total = redis.call('llen', 'ql:f:' .. group), + jobs = redis.call('lrange', 'ql:f:' .. group, start, start + limit - 1) + } + else + local response = {} + local groups = redis.call('smembers', 'ql:failures') + for index, group in ipairs(groups) do + response[group] = redis.call('llen', 'ql:f:' .. group) + end + return response + end +end + +function Qless.jobs(now, state, ...) + assert(state, 'Jobs(): Arg "state" missing') + if state == 'complete' then + local offset = assert(tonumber(arg[1] or 0), + 'Jobs(): Arg "offset" not a number: ' .. tostring(arg[1])) + local count = assert(tonumber(arg[2] or 25), + 'Jobs(): Arg "count" not a number: ' .. tostring(arg[2])) + return redis.call('zrevrange', 'ql:completed', offset, + offset + count - 1) + else + local name = assert(arg[1], 'Jobs(): Arg "queue" missing') + local offset = assert(tonumber(arg[2] or 0), + 'Jobs(): Arg "offset" not a number: ' .. tostring(arg[2])) + local count = assert(tonumber(arg[3] or 25), + 'Jobs(): Arg "count" not a number: ' .. tostring(arg[3])) + + local queue = Qless.queue(name) + if state == 'running' then + return queue.locks.peek(now, offset, count) + elseif state == 'stalled' then + return queue.locks.expired(now, offset, count) + elseif state == 'scheduled' then + queue:check_scheduled(now, queue.scheduled.length()) + return queue.scheduled.peek(now, offset, count) + elseif state == 'depends' then + return queue.depends.peek(now, offset, count) + elseif state == 'recurring' then + return queue.recurring.peek(math.huge, offset, count) + else + error('Jobs(): Unknown type "' .. state .. '"') + end + end +end + +function Qless.track(now, command, jid) + if command ~= nil then + assert(jid, 'Track(): Arg "jid" missing') + assert(Qless.job(jid):exists(), 'Track(): Job does not exist') + if string.lower(command) == 'track' then + Qless.publish('track', jid) + return redis.call('zadd', 'ql:tracked', now, jid) + elseif string.lower(command) == 'untrack' then + Qless.publish('untrack', jid) + return redis.call('zrem', 'ql:tracked', jid) + else + error('Track(): Unknown action "' .. command .. '"') + end + else + local response = { + jobs = {}, + expired = {} + } + local jids = redis.call('zrange', 'ql:tracked', 0, -1) + for index, jid in ipairs(jids) do + local data = Qless.job(jid):data() + if data then + table.insert(response.jobs, data) + else + table.insert(response.expired, jid) + end + end + return response + end +end + +function Qless.tag(now, command, ...) + assert(command, + 'Tag(): Arg "command" must be "add", "remove", "get" or "top"') + + if command == 'add' then + local jid = assert(arg[1], 'Tag(): Arg "jid" missing') + local tags = redis.call('hget', QlessJob.ns .. jid, 'tags') + if tags then + tags = cjson.decode(tags) + local _tags = {} + for i,v in ipairs(tags) do _tags[v] = true end + + for i=2,#arg do + local tag = arg[i] + if _tags[tag] == nil then + _tags[tag] = true + table.insert(tags, tag) + end + redis.call('zadd', 'ql:t:' .. tag, now, jid) + redis.call('zincrby', 'ql:tags', 1, tag) + end + + tags = cjson.encode(tags) + redis.call('hset', QlessJob.ns .. jid, 'tags', tags) + return tags + else + error('Tag(): Job ' .. jid .. ' does not exist') + end + elseif command == 'remove' then + local jid = assert(arg[1], 'Tag(): Arg "jid" missing') + local tags = redis.call('hget', QlessJob.ns .. jid, 'tags') + if tags then + tags = cjson.decode(tags) + local _tags = {} + for i,v in ipairs(tags) do _tags[v] = true end + + for i=2,#arg do + local tag = arg[i] + _tags[tag] = nil + redis.call('zrem', 'ql:t:' .. tag, jid) + redis.call('zincrby', 'ql:tags', -1, tag) + end + + local results = {} + for i,tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end + + tags = cjson.encode(results) + redis.call('hset', QlessJob.ns .. jid, 'tags', tags) + return results + else + error('Tag(): Job ' .. jid .. ' does not exist') + end + elseif command == 'get' then + local tag = assert(arg[1], 'Tag(): Arg "tag" missing') + local offset = assert(tonumber(arg[2] or 0), + 'Tag(): Arg "offset" not a number: ' .. tostring(arg[2])) + local count = assert(tonumber(arg[3] or 25), + 'Tag(): Arg "count" not a number: ' .. tostring(arg[3])) + return { + total = redis.call('zcard', 'ql:t:' .. tag), + jobs = redis.call('zrange', 'ql:t:' .. tag, offset, offset + count - 1) + } + elseif command == 'top' then + local offset = assert(tonumber(arg[1] or 0) , 'Tag(): Arg "offset" not a number: ' .. tostring(arg[1])) + local count = assert(tonumber(arg[2] or 25), 'Tag(): Arg "count" not a number: ' .. tostring(arg[2])) + return redis.call('zrevrangebyscore', 'ql:tags', '+inf', 2, 'limit', offset, count) + else + error('Tag(): First argument must be "add", "remove" or "get"') + end +end + +function Qless.cancel(now, ...) + local dependents = {} + for _, jid in ipairs(arg) do + dependents[jid] = redis.call( + 'smembers', QlessJob.ns .. jid .. '-dependents') or {} + end + + for i, jid in ipairs(arg) do + for j, dep in ipairs(dependents[jid]) do + if dependents[dep] == nil then + error('Cancel(): ' .. jid .. ' is a dependency of ' .. dep .. + ' but is not mentioned to be canceled') + end + end + end + + local cancelled_jids = {} + + for _, jid in ipairs(arg) do + local real_jid, state, queue, failure, worker = unpack(redis.call( + 'hmget', QlessJob.ns .. jid, 'jid', 'state', 'queue', 'failure', 'worker')) + + if state ~= false and state ~= 'complete' then + table.insert(cancelled_jids, jid) + + local encoded = cjson.encode({ + jid = jid, + worker = worker, + event = 'canceled', + queue = queue + }) + Qless.publish('log', encoded) + + if worker and (worker ~= '') then + redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid) + Qless.publish('w:' .. worker, encoded) + end + + if queue then + local queue = Qless.queue(queue) + queue.work.remove(jid) + queue.locks.remove(jid) + queue.scheduled.remove(jid) + queue.depends.remove(jid) + end + + Qless.job(jid):release_resources(now) + + for i, j in ipairs(redis.call( + 'smembers', QlessJob.ns .. jid .. '-dependencies')) do + redis.call('srem', QlessJob.ns .. j .. '-dependents', jid) + end + + redis.call('del', QlessJob.ns .. jid .. '-dependencies') + + if state == 'failed' then + failure = cjson.decode(failure) + redis.call('lrem', 'ql:f:' .. failure.group, 0, jid) + if redis.call('llen', 'ql:f:' .. failure.group) == 0 then + redis.call('srem', 'ql:failures', failure.group) + end + local bin = failure.when - (failure.when % 86400) + local failed = redis.call( + 'hget', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed') + redis.call('hset', + 'ql:s:stats:' .. bin .. ':' .. queue, 'failed', failed - 1) + end + + local tags = cjson.decode( + redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}') + for i, tag in ipairs(tags) do + redis.call('zrem', 'ql:t:' .. tag, jid) + redis.call('zincrby', 'ql:tags', -1, tag) + end + + if redis.call('zscore', 'ql:tracked', jid) ~= false then + Qless.publish('canceled', jid) + end + + redis.call('del', QlessJob.ns .. jid) + redis.call('del', QlessJob.ns .. jid .. '-history') + end + end + + return cancelled_jids +end + +local Set = {} + +function Set.new (t) + local set = {} + for _, l in ipairs(t) do set[l] = true end + return set +end + +function Set.union (a,b) + local res = Set.new{} + for k in pairs(a) do res[k] = true end + for k in pairs(b) do res[k] = true end + return res +end + +function Set.intersection (a,b) + local res = Set.new{} + for k in pairs(a) do + res[k] = b[k] + end + return res +end + +function Set.diff(a,b) + local res = Set.new{} + for k in pairs(a) do + if not b[k] then res[k] = true end + end + + return res +end + +Qless.config.defaults = { + ['application'] = 'qless', + ['heartbeat'] = 60, + ['grace-period'] = 10, + ['stats-history'] = 30, + ['histogram-history'] = 7, + ['jobs-history-count'] = 50000, + ['jobs-history'] = 604800 +} + +Qless.config.get = function(key, default) + if key then + return redis.call('hget', 'ql:config', key) or + Qless.config.defaults[key] or default + else + local reply = redis.call('hgetall', 'ql:config') + for i = 1, #reply, 2 do + Qless.config.defaults[reply[i]] = reply[i + 1] + end + return Qless.config.defaults + end +end + +Qless.config.set = function(option, value) + assert(option, 'config.set(): Arg "option" missing') + assert(value , 'config.set(): Arg "value" missing') + Qless.publish('log', cjson.encode({ + event = 'config_set', + option = option, + value = value + })) + + redis.call('hset', 'ql:config', option, value) +end + +Qless.config.unset = function(option) + assert(option, 'config.unset(): Arg "option" missing') + Qless.publish('log', cjson.encode({ + event = 'config_unset', + option = option + })) + + redis.call('hdel', 'ql:config', option) +end + +function QlessJob:data(...) + local job = redis.call( + 'hmget', QlessJob.ns .. self.jid, 'jid', 'klass', 'state', 'queue', + 'worker', 'priority', 'expires', 'retries', 'remaining', 'data', + 'tags', 'failure', 'resources', 'result_data', 'throttle_interval') + + if not job[1] then + return nil + end + + local data = { + jid = job[1], + klass = job[2], + state = job[3], + queue = job[4], + worker = job[5] or '', + tracked = redis.call( + 'zscore', 'ql:tracked', self.jid) ~= false, + priority = tonumber(job[6]), + expires = tonumber(job[7]) or 0, + retries = tonumber(job[8]), + remaining = math.floor(tonumber(job[9])), + data = job[10], + tags = cjson.decode(job[11]), + history = self:history(), + failure = cjson.decode(job[12] or '{}'), + resources = cjson.decode(job[13] or '[]'), + result_data = cjson.decode(job[14] or '{}'), + interval = tonumber(job[15]) or 0, + dependents = redis.call( + 'smembers', QlessJob.ns .. self.jid .. '-dependents'), + dependencies = redis.call( + 'smembers', QlessJob.ns .. self.jid .. '-dependencies') + } + + if #arg > 0 then + local response = {} + for index, key in ipairs(arg) do + table.insert(response, data[key]) + end + return response + else + return data + end +end + +function QlessJob:complete(now, worker, queue, data, ...) + assert(worker, 'Complete(): Arg "worker" missing') + assert(queue , 'Complete(): Arg "queue" missing') + data = assert(cjson.decode(data), + 'Complete(): Arg "data" missing or not JSON: ' .. tostring(data)) + + local options = {} + for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end + + local nextq = options['next'] + local result_data = options['result_data'] + local delay = assert(tonumber(options['delay'] or 0)) + local depends = assert(cjson.decode(options['depends'] or '[]'), + 'Complete(): Arg "depends" not JSON: ' .. tostring(options['depends'])) + + if options['delay'] and nextq == nil then + error('Complete(): "delay" cannot be used without a "next".') + end + + if options['depends'] and nextq == nil then + error('Complete(): "depends" cannot be used without a "next".') + end + + local bin = now - (now % 86400) + + local lastworker, state, priority, retries, interval = unpack( + redis.call('hmget', QlessJob.ns .. self.jid, 'worker', 'state', + 'priority', 'retries', 'throttle_interval')) + + if lastworker == false then + error('Complete(): Job does not exist') + elseif (state ~= 'running') then + error('Complete(): Job is not currently running: ' .. state) + elseif lastworker ~= worker then + error('Complete(): Job has been handed out to another worker: ' .. + tostring(lastworker)) + end + + local next_run = 0 + if interval then + interval = tonumber(interval) + if interval > 0 then + next_run = now + interval + else + next_run = -1 + end + end + + self:history(now, 'done') + + if data then + redis.call('hset', QlessJob.ns .. self.jid, 'data', cjson.encode(data)) + end + + if result_data then + redis.call('hset', QlessJob.ns .. self.jid, 'result_data', result_data) + end + + local queue_obj = Qless.queue(queue) + queue_obj.work.remove(self.jid) + queue_obj.locks.remove(self.jid) + queue_obj.scheduled.remove(self.jid) + + self:release_resources(now) + + local time = tonumber( + redis.call('hget', QlessJob.ns .. self.jid, 'time') or now) + local waiting = now - time + Qless.queue(queue):stat(now, 'run', waiting) + redis.call('hset', QlessJob.ns .. self.jid, + 'time', string.format("%.20f", now)) + + redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid) + + if redis.call('zscore', 'ql:tracked', self.jid) ~= false then + Qless.publish('completed', self.jid) + end + + if nextq then + queue_obj = Qless.queue(nextq) + Qless.publish('log', cjson.encode({ + jid = self.jid, + event = 'advanced', + queue = queue, + to = nextq + })) + + self:history(now, 'put', {q = nextq}) + + if redis.call('zscore', 'ql:queues', nextq) == false then + redis.call('zadd', 'ql:queues', now, nextq) + end + + redis.call('hmset', QlessJob.ns .. self.jid, + 'state', 'waiting', + 'worker', '', + 'failure', '{}', + 'queue', nextq, + 'expires', 0, + 'remaining', tonumber(retries)) + + if (delay > 0) and (#depends == 0) then + queue_obj.scheduled.add(now + delay, self.jid) + return 'scheduled' + else + local count = 0 + for i, j in ipairs(depends) do + local state = redis.call('hget', QlessJob.ns .. j, 'state') + if (state and state ~= 'complete') then + count = count + 1 + redis.call( + 'sadd', QlessJob.ns .. j .. '-dependents',self.jid) + redis.call( + 'sadd', QlessJob.ns .. self.jid .. '-dependencies', j) + end + end + if count > 0 then + queue_obj.depends.add(now, self.jid) + redis.call('hset', QlessJob.ns .. self.jid, 'state', 'depends') + if delay > 0 then + queue_obj.depends.add(now, self.jid) + redis.call('hset', QlessJob.ns .. self.jid, 'scheduled', now + delay) + end + return 'depends' + else + if self:acquire_resources(now) then + queue_obj.work.add(now, priority, self.jid) + end + return 'waiting' + end + end + else + Qless.publish('log', cjson.encode({ + jid = self.jid, + event = 'completed', + queue = queue + })) + + redis.call('hmset', QlessJob.ns .. self.jid, + 'state', 'complete', + 'worker', '', + 'failure', '{}', + 'queue', '', + 'expires', 0, + 'remaining', tonumber(retries), + 'throttle_next_run', next_run + ) + + local count = Qless.config.get('jobs-history-count') + local time = Qless.config.get('jobs-history') + + count = tonumber(count or 50000) + time = tonumber(time or 7 * 24 * 60 * 60) + + redis.call('zadd', 'ql:completed', now, self.jid) + + local jids = redis.call('zrangebyscore', 'ql:completed', 0, now - time) + for index, jid in ipairs(jids) do + local tags = cjson.decode( + redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}') + for i, tag in ipairs(tags) do + redis.call('zrem', 'ql:t:' .. tag, jid) + redis.call('zincrby', 'ql:tags', -1, tag) + end + redis.call('del', QlessJob.ns .. jid) + redis.call('del', QlessJob.ns .. jid .. '-history') + end + redis.call('zremrangebyscore', 'ql:completed', 0, now - time) + + jids = redis.call('zrange', 'ql:completed', 0, (-1-count)) + for index, jid in ipairs(jids) do + local tags = cjson.decode( + redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}') + for i, tag in ipairs(tags) do + redis.call('zrem', 'ql:t:' .. tag, jid) + redis.call('zincrby', 'ql:tags', -1, tag) + end + redis.call('del', QlessJob.ns .. jid) + redis.call('del', QlessJob.ns .. jid .. '-history') + end + redis.call('zremrangebyrank', 'ql:completed', 0, (-1-count)) + + for i, j in ipairs(redis.call( + 'smembers', QlessJob.ns .. self.jid .. '-dependents')) do + redis.call('srem', QlessJob.ns .. j .. '-dependencies', self.jid) + if redis.call( + 'scard', QlessJob.ns .. j .. '-dependencies') == 0 then + local q, p, scheduled = unpack( + redis.call('hmget', QlessJob.ns .. j, 'queue', 'priority', 'scheduled')) + if q then + local queue = Qless.queue(q) + queue.depends.remove(j) + if scheduled then + queue.scheduled.add(scheduled, j) + redis.call('hset', QlessJob.ns .. j, 'state', 'scheduled') + redis.call('hdel', QlessJob.ns .. j, 'scheduled') + else + if Qless.job(j):acquire_resources(now) then + queue.work.add(now, p, j) + end + redis.call('hset', QlessJob.ns .. j, 'state', 'waiting') + end + end + end + end + + redis.call('del', QlessJob.ns .. self.jid .. '-dependents') + + return 'complete' + end +end + +function QlessJob:fail(now, worker, group, message, data) + local worker = assert(worker , 'Fail(): Arg "worker" missing') + local group = assert(group , 'Fail(): Arg "group" missing') + local message = assert(message , 'Fail(): Arg "message" missing') + + local bin = now - (now % 86400) + + if data then + data = cjson.decode(data) + end + + local queue, state, oldworker = unpack(redis.call( + 'hmget', QlessJob.ns .. self.jid, 'queue', 'state', 'worker')) + + if not state then + error('Fail(): Job does not exist') + elseif state ~= 'running' then + error('Fail(): Job not currently running: ' .. state) + elseif worker ~= oldworker then + error('Fail(): Job running with another worker: ' .. oldworker) + end + + Qless.publish('log', cjson.encode({ + jid = self.jid, + event = 'failed', + worker = worker, + group = group, + message = message + })) + + if redis.call('zscore', 'ql:tracked', self.jid) ~= false then + Qless.publish('failed', self.jid) + end + + redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid) + + self:history(now, 'failed', {worker = worker, group = group}) + + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failures', 1) + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed' , 1) + + local queue_obj = Qless.queue(queue) + queue_obj.work.remove(self.jid) + queue_obj.locks.remove(self.jid) + queue_obj.scheduled.remove(self.jid) + + self:release_resources(now) + + if data then + redis.call('hset', QlessJob.ns .. self.jid, 'data', cjson.encode(data)) + end + + redis.call('hmset', QlessJob.ns .. self.jid, + 'state', 'failed', + 'worker', '', + 'expires', '', + 'failure', cjson.encode({ + ['group'] = group, + ['message'] = message, + ['when'] = math.floor(now), + ['worker'] = worker + })) + + redis.call('sadd', 'ql:failures', group) + redis.call('lpush', 'ql:f:' .. group, self.jid) + + + return self.jid +end + +function QlessJob:retry(now, queue, worker, delay, group, message) + assert(queue , 'Retry(): Arg "queue" missing') + assert(worker, 'Retry(): Arg "worker" missing') + delay = assert(tonumber(delay or 0), + 'Retry(): Arg "delay" not a number: ' .. tostring(delay)) + + local oldqueue, state, retries, oldworker, priority, failure = unpack( + redis.call('hmget', QlessJob.ns .. self.jid, 'queue', 'state', + 'retries', 'worker', 'priority', 'failure')) + + if oldworker == false then + error('Retry(): Job does not exist') + elseif state ~= 'running' then + error('Retry(): Job is not currently running: ' .. state) + elseif oldworker ~= worker then + error('Retry(): Job has been given to another worker: ' .. oldworker) + end + + local remaining = tonumber(redis.call( + 'hincrby', QlessJob.ns .. self.jid, 'remaining', -1)) + redis.call('hdel', QlessJob.ns .. self.jid, 'grace') + + Qless.queue(oldqueue).locks.remove(self.jid) + self:release_resources(now) + + redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid) + + if remaining < 0 then + local group = group or 'failed-retries-' .. queue + self:history(now, 'failed', {['group'] = group}) + + redis.call('hmset', QlessJob.ns .. self.jid, 'state', 'failed', + 'worker', '', + 'expires', '') + if group ~= nil and message ~= nil then + redis.call('hset', QlessJob.ns .. self.jid, + 'failure', cjson.encode({ + ['group'] = group, + ['message'] = message, + ['when'] = math.floor(now), + ['worker'] = worker + }) + ) + else + redis.call('hset', QlessJob.ns .. self.jid, + 'failure', cjson.encode({ + ['group'] = group, + ['message'] = + 'Job exhausted retries in queue "' .. oldqueue .. '"', + ['when'] = now, + ['worker'] = unpack(self:data('worker')) + })) + end + + redis.call('sadd', 'ql:failures', group) + redis.call('lpush', 'ql:f:' .. group, self.jid) + local bin = now - (now % 86400) + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failures', 1) + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed' , 1) + else + local queue_obj = Qless.queue(queue) + if delay > 0 then + queue_obj.scheduled.add(now + delay, self.jid) + redis.call('hset', QlessJob.ns .. self.jid, 'state', 'scheduled') + else + if self:acquire_resources(now) then + queue_obj.work.add(now, priority, self.jid) + end + redis.call('hset', QlessJob.ns .. self.jid, 'state', 'waiting') + end + + if group ~= nil and message ~= nil then + redis.call('hset', QlessJob.ns .. self.jid, + 'failure', cjson.encode({ + ['group'] = group, + ['message'] = message, + ['when'] = math.floor(now), + ['worker'] = worker + }) + ) + end + end + + return math.floor(remaining) +end + +function QlessJob:depends(now, command, ...) + assert(command, 'Depends(): Arg "command" missing') + local state = redis.call('hget', QlessJob.ns .. self.jid, 'state') + if state ~= 'depends' then + error('Depends(): Job ' .. self.jid .. + ' not in the depends state: ' .. tostring(state)) + end + + if command == 'on' then + for i, j in ipairs(arg) do + local state = redis.call('hget', QlessJob.ns .. j, 'state') + if (state and state ~= 'complete') then + redis.call( + 'sadd', QlessJob.ns .. j .. '-dependents' , self.jid) + redis.call( + 'sadd', QlessJob.ns .. self.jid .. '-dependencies', j) + end + end + return true + elseif command == 'off' then + if arg[1] == 'all' then + for i, j in ipairs(redis.call( + 'smembers', QlessJob.ns .. self.jid .. '-dependencies')) do + redis.call('srem', QlessJob.ns .. j .. '-dependents', self.jid) + end + redis.call('del', QlessJob.ns .. self.jid .. '-dependencies') + local q, p = unpack(redis.call( + 'hmget', QlessJob.ns .. self.jid, 'queue', 'priority')) + if q then + local queue_obj = Qless.queue(q) + queue_obj.depends.remove(self.jid) + if self:acquire_resources(now) then + queue_obj.work.add(now, p, self.jid) + end + redis.call('hset', QlessJob.ns .. self.jid, 'state', 'waiting') + end + else + for i, j in ipairs(arg) do + redis.call('srem', QlessJob.ns .. j .. '-dependents', self.jid) + redis.call( + 'srem', QlessJob.ns .. self.jid .. '-dependencies', j) + if redis.call('scard', + QlessJob.ns .. self.jid .. '-dependencies') == 0 then + local q, p = unpack(redis.call( + 'hmget', QlessJob.ns .. self.jid, 'queue', 'priority')) + if q then + local queue_obj = Qless.queue(q) + queue_obj.depends.remove(self.jid) + if self:acquire_resources(now) then + queue_obj.work.add(now, p, self.jid) + end + redis.call('hset', + QlessJob.ns .. self.jid, 'state', 'waiting') + end + end + end + end + return true + else + error('Depends(): Argument "command" must be "on" or "off"') + end +end + +function QlessJob:heartbeat(now, worker, data) + assert(worker, 'Heatbeat(): Arg "worker" missing') + + local queue = redis.call('hget', QlessJob.ns .. self.jid, 'queue') or '' + local expires = now + tonumber( + Qless.config.get(queue .. '-heartbeat') or + Qless.config.get('heartbeat', 60)) + + if data then + data = cjson.decode(data) + end + + local job_worker, state = unpack( + redis.call('hmget', QlessJob.ns .. self.jid, 'worker', 'state')) + if job_worker == false then + error('Heartbeat(): Job does not exist') + elseif state ~= 'running' then + error('Heartbeat(): Job not currently running: ' .. state) + elseif job_worker ~= worker or #job_worker == 0 then + error('Heartbeat(): Job given out to another worker: ' .. job_worker) + else + if data then + redis.call('hmset', QlessJob.ns .. self.jid, 'expires', + expires, 'worker', worker, 'data', cjson.encode(data)) + else + redis.call('hmset', QlessJob.ns .. self.jid, + 'expires', expires, 'worker', worker) + end + + redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, self.jid) + + local queue = Qless.queue( + redis.call('hget', QlessJob.ns .. self.jid, 'queue')) + queue.locks.add(expires, self.jid) + return expires + end +end + +function QlessJob:priority(priority) + priority = assert(tonumber(priority), + 'Priority(): Arg "priority" missing or not a number: ' .. + tostring(priority)) + + local queue = redis.call('hget', QlessJob.ns .. self.jid, 'queue') + + if queue == nil then + error('Priority(): Job ' .. self.jid .. ' does not exist') + elseif queue == '' then + redis.call('hset', QlessJob.ns .. self.jid, 'priority', priority) + return priority + else + local queue_obj = Qless.queue(queue) + if queue_obj.work.score(self.jid) then + queue_obj.work.add(0, priority, self.jid) + end + redis.call('hset', QlessJob.ns .. self.jid, 'priority', priority) + return priority + end +end + +function QlessJob:update(data) + local tmp = {} + for k, v in pairs(data) do + table.insert(tmp, k) + table.insert(tmp, v) + end + redis.call('hmset', QlessJob.ns .. self.jid, unpack(tmp)) +end + +function QlessJob:timeout(now) + local queue_name, state, worker = unpack(redis.call('hmget', + QlessJob.ns .. self.jid, 'queue', 'state', 'worker')) + if queue_name == nil then + error('Timeout(): Job does not exist') + elseif state ~= 'running' then + error('Timeout(): Job ' .. self.jid .. ' not running') + else + self:history(now, 'timed-out') + local queue = Qless.queue(queue_name) + queue.locks.remove(self.jid) + queue.work.add(now, math.huge, self.jid) + redis.call('hmset', QlessJob.ns .. self.jid, + 'state', 'stalled', 'expires', 0) + local encoded = cjson.encode({ + jid = self.jid, + event = 'lock_lost', + worker = worker + }) + Qless.publish('w:' .. worker, encoded) + Qless.publish('log', encoded) + return queue_name + end +end + +function QlessJob:exists() + return redis.call('exists', QlessJob.ns .. self.jid) == 1 +end + +function QlessJob:history(now, what, item) + local history = redis.call('hget', QlessJob.ns .. self.jid, 'history') + if history then + history = cjson.decode(history) + for i, value in ipairs(history) do + redis.call('rpush', QlessJob.ns .. self.jid .. '-history', + cjson.encode({math.floor(value.put), 'put', {q = value.q}})) + + if value.popped then + redis.call('rpush', QlessJob.ns .. self.jid .. '-history', + cjson.encode({math.floor(value.popped), 'popped', + {worker = value.worker}})) + end + + if value.failed then + redis.call('rpush', QlessJob.ns .. self.jid .. '-history', + cjson.encode( + {math.floor(value.failed), 'failed', nil})) + end + + if value.done then + redis.call('rpush', QlessJob.ns .. self.jid .. '-history', + cjson.encode( + {math.floor(value.done), 'done', nil})) + end + end + redis.call('hdel', QlessJob.ns .. self.jid, 'history') + end + + if what == nil then + local response = {} + for i, value in ipairs(redis.call('lrange', + QlessJob.ns .. self.jid .. '-history', 0, -1)) do + value = cjson.decode(value) + local dict = value[3] or {} + dict['when'] = value[1] + dict['what'] = value[2] + table.insert(response, dict) + end + return response + else + local count = tonumber(Qless.config.get('max-job-history', 100)) + if count > 0 then + local obj = redis.call('lpop', QlessJob.ns .. self.jid .. '-history') + redis.call('ltrim', QlessJob.ns .. self.jid .. '-history', -count + 2, -1) + if obj ~= nil then + redis.call('lpush', QlessJob.ns .. self.jid .. '-history', obj) + end + end + return redis.call('rpush', QlessJob.ns .. self.jid .. '-history', + cjson.encode({math.floor(now), what, item})) + end +end + +function QlessJob:release_resources(now) + local resources = redis.call('hget', QlessJob.ns .. self.jid, 'resources') + resources = cjson.decode(resources or '[]') + for _, res in ipairs(resources) do + Qless.resource(res):release(now, self.jid) + end +end + +function QlessJob:acquire_resources(now) + local resources, priority = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'resources', 'priority')) + resources = cjson.decode(resources or '[]') + if (#resources == 0) then + return true + end + + local acquired_all = true + + for _, res in ipairs(resources) do + local ok, res = pcall(function() return Qless.resource(res):acquire(now, priority, self.jid) end) + if not ok then + self:set_failed(now, 'system:fatal', res.msg) + return false + end + acquired_all = acquired_all and res + end + + return acquired_all +end + +function QlessJob:set_failed(now, group, message, worker, release_work, release_resources) + local group = assert(group, 'Fail(): Arg "group" missing') + local message = assert(message, 'Fail(): Arg "message" missing') + local worker = worker or 'none' + local release_work = release_work or true + local release_resources = release_resources or false + + local bin = now - (now % 86400) + + local queue = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'queue')) + + Qless.publish('log', cjson.encode({ + jid = self.jid, + event = 'failed', + worker = worker, + group = group, + message = message + })) + + if redis.call('zscore', 'ql:tracked', self.jid) ~= false then + Qless.publish('failed', self.jid) + end + + self:history(now, 'failed', {group = group}) + + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failures', 1) + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed' , 1) + + if release_work then + local queue_obj = Qless.queue(queue) + queue_obj.work.remove(self.jid) + queue_obj.locks.remove(self.jid) + queue_obj.scheduled.remove(self.jid) + end + + if release_resources then + self:release_resources(now) + end + + redis.call('hmset', QlessJob.ns .. self.jid, + 'state', 'failed', + 'worker', '', + 'expires', '', + 'failure', cjson.encode({ + ['group'] = group, + ['message'] = message, + ['when'] = math.floor(now) + })) + + redis.call('sadd', 'ql:failures', group) + redis.call('lpush', 'ql:f:' .. group, self.jid) + + + return self.jid +end +function Qless.queue(name) + assert(name, 'Queue(): no queue name provided') + local queue = {} + setmetatable(queue, QlessQueue) + queue.name = name + + queue.work = { + peek = function(count) + if count == 0 then + return {} + end + local jids = {} + for index, jid in ipairs(redis.call( + 'zrevrange', queue:prefix('work'), 0, count - 1)) do + table.insert(jids, jid) + end + return jids + end, remove = function(...) + if #arg > 0 then + return redis.call('zrem', queue:prefix('work'), unpack(arg)) + end + end, add = function(now, priority, jid) + return redis.call('zadd', + queue:prefix('work'), priority - (now / 10000000000), jid) + end, score = function(jid) + return redis.call('zscore', queue:prefix('work'), jid) + end, length = function() + return redis.call('zcard', queue:prefix('work')) + end + } + + queue.locks = { + expired = function(now, offset, count) + return redis.call('zrangebyscore', + queue:prefix('locks'), -math.huge, now, 'LIMIT', offset, count) + end, peek = function(now, offset, count) + return redis.call('zrangebyscore', queue:prefix('locks'), + now, math.huge, 'LIMIT', offset, count) + end, add = function(expires, jid) + redis.call('zadd', queue:prefix('locks'), expires, jid) + end, remove = function(...) + if #arg > 0 then + return redis.call('zrem', queue:prefix('locks'), unpack(arg)) + end + end, running = function(now) + return redis.call('zcount', queue:prefix('locks'), now, math.huge) + end, length = function(now) + if now then + return redis.call('zcount', queue:prefix('locks'), 0, now) + else + return redis.call('zcard', queue:prefix('locks')) + end + end, job_time_left = function(now, jid) + return tonumber(redis.call('zscore', queue:prefix('locks'), jid) or 0) - now + end + } + + queue.depends = { + peek = function(now, offset, count) + return redis.call('zrange', + queue:prefix('depends'), offset, offset + count - 1) + end, add = function(now, jid) + redis.call('zadd', queue:prefix('depends'), now, jid) + end, remove = function(...) + if #arg > 0 then + return redis.call('zrem', queue:prefix('depends'), unpack(arg)) + end + end, length = function() + return redis.call('zcard', queue:prefix('depends')) + end + } + + queue.scheduled = { + peek = function(now, offset, count) + return redis.call('zrange', + queue:prefix('scheduled'), offset, offset + count - 1) + end, ready = function(now, offset, count) + return redis.call('zrangebyscore', + queue:prefix('scheduled'), 0, now, 'LIMIT', offset, count) + end, add = function(when, jid) + redis.call('zadd', queue:prefix('scheduled'), when, jid) + end, remove = function(...) + if #arg > 0 then + return redis.call('zrem', queue:prefix('scheduled'), unpack(arg)) + end + end, length = function() + return redis.call('zcard', queue:prefix('scheduled')) + end + } + + queue.recurring = { + peek = function(now, offset, count) + return redis.call('zrangebyscore', queue:prefix('recur'), + 0, now, 'LIMIT', offset, count) + end, ready = function(now, offset, count) + end, add = function(when, jid) + redis.call('zadd', queue:prefix('recur'), when, jid) + end, remove = function(...) + if #arg > 0 then + return redis.call('zrem', queue:prefix('recur'), unpack(arg)) + end + end, update = function(increment, jid) + redis.call('zincrby', queue:prefix('recur'), increment, jid) + end, score = function(jid) + return redis.call('zscore', queue:prefix('recur'), jid) + end, length = function() + return redis.call('zcard', queue:prefix('recur')) + end + } + return queue +end + +function QlessQueue:prefix(group) + if group then + return QlessQueue.ns..self.name..'-'..group + else + return QlessQueue.ns..self.name + end +end + +function QlessQueue:stats(now, date) + date = assert(tonumber(date), + 'Stats(): Arg "date" missing or not a number: '.. (date or 'nil')) + + local bin = date - (date % 86400) + + local histokeys = { + 's0','s1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11','s12','s13','s14','s15','s16','s17','s18','s19','s20','s21','s22','s23','s24','s25','s26','s27','s28','s29','s30','s31','s32','s33','s34','s35','s36','s37','s38','s39','s40','s41','s42','s43','s44','s45','s46','s47','s48','s49','s50','s51','s52','s53','s54','s55','s56','s57','s58','s59', + 'm1','m2','m3','m4','m5','m6','m7','m8','m9','m10','m11','m12','m13','m14','m15','m16','m17','m18','m19','m20','m21','m22','m23','m24','m25','m26','m27','m28','m29','m30','m31','m32','m33','m34','m35','m36','m37','m38','m39','m40','m41','m42','m43','m44','m45','m46','m47','m48','m49','m50','m51','m52','m53','m54','m55','m56','m57','m58','m59', + 'h1','h2','h3','h4','h5','h6','h7','h8','h9','h10','h11','h12','h13','h14','h15','h16','h17','h18','h19','h20','h21','h22','h23', + 'd1','d2','d3','d4','d5','d6' + } + + local mkstats = function(name, bin, queue) + local results = {} + + local key = 'ql:s:' .. name .. ':' .. bin .. ':' .. queue + local count, mean, vk = unpack(redis.call('hmget', key, 'total', 'mean', 'vk')) + + count = tonumber(count) or 0 + mean = tonumber(mean) or 0 + vk = tonumber(vk) + + results.count = count or 0 + results.mean = mean or 0 + results.histogram = {} + + if not count then + results.std = 0 + else + if count > 1 then + results.std = math.sqrt(vk / (count - 1)) + else + results.std = 0 + end + end + + local histogram = redis.call('hmget', key, unpack(histokeys)) + for i=1,#histokeys do + table.insert(results.histogram, tonumber(histogram[i]) or 0) + end + return results + end + + local retries, failed, failures = unpack(redis.call('hmget', 'ql:s:stats:' .. bin .. ':' .. self.name, 'retries', 'failed', 'failures')) + return { + retries = tonumber(retries or 0), + failed = tonumber(failed or 0), + failures = tonumber(failures or 0), + wait = mkstats('wait', bin, self.name), + run = mkstats('run' , bin, self.name) + } +end + +function QlessQueue:peek(now, count) + count = assert(tonumber(count), + 'Peek(): Arg "count" missing or not a number: ' .. tostring(count)) + + local jids = self.locks.expired(now, 0, count) + + self:check_recurring(now, count - #jids) + + self:check_scheduled(now, count - #jids) + + table.extend(jids, self.work.peek(count - #jids)) + + return jids +end + +function QlessQueue:paused() + return redis.call('sismember', 'ql:paused_queues', self.name) == 1 +end + +function QlessQueue.pause(now, ...) + redis.call('sadd', 'ql:paused_queues', unpack(arg)) +end + +function QlessQueue.unpause(...) + redis.call('srem', 'ql:paused_queues', unpack(arg)) +end + +function QlessQueue:pop(now, worker, count) + assert(worker, 'Pop(): Arg "worker" missing') + count = assert(tonumber(count), + 'Pop(): Arg "count" missing or not a number: ' .. tostring(count)) + + local expires = now + tonumber( + Qless.config.get(self.name .. '-heartbeat') or + Qless.config.get('heartbeat', 60)) + + if self:paused() then + return {} + end + + redis.call('zadd', 'ql:workers', now, worker) + + local max_concurrency = tonumber( + Qless.config.get(self.name .. '-max-concurrency', 0)) + + if max_concurrency > 0 then + local allowed = math.max(0, max_concurrency - self.locks.running(now)) + count = math.min(allowed, count) + if count == 0 then + return {} + end + end + + local jids = self:invalidate_locks(now, count) + + self:check_recurring(now, count - #jids) + + self:check_scheduled(now, count - #jids) + + table.extend(jids, self.work.peek(count - #jids)) + + local state + for index, jid in ipairs(jids) do + local job = Qless.job(jid) + state = unpack(job:data('state')) + job:history(now, 'popped', {worker = worker}) + + local time = tonumber( + redis.call('hget', QlessJob.ns .. jid, 'time') or now) + local waiting = now - time + self:stat(now, 'wait', waiting) + redis.call('hset', QlessJob.ns .. jid, + 'time', string.format("%.20f", now)) + + redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, jid) + + job:update({ + worker = worker, + expires = expires, + state = 'running' + }) + + self.locks.add(expires, jid) + + local tracked = redis.call('zscore', 'ql:tracked', jid) ~= false + if tracked then + Qless.publish('popped', jid) + end + end + + self.work.remove(unpack(jids)) + + return jids +end + +function QlessQueue:stat(now, stat, val) + local bin = now - (now % 86400) + local key = 'ql:s:' .. stat .. ':' .. bin .. ':' .. self.name + + local count, mean, vk = unpack( + redis.call('hmget', key, 'total', 'mean', 'vk')) + + count = count or 0 + if count == 0 then + mean = val + vk = 0 + count = 1 + else + count = count + 1 + local oldmean = mean + mean = mean + (val - mean) / count + vk = vk + (val - mean) * (val - oldmean) + end + + val = math.floor(val) + if val < 60 then -- seconds + redis.call('hincrby', key, 's' .. val, 1) + elseif val < 3600 then -- minutes + redis.call('hincrby', key, 'm' .. math.floor(val / 60), 1) + elseif val < 86400 then -- hours + redis.call('hincrby', key, 'h' .. math.floor(val / 3600), 1) + else -- days + redis.call('hincrby', key, 'd' .. math.floor(val / 86400), 1) + end + redis.call('hmset', key, 'total', count, 'mean', mean, 'vk', vk) +end + +function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...) + assert(jid , 'Put(): Arg "jid" missing') + assert(klass, 'Put(): Arg "klass" missing') + local data = assert(cjson.decode(raw_data), + 'Put(): Arg "data" missing or not JSON: ' .. tostring(raw_data)) + delay = assert(tonumber(delay), + 'Put(): Arg "delay" not a number: ' .. tostring(delay)) + + if #arg % 2 == 1 then + error('Odd number of additional args: ' .. tostring(arg)) + end + local options = {} + for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end + + local job = Qless.job(jid) + local priority, tags, oldqueue, state, failure, retries, oldworker, interval, next_run, old_resources = + unpack(redis.call('hmget', QlessJob.ns .. jid, 'priority', 'tags', + 'queue', 'state', 'failure', 'retries', 'worker', 'throttle_interval', 'throttle_next_run', 'resources')) + + next_run = next_run or now + + local replace = assert(tonumber(options['replace'] or 1) , + 'Put(): Arg "replace" not a number: ' .. tostring(options['replace'])) + + if replace == 0 and state == 'running' then + local time_left = self.locks.job_time_left(now, jid) + if time_left > 0 then + return time_left + end + end + + if tags then + Qless.tag(now, 'remove', jid, unpack(cjson.decode(tags))) + end + + retries = assert(tonumber(options['retries'] or retries or 5) , + 'Put(): Arg "retries" not a number: ' .. tostring(options['retries'])) + tags = assert(cjson.decode(options['tags'] or tags or '[]' ), + 'Put(): Arg "tags" not JSON' .. tostring(options['tags'])) + priority = assert(tonumber(options['priority'] or priority or 0), + 'Put(): Arg "priority" not a number' .. tostring(options['priority'])) + local depends = assert(cjson.decode(options['depends'] or '[]') , + 'Put(): Arg "depends" not JSON: ' .. tostring(options['depends'])) + + local resources = assert(cjson.decode(options['resources'] or '[]'), + 'Put(): Arg "resources" not JSON array: ' .. tostring(options['resources'])) + assert(#resources == 0 or QlessResource.all_exist(resources), 'Put(): invalid resources requested') + + if old_resources then + old_resources = Set.new(cjson.decode(old_resources)) + local removed_resources = Set.diff(old_resources, Set.new(resources)) + for k in pairs(removed_resources) do + Qless.resource(k):release(now, jid) + end + end + + local interval = assert(tonumber(options['interval'] or interval or 0), + 'Put(): Arg "interval" not a number: ' .. tostring(options['interval'])) + + if interval > 0 then + local minimum_delay = next_run - now + if minimum_delay < 0 then + minimum_delay = 0 + elseif minimum_delay > delay then + delay = minimum_delay + end + else + next_run = 0 + end + + if #depends > 0 then + local new = {} + for _, d in ipairs(depends) do new[d] = 1 end + + local original = redis.call( + 'smembers', QlessJob.ns .. jid .. '-dependencies') + for _, dep in pairs(original) do + if new[dep] == nil then + redis.call('srem', QlessJob.ns .. dep .. '-dependents' , jid) + redis.call('srem', QlessJob.ns .. jid .. '-dependencies', dep) + end + end + end + + Qless.publish('log', cjson.encode({ + jid = jid, + event = 'put', + queue = self.name + })) + + job:history(now, 'put', {q = self.name}) + + if oldqueue then + local queue_obj = Qless.queue(oldqueue) + queue_obj.work.remove(jid) + queue_obj.locks.remove(jid) + queue_obj.depends.remove(jid) + queue_obj.scheduled.remove(jid) + end + + if oldworker and oldworker ~= '' then + redis.call('zrem', 'ql:w:' .. oldworker .. ':jobs', jid) + if oldworker ~= worker then + local encoded = cjson.encode({ + jid = jid, + event = 'lock_lost', + worker = oldworker + }) + Qless.publish('w:' .. oldworker, encoded) + Qless.publish('log', encoded) + end + end + + if state == 'complete' then + redis.call('zrem', 'ql:completed', jid) + end + + for i, tag in ipairs(tags) do + redis.call('zadd', 'ql:t:' .. tag, now, jid) + redis.call('zincrby', 'ql:tags', 1, tag) + end + + if state == 'failed' then + failure = cjson.decode(failure) + redis.call('lrem', 'ql:f:' .. failure.group, 0, jid) + if redis.call('llen', 'ql:f:' .. failure.group) == 0 then + redis.call('srem', 'ql:failures', failure.group) + end + local bin = failure.when - (failure.when % 86400) + redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. self.name, 'failed' , -1) + end + + redis.call('hmset', QlessJob.ns .. jid, + 'jid' , jid, + 'klass' , klass, + 'data' , raw_data, + 'priority' , priority, + 'tags' , cjson.encode(tags), + 'resources', cjson.encode(resources), + 'state' , ((delay > 0) and 'scheduled') or 'waiting', + 'worker' , '', + 'expires' , 0, + 'queue' , self.name, + 'retries' , retries, + 'remaining', retries, + 'throttle_interval', interval, + 'throttle_next_run', next_run, + 'time' , string.format("%.20f", now), + 'result_data', '{}') + + for i, j in ipairs(depends) do + local state = redis.call('hget', QlessJob.ns .. j, 'state') + if (state and state ~= 'complete') then + redis.call('sadd', QlessJob.ns .. j .. '-dependents' , jid) + redis.call('sadd', QlessJob.ns .. jid .. '-dependencies', j) + end + end + + if delay > 0 then + if redis.call('scard', QlessJob.ns .. jid .. '-dependencies') > 0 then + self.depends.add(now, jid) + redis.call('hmset', QlessJob.ns .. jid, + 'state', 'depends', + 'scheduled', now + delay) + else + self.scheduled.add(now + delay, jid) + end + else + if redis.call('scard', QlessJob.ns .. jid .. '-dependencies') > 0 then + self.depends.add(now, jid) + redis.call('hset', QlessJob.ns .. jid, 'state', 'depends') + elseif #resources > 0 then + if Qless.job(jid):acquire_resources(now) then + self.work.add(now, priority, jid) + end + else + self.work.add(now, priority, jid) + end + end + + if redis.call('zscore', 'ql:queues', self.name) == false then + redis.call('zadd', 'ql:queues', now, self.name) + end + + if redis.call('zscore', 'ql:tracked', jid) ~= false then + Qless.publish('put', jid) + end + + return jid +end + +function QlessQueue:unfail(now, group, count) + assert(group, 'Unfail(): Arg "group" missing') + count = assert(tonumber(count or 25), + 'Unfail(): Arg "count" not a number: ' .. tostring(count)) + + local jids = redis.call('lrange', 'ql:f:' .. group, -count, -1) + + local toinsert = {} + for index, jid in ipairs(jids) do + local job = Qless.job(jid) + local data = job:data() + job:history(now, 'put', {q = self.name}) + redis.call('hmset', QlessJob.ns .. data.jid, + 'state' , 'waiting', + 'worker' , '', + 'expires' , 0, + 'queue' , self.name, + 'remaining', data.retries or 5) + + if #data['resources'] then + if job:acquire_resources(now) then + self.work.add(now, data.priority, data.jid) + end + else + self.work.add(now, data.priority, data.jid) + end + end + + redis.call('ltrim', 'ql:f:' .. group, 0, -count - 1) + if (redis.call('llen', 'ql:f:' .. group) == 0) then + redis.call('srem', 'ql:failures', group) + end + + return #jids +end + +function QlessQueue:recur(now, jid, klass, raw_data, spec, ...) + assert(jid , 'RecurringJob On(): Arg "jid" missing') + assert(klass, 'RecurringJob On(): Arg "klass" missing') + assert(spec , 'RecurringJob On(): Arg "spec" missing') + local data = assert(cjson.decode(raw_data), + 'RecurringJob On(): Arg "data" not JSON: ' .. tostring(raw_data)) + + if spec == 'interval' then + local interval = assert(tonumber(arg[1]), + 'Recur(): Arg "interval" not a number: ' .. tostring(arg[1])) + local offset = assert(tonumber(arg[2]), + 'Recur(): Arg "offset" not a number: ' .. tostring(arg[2])) + if interval <= 0 then + error('Recur(): Arg "interval" must be greater than or equal to 0') + end + + if #arg % 2 == 1 then + error('Odd number of additional args: ' .. tostring(arg)) + end + + local options = {} + for i = 3, #arg, 2 do options[arg[i]] = arg[i + 1] end + options.tags = assert(cjson.decode(options.tags or '{}'), + 'Recur(): Arg "tags" must be JSON string array: ' .. tostring( + options.tags)) + options.priority = assert(tonumber(options.priority or 0), + 'Recur(): Arg "priority" not a number: ' .. tostring( + options.priority)) + options.retries = assert(tonumber(options.retries or 0), + 'Recur(): Arg "retries" not a number: ' .. tostring( + options.retries)) + options.backlog = assert(tonumber(options.backlog or 0), + 'Recur(): Arg "backlog" not a number: ' .. tostring( + options.backlog)) + options.resources = assert(cjson.decode(options['resources'] or '[]'), + 'Recur(): Arg "resources" not JSON array: ' .. tostring(options['resources'])) + + local count, old_queue = unpack(redis.call('hmget', 'ql:r:' .. jid, 'count', 'queue')) + count = count or 0 + + if old_queue then + Qless.queue(old_queue).recurring.remove(jid) + end + + redis.call('hmset', 'ql:r:' .. jid, + 'jid' , jid, + 'klass' , klass, + 'data' , raw_data, + 'priority' , options.priority, + 'tags' , cjson.encode(options.tags or {}), + 'state' , 'recur', + 'queue' , self.name, + 'type' , 'interval', + 'count' , count, + 'interval' , interval, + 'retries' , options.retries, + 'backlog' , options.backlog, + 'resources', cjson.encode(options.resources)) + self.recurring.add(now + offset, jid) + + if redis.call('zscore', 'ql:queues', self.name) == false then + redis.call('zadd', 'ql:queues', now, self.name) + end + + return jid + else + error('Recur(): schedule type "' .. tostring(spec) .. '" unknown') + end +end + +function QlessQueue:length() + return self.locks.length() + self.work.length() + self.scheduled.length() +end + +function QlessQueue:check_recurring(now, count) + local moved = 0 + local r = self.recurring.peek(now, 0, count) + for index, jid in ipairs(r) do + local klass, data, priority, tags, retries, interval, backlog, resources = unpack( + redis.call('hmget', 'ql:r:' .. jid, 'klass', 'data', 'priority', + 'tags', 'retries', 'interval', 'backlog', 'resources')) + local _tags = cjson.decode(tags) + local resources = cjson.decode(resources or '[]') + local score = math.floor(tonumber(self.recurring.score(jid))) + interval = tonumber(interval) + + backlog = tonumber(backlog or 0) + if backlog ~= 0 then + local num = ((now - score) / interval) + if num > backlog then + score = score + ( + math.ceil(num - backlog) * interval + ) + end + end + + while (score <= now) and (moved < count) do + local count = redis.call('hincrby', 'ql:r:' .. jid, 'count', 1) + moved = moved + 1 + + local child_jid = jid .. '-' .. count + + for i, tag in ipairs(_tags) do + redis.call('zadd', 'ql:t:' .. tag, now, child_jid) + redis.call('zincrby', 'ql:tags', 1, tag) + end + + redis.call('hmset', QlessJob.ns .. child_jid, + 'jid' , child_jid, + 'klass' , klass, + 'data' , data, + 'priority' , priority, + 'tags' , tags, + 'state' , 'waiting', + 'worker' , '', + 'expires' , 0, + 'queue' , self.name, + 'retries' , retries, + 'remaining' , retries, + 'resources' , cjson.encode(resources), + 'throttle_interval', 0, + 'time' , string.format("%.20f", score)) + + local job = Qless.job(child_jid) + job:history(score, 'put', {q = self.name}) + + + local add_job = true + if #resources then + add_job = job:acquire_resources(score) + end + if add_job then + self.work.add(score, priority, child_jid) + end + + score = score + interval + self.recurring.add(score, jid) + end + end +end + +function QlessQueue:check_scheduled(now, count) + local scheduled = self.scheduled.ready(now, 0, count) + for index, jid in ipairs(scheduled) do + local priority = tonumber( + redis.call('hget', QlessJob.ns .. jid, 'priority') or 0) + if Qless.job(jid):acquire_resources(now) then + self.work.add(now, priority, jid) + end + self.scheduled.remove(jid) + + redis.call('hset', QlessJob.ns .. jid, 'state', 'waiting') + end +end + +function QlessQueue:invalidate_locks(now, count) + local jids = {} + for index, jid in ipairs(self.locks.expired(now, 0, count)) do + local worker, failure = unpack( + redis.call('hmget', QlessJob.ns .. jid, 'worker', 'failure')) + redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid) + + local grace_period = tonumber(Qless.config.get('grace-period')) + + local courtesy_sent = tonumber( + redis.call('hget', QlessJob.ns .. jid, 'grace') or 0) + + local send_message = (courtesy_sent ~= 1) + local invalidate = not send_message + + if grace_period <= 0 then + send_message = true + invalidate = true + end + + if send_message then + if redis.call('zscore', 'ql:tracked', jid) ~= false then + Qless.publish('stalled', jid) + end + Qless.job(jid):history(now, 'timed-out') + redis.call('hset', QlessJob.ns .. jid, 'grace', 1) + + local encoded = cjson.encode({ + jid = jid, + event = 'lock_lost', + worker = worker + }) + Qless.publish('w:' .. worker, encoded) + Qless.publish('log', encoded) + self.locks.add(now + grace_period, jid) + + local bin = now - (now % 86400) + redis.call('hincrby', + 'ql:s:stats:' .. bin .. ':' .. self.name, 'retries', 1) + end + + if invalidate then + redis.call('hdel', QlessJob.ns .. jid, 'grace', 0) + + local remaining = tonumber(redis.call( + 'hincrby', QlessJob.ns .. jid, 'remaining', -1)) + + if remaining < 0 then + Qless.job(jid):release_resources(now) + + self.work.remove(jid) + self.locks.remove(jid) + self.scheduled.remove(jid) + + local group = 'failed-retries-' .. Qless.job(jid):data()['queue'] + local job = Qless.job(jid) + job:history(now, 'failed', {group = group}) + redis.call('hmset', QlessJob.ns .. jid, 'state', 'failed', + 'worker', '', + 'expires', '') + redis.call('hset', QlessJob.ns .. jid, + 'failure', cjson.encode({ + ['group'] = group, + ['message'] = + 'Job exhausted retries in queue "' .. self.name .. '"', + ['when'] = now, + ['worker'] = unpack(job:data('worker')) + })) + + redis.call('sadd', 'ql:failures', group) + redis.call('lpush', 'ql:f:' .. group, jid) + + if redis.call('zscore', 'ql:tracked', jid) ~= false then + Qless.publish('failed', jid) + end + Qless.publish('log', cjson.encode({ + jid = jid, + event = 'failed', + group = group, + worker = worker, + message = + 'Job exhausted retries in queue "' .. self.name .. '"' + })) + + local bin = now - (now % 86400) + redis.call('hincrby', + 'ql:s:stats:' .. bin .. ':' .. self.name, 'failures', 1) + redis.call('hincrby', + 'ql:s:stats:' .. bin .. ':' .. self.name, 'failed' , 1) + else + table.insert(jids, jid) + end + end + end + + return jids +end + +function QlessQueue.deregister(...) + redis.call('zrem', Qless.ns .. 'queues', unpack(arg)) +end + +function QlessQueue.counts(now, name) + if name then + local queue = Qless.queue(name) + local stalled = queue.locks.length(now) + queue:check_scheduled(now, queue.scheduled.length()) + return { + name = name, + waiting = queue.work.length(), + stalled = stalled, + running = queue.locks.length() - stalled, + scheduled = queue.scheduled.length(), + depends = queue.depends.length(), + recurring = queue.recurring.length(), + paused = queue:paused() + } + else + local queues = redis.call('zrange', 'ql:queues', 0, -1) + local response = {} + for index, qname in ipairs(queues) do + table.insert(response, QlessQueue.counts(now, qname)) + end + return response + end +end +function QlessRecurringJob:data() + local job = redis.call( + 'hmget', 'ql:r:' .. self.jid, 'jid', 'klass', 'state', 'queue', + 'priority', 'interval', 'retries', 'count', 'data', 'tags', 'backlog') + + if not job[1] then + return nil + end + + return { + jid = job[1], + klass = job[2], + state = job[3], + queue = job[4], + priority = tonumber(job[5]), + interval = tonumber(job[6]), + retries = tonumber(job[7]), + count = tonumber(job[8]), + data = job[9], + tags = cjson.decode(job[10]), + backlog = tonumber(job[11] or 0) + } +end + +function QlessRecurringJob:update(now, ...) + local options = {} + if redis.call('exists', 'ql:r:' .. self.jid) ~= 0 then + for i = 1, #arg, 2 do + local key = arg[i] + local value = arg[i+1] + assert(value, 'No value provided for ' .. tostring(key)) + if key == 'priority' or key == 'interval' or key == 'retries' then + value = assert(tonumber(value), 'Recur(): Arg "' .. key .. '" must be a number: ' .. tostring(value)) + if key == 'interval' then + local queue, interval = unpack(redis.call('hmget', 'ql:r:' .. self.jid, 'queue', 'interval')) + Qless.queue(queue).recurring.update( + value - tonumber(interval), self.jid) + end + redis.call('hset', 'ql:r:' .. self.jid, key, value) + elseif key == 'data' then + assert(cjson.decode(value), 'Recur(): Arg "data" is not JSON-encoded: ' .. tostring(value)) + redis.call('hset', 'ql:r:' .. self.jid, 'data', value) + elseif key == 'klass' then + redis.call('hset', 'ql:r:' .. self.jid, 'klass', value) + elseif key == 'queue' then + local queue_obj = Qless.queue( + redis.call('hget', 'ql:r:' .. self.jid, 'queue')) + local score = queue_obj.recurring.score(self.jid) + queue_obj.recurring.remove(self.jid) + Qless.queue(value).recurring.add(score, self.jid) + redis.call('hset', 'ql:r:' .. self.jid, 'queue', value) + if redis.call('zscore', 'ql:queues', value) == false then + redis.call('zadd', 'ql:queues', now, value) + end + elseif key == 'backlog' then + value = assert(tonumber(value), + 'Recur(): Arg "backlog" not a number: ' .. tostring(value)) + redis.call('hset', 'ql:r:' .. self.jid, 'backlog', value) + else + error('Recur(): Unrecognized option "' .. key .. '"') + end + end + return true + else + error('Recur(): No recurring job ' .. self.jid) + end +end + +function QlessRecurringJob:tag(...) + local tags = redis.call('hget', 'ql:r:' .. self.jid, 'tags') + if tags then + tags = cjson.decode(tags) + local _tags = {} + for i,v in ipairs(tags) do _tags[v] = true end + + for i=1,#arg do if _tags[arg[i]] == nil then table.insert(tags, arg[i]) end end + + tags = cjson.encode(tags) + redis.call('hset', 'ql:r:' .. self.jid, 'tags', tags) + return tags + else + error('Tag(): Job ' .. self.jid .. ' does not exist') + end +end + +function QlessRecurringJob:untag(...) + local tags = redis.call('hget', 'ql:r:' .. self.jid, 'tags') + if tags then + tags = cjson.decode(tags) + local _tags = {} + for i,v in ipairs(tags) do _tags[v] = true end + for i = 1,#arg do _tags[arg[i]] = nil end + local results = {} + for i, tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end + tags = cjson.encode(results) + redis.call('hset', 'ql:r:' .. self.jid, 'tags', tags) + return tags + else + error('Untag(): Job ' .. self.jid .. ' does not exist') + end +end + +function QlessRecurringJob:unrecur() + local queue = redis.call('hget', 'ql:r:' .. self.jid, 'queue') + if queue then + Qless.queue(queue).recurring.remove(self.jid) + redis.call('del', 'ql:r:' .. self.jid) + return true + else + return true + end +end +function QlessWorker.deregister(...) + redis.call('zrem', 'ql:workers', unpack(arg)) +end + +function QlessWorker.counts(now, worker) + local interval = tonumber(Qless.config.get('max-worker-age', 86400)) + + local workers = redis.call('zrangebyscore', 'ql:workers', 0, now - interval) + for index, worker in ipairs(workers) do + redis.call('del', 'ql:w:' .. worker .. ':jobs') + end + + redis.call('zremrangebyscore', 'ql:workers', 0, now - interval) + + if worker then + return { + jobs = redis.call('zrevrangebyscore', 'ql:w:' .. worker .. ':jobs', now + 8640000, now), + stalled = redis.call('zrevrangebyscore', 'ql:w:' .. worker .. ':jobs', now, 0) + } + else + local response = {} + local workers = redis.call('zrevrange', 'ql:workers', 0, -1) + for index, worker in ipairs(workers) do + table.insert(response, { + name = worker, + jobs = redis.call('zcount', 'ql:w:' .. worker .. ':jobs', now, now + 8640000), + stalled = redis.call('zcount', 'ql:w:' .. worker .. ':jobs', 0, now) + }) + end + return response + end +end + +function QlessResource:data(...) + local res = redis.call( + 'hmget', QlessResource.ns .. self.rid, 'rid', 'max') + + if not res[1] then + return nil + end + + local data = { + rid = res[1], + max = tonumber(res[2] or 0), + pending = self:pending(), + locks = self:locks(), + } + + return data +end + +function QlessResource:get(...) + local res = redis.call( + 'hmget', QlessResource.ns .. self.rid, 'rid', 'max') + + if not res[1] then + return nil + end + + return tonumber(res[2] or 0) +end + +function QlessResource:set(now, max) + local max = assert(tonumber(max), 'Set(): Arg "max" not a number: ' .. tostring(max)) + + local current_max = self:get() + if current_max == nil then + current_max = max + end + + local keyLocks = self:prefix('locks') + local current_locks = redis.pcall('scard', keyLocks) + local confirm_limit = math.max(current_max,current_locks) + local max_change = max - confirm_limit + local keyPending = self:prefix('pending') + + redis.call('hmset', QlessResource.ns .. self.rid, 'rid', self.rid, 'max', max); + + if max_change > 0 then + local jids = redis.call('zrevrange', keyPending, 0, max_change - 1, 'withscores') + local jid_count = #jids + if jid_count == 0 then + return self.rid + end + + for i = 1, jid_count, 2 do + + local newJid = jids[i] + local score = jids[i + 1] + + if Qless.job(newJid):acquire_resources(now) then + local data = Qless.job(newJid):data() + local queue = Qless.queue(data['queue']) + queue.work.add(score, 0, newJid) + end + end + end + + return self.rid +end + +function QlessResource:unset() + return redis.call('del', QlessResource.ns .. self.rid); +end + +function QlessResource:prefix(group) + if group then + return QlessResource.ns..self.rid..'-'..group + end + + return QlessResource.ns..self.rid +end + +function QlessResource:acquire(now, priority, jid) + local keyLocks = self:prefix('locks') + local max = self:get() + if max == nil then + error({code=1, msg='Acquire(): resource ' .. self.rid .. ' does not exist'}) + end + + if type(jid) ~= 'string' then + error({code=2, msg='Acquire(): invalid jid; expected string, got \'' .. type(jid) .. '\''}) + end + + + if redis.call('sismember', self:prefix('locks'), jid) == 1 then + return true + end + + local remaining = max - redis.pcall('scard', keyLocks) + + if remaining > 0 then + redis.call('sadd', keyLocks, jid) + redis.call('zrem', self:prefix('pending'), jid) + + return true + end + + if redis.call('zscore', self:prefix('pending'), jid) == false then + redis.call('zadd', self:prefix('pending'), priority - (now / 10000000000), jid) + end + + return false +end + +function QlessResource:release(now, jid) + local keyLocks = self:prefix('locks') + local keyPending = self:prefix('pending') + + redis.call('srem', keyLocks, jid) + redis.call('zrem', keyPending, jid) + + local jids = redis.call('zrevrange', keyPending, 0, 0, 'withscores') + if #jids == 0 then + return false + end + + local newJid = jids[1] + local score = jids[2] + + if Qless.job(newJid):acquire_resources(now) then + local data = Qless.job(newJid):data() + local queue = Qless.queue(data['queue']) + queue.work.add(score, 0, newJid) + end + + return newJid +end + +function QlessResource:locks() + return redis.call('smembers', self:prefix('locks')) +end + +function QlessResource:lock_count() + return redis.call('scard', self:prefix('locks')) +end + +function QlessResource:pending() + return redis.call('zrevrange', self:prefix('pending'), 0, -1) +end + +function QlessResource:pending_count() + return redis.call('zcard', self:prefix('pending')) +end + +function QlessResource:exists() + return redis.call('exists', self:prefix()) == 1 +end + +function QlessResource.all_exist(resources) + for _, res in ipairs(resources) do + if redis.call('exists', QlessResource.ns .. res) == 0 then + return false + end + end + return true +end + +function QlessResource.pending_counts(now) + local search = QlessResource.ns..'*-pending' + local reply = redis.call('keys', search) + local response = {} + for index, rname in ipairs(reply) do + local count = redis.call('zcard', rname) + local resStat = {name = rname, count = count} + table.insert(response,resStat) + end + return response +end + +function QlessResource.locks_counts(now) + local search = QlessResource.ns..'*-locks' + local reply = redis.call('keys', search) + local response = {} + for index, rname in ipairs(reply) do + local count = redis.call('scard', rname) + local resStat = {name = rname, count = count} + table.insert(response,resStat) + end + return response +end------------------------------------------------------------------------------- +local QlessAPI = {} + +local function tonil(value) + if (value == '') then return nil else return value end +end + +function QlessAPI.get(now, jid) + local data = Qless.job(jid):data() + if not data then + return nil + end + return cjson.encode(data) +end + +function QlessAPI.multiget(now, ...) + local results = {} + for i, jid in ipairs(arg) do + table.insert(results, Qless.job(jid):data()) + end + return cjson.encode(results) +end + +QlessAPI['config.get'] = function(now, key) + key = tonil(key) + if not key then + return cjson.encode(Qless.config.get(key)) + else + return Qless.config.get(key) + end +end + +QlessAPI['config.set'] = function(now, key, value) + key = tonil(key) + return Qless.config.set(key, value) +end + +QlessAPI['config.unset'] = function(now, key) + key = tonil(key) + return Qless.config.unset(key) +end + +QlessAPI.queues = function(now, queue) + return cjson.encode(QlessQueue.counts(now, queue)) +end + +QlessAPI.complete = function(now, jid, worker, queue, data, ...) + data = tonil(data) + return Qless.job(jid):complete(now, worker, queue, data, unpack(arg)) +end + +QlessAPI.failed = function(now, group, start, limit) + return cjson.encode(Qless.failed(group, start, limit)) +end + +QlessAPI.fail = function(now, jid, worker, group, message, data) + data = tonil(data) + return Qless.job(jid):fail(now, worker, group, message, data) +end + +QlessAPI.jobs = function(now, state, ...) + return Qless.jobs(now, state, unpack(arg)) +end + +QlessAPI.retry = function(now, jid, queue, worker, delay, group, message) + return Qless.job(jid):retry(now, queue, worker, delay, group, message) +end + +QlessAPI.depends = function(now, jid, command, ...) + return Qless.job(jid):depends(now, command, unpack(arg)) +end + +QlessAPI.heartbeat = function(now, jid, worker, data) + data = tonil(data) + return Qless.job(jid):heartbeat(now, worker, data) +end + +QlessAPI.workers = function(now, worker) + return cjson.encode(QlessWorker.counts(now, worker)) +end + +QlessAPI.track = function(now, command, jid) + return cjson.encode(Qless.track(now, command, jid)) +end + +QlessAPI.tag = function(now, command, ...) + return cjson.encode(Qless.tag(now, command, unpack(arg))) +end + +QlessAPI.stats = function(now, queue, date) + return cjson.encode(Qless.queue(queue):stats(now, date)) +end + +QlessAPI.priority = function(now, jid, priority) + return Qless.job(jid):priority(priority) +end + +QlessAPI.log = function(now, jid, message, data) + assert(jid, "Log(): Argument 'jid' missing") + assert(message, "Log(): Argument 'message' missing") + if data then + data = assert(cjson.decode(data), + "Log(): Argument 'data' not cjson: " .. tostring(data)) + end + + local job = Qless.job(jid) + assert(job:exists(), 'Log(): Job ' .. jid .. ' does not exist') + job:history(now, message, data) +end + +QlessAPI.peek = function(now, queue, count) + local jids = Qless.queue(queue):peek(now, count) + local response = {} + for i, jid in ipairs(jids) do + table.insert(response, Qless.job(jid):data()) + end + return cjson.encode(response) +end + +QlessAPI.pop = function(now, queue, worker, count) + local jids = Qless.queue(queue):pop(now, worker, count) + local response = {} + for i, jid in ipairs(jids) do + table.insert(response, Qless.job(jid):data()) + end + return cjson.encode(response) +end + +QlessAPI.pause = function(now, ...) + return QlessQueue.pause(now, unpack(arg)) +end + +QlessAPI.unpause = function(now, ...) + return QlessQueue.unpause(unpack(arg)) +end + +QlessAPI.paused = function(now, queue) + return Qless.queue(queue):paused() +end + +QlessAPI.cancel = function(now, ...) + return Qless.cancel(now, unpack(arg)) +end + +QlessAPI.timeout = function(now, ...) + for _, jid in ipairs(arg) do + Qless.job(jid):timeout(now) + end +end + +QlessAPI.put = function(now, me, queue, jid, klass, data, delay, ...) + data = tonil(data) + return Qless.queue(queue):put(now, me, jid, klass, data, delay, unpack(arg)) +end + +QlessAPI.requeue = function(now, me, queue, jid, ...) + local job = Qless.job(jid) + assert(job:exists(), 'Requeue(): Job ' .. jid .. ' does not exist') + return QlessAPI.put(now, me, queue, jid, unpack(arg)) +end + +QlessAPI.unfail = function(now, queue, group, count) + return Qless.queue(queue):unfail(now, group, count) +end + +QlessAPI.recur = function(now, queue, jid, klass, data, spec, ...) + data = tonil(data) + return Qless.queue(queue):recur(now, jid, klass, data, spec, unpack(arg)) +end + +QlessAPI.unrecur = function(now, jid) + return Qless.recurring(jid):unrecur() +end + +QlessAPI['recur.get'] = function(now, jid) + local data = Qless.recurring(jid):data() + if not data then + return nil + end + return cjson.encode(data) +end + +QlessAPI['recur.update'] = function(now, jid, ...) + return Qless.recurring(jid):update(now, unpack(arg)) +end + +QlessAPI['recur.tag'] = function(now, jid, ...) + return Qless.recurring(jid):tag(unpack(arg)) +end + +QlessAPI['recur.untag'] = function(now, jid, ...) + return Qless.recurring(jid):untag(unpack(arg)) +end + +QlessAPI.length = function(now, queue) + return Qless.queue(queue):length() +end + +QlessAPI['worker.deregister'] = function(now, ...) + return QlessWorker.deregister(unpack(arg)) +end + +QlessAPI['queue.forget'] = function(now, ...) + QlessQueue.deregister(unpack(arg)) +end + +QlessAPI['resource.set'] = function(now, rid, max) + return Qless.resource(rid):set(now, max) +end + +QlessAPI['resource.get'] = function(now, rid) + return Qless.resource(rid):get() +end + +QlessAPI['resource.data'] = function(now, rid) + local data = Qless.resource(rid):data() + if not data then + return nil + end + + return cjson.encode(data) +end + +QlessAPI['resource.exists'] = function(now, rid) + return Qless.resource(rid):exists() +end + +QlessAPI['resource.unset'] = function(now, rid) + return Qless.resource(rid):unset() +end + +QlessAPI['resource.locks'] = function(now, rid) + local data = Qless.resource(rid):locks() + if not data then + return nil + end + + return cjson.encode(data) +end + +QlessAPI['resource.lock_count'] = function(now, rid) + return Qless.resource(rid):lock_count() +end + +QlessAPI['resource.pending'] = function(now, rid) + local data = Qless.resource(rid):pending() + if not data then + return nil + end + + return cjson.encode(data) + +end + +QlessAPI['resource.pending_count'] = function(now, rid) + return Qless.resource(rid):pending_count() +end + +QlessAPI['resource.stats_pending'] = function(now) + return cjson.encode(QlessResource.pending_counts(now)) +end + +QlessAPI['resource.stats_locks'] = function(now) + return cjson.encode(QlessResource.locks_counts(now)) +end + + +if #KEYS > 0 then error('No Keys should be provided') end + +local command_name = assert(table.remove(ARGV, 1), 'Must provide a command') +local command = assert( + QlessAPI[command_name], 'Unknown command ' .. command_name) + +local now = tonumber(table.remove(ARGV, 1)) +local now = assert( + now, 'Arg "now" missing or not a number: ' .. (now or 'nil')) + +return command(now, unpack(ARGV))` diff --git a/qless_test.go b/qless_test.go new file mode 100644 index 0000000..5f62767 --- /dev/null +++ b/qless_test.go @@ -0,0 +1,51 @@ +package qless + +import ( + "os" + "strconv" + "testing" +) + +var ( + redisHost string + redisPort string + redisDB int +) + +func init() { + redisHost = os.Getenv("REDIS_HOST") + if redisHost == "" { + panic("invalid REDIS_HOST") + } + redisPort = os.Getenv("REDIS_PORT") + + if redisPort == "" { + panic("invalid REDIS_PORT") + } + + if db, err := strconv.Atoi(os.Getenv("REDIS_DB")); err != nil { + panic("invalid REDIS_DB") + } else { + redisDB = db + } +} + +func newClient() *Client { + c, err := Dial(redisHost, redisPort, redisDB) + if err != nil { + panic(err.Error()) + } + + return c +} + +func flushDB() { + c := newClient() + defer c.Close() + c.conn.Do("FLUSHDB") +} + +func TestMain(m *testing.M) { + flushDB() + os.Exit(m.Run()) +} diff --git a/queue.go b/queue.go old mode 100644 new mode 100755 index d1dedf6..517bc8c --- a/queue.go +++ b/queue.go @@ -1,4 +1,4 @@ -package goqless +package qless import ( "encoding/json" @@ -24,12 +24,8 @@ func NewQueue(cli *Client) *Queue { return &Queue{cli: cli} } -func (q *Queue) SetClient(cli *Client) { - q.cli = cli -} - func (q *Queue) Jobs(state string, start, count int) ([]string, error) { - reply, err := redis.Values(q.cli.Do("qless", 0, "jobs", timestamp(), state, q.Name)) + reply, err := redis.Values(q.cli.Do("jobs", timestamp(), state, q.Name)) if err != nil { return nil, err } @@ -63,70 +59,137 @@ func (q *Queue) CancelAll() { } func (q *Queue) Pause() { - q.cli.Do("qless", 0, "pause", timestamp(), q.Name) + q.cli.Do("pause", timestamp(), q.Name) } func (q *Queue) Unpause() { - q.cli.Do("qless", 0, "unpause", timestamp(), q.Name) + q.cli.Do("unpause", timestamp(), q.Name) } -// Puts a job into the queue -// returns jid, error -func (q *Queue) Put(jid, klass string, data interface{}, delay, priority int, tags []string, retries int, depends []string) (string, error) { - if jid == "" { - jid = generateJID() +type putData struct { + jid string + delay int + args []interface{} +} + +func newPutData() putData { + return putData{} +} + +func (p *putData) setOptions(opt []putOptionFn) { + for _, fn := range opt { + fn(p) } - if delay == -1 { - delay = 0 + + if p.jid == "" { + p.jid = generateJID() } - if priority == -1 { - priority = 0 +} + +type putOptionFn func(d *putData) + +func putOptionNoOp(*putData) {} + +func PutJID(v string) putOptionFn { + return func(p *putData) { + p.jid = v } - if retries == -1 { - retries = 5 +} + +func PutDelay(v int) putOptionFn { + return func(p *putData) { + p.delay = v } +} - return redis.String(q.cli.Do( - "qless", 0, "put", timestamp(), q.Name, jid, klass, - marshal(data), - delay, "priority", priority, - "tags", marshal(tags), "retries", - retries, "depends", marshal(depends))) +func PutPriority(v int) putOptionFn { + return func(p *putData) { + p.args = append(p.args, "priority", v) + } +} + +func PutRetries(v int) putOptionFn { + return func(p *putData) { + p.args = append(p.args, "retries", v) + } +} + +func PutTags(v []string) putOptionFn { + if v == nil { + return putOptionNoOp + } + return func(p *putData) { + p.args = append(p.args, "tags", marshal(v)) + } +} + +func PutDepends(v []string) putOptionFn { + if v == nil { + return putOptionNoOp + } + return func(p *putData) { + p.args = append(p.args, "depends", marshal(v)) + } +} + +func PutResources(v []string) putOptionFn { + if v == nil { + return putOptionNoOp + } + return func(p *putData) { + p.args = append(p.args, "resources", marshal(v)) + } +} + +// Put enqueues a job to the named queue +func (q *Queue) Put(class string, data interface{}, opt ...putOptionFn) (string, error) { + pd := newPutData() + pd.setOptions(opt) + args := []interface{}{"put", timestamp(), "", q.Name, pd.jid, class, marshal(data), pd.delay} + args = append(args, pd.args...) + + return redis.String(q.cli.Do(args...)) +} + +func (q *Queue) PopOne() (j Job, err error) { + var jobs []Job + if jobs, err = q.Pop(1); err == nil && len(jobs) == 1 { + j = jobs[0] + } + return } // Pops a job off the queue. -func (q *Queue) Pop(count int) ([]*Job, error) { +func (q *Queue) Pop(count int) ([]Job, error) { if count == 0 { count = 1 } - reply, err := redis.Bytes(q.cli.Do("qless", 0, "pop", timestamp(), q.Name, workerName(), count)) + reply, err := redis.Bytes(q.cli.Do("pop", timestamp(), q.Name, workerName(), count)) if err != nil { return nil, err } - //"{}" if len(reply) == 2 { return nil, nil } - //println(string(reply)) - - var jobs []*Job - err = json.Unmarshal(reply, &jobs) + var jobsData []jobData + err = json.Unmarshal(reply, &jobsData) if err != nil { return nil, err } - for _, v := range jobs { - v.cli = q.cli + jobs := make([]Job, len(jobsData)) + for i, v := range jobsData { + jobs[i] = &job{&v, q.cli} } return jobs, nil } // Put a recurring job in this queue -func (q *Queue) Recur(jid, klass string, data interface{}, interval, offset, priority int, tags []string, retries int) (string, error) { +func (q *Queue) Recur(jid, class string, data interface{}, interval, offset, priority int, tags []string, retries int) (string, error) { if jid == "" { jid = generateJID() } @@ -144,7 +207,7 @@ func (q *Queue) Recur(jid, klass string, data interface{}, interval, offset, pri } return redis.String(q.cli.Do( - "qless", 0, "recur", timestamp(), "on", q.Name, jid, klass, + "recur", timestamp(), "on", q.Name, jid, class, data, "interval", interval, offset, "priority", priority, "tags", marshal(tags), "retries", retries)) diff --git a/queue_test.go b/queue_test.go new file mode 100644 index 0000000..46be3dd --- /dev/null +++ b/queue_test.go @@ -0,0 +1,19 @@ +package qless + +import ( + "testing" + "github.com/stretchr/testify/assert" + "fmt" +) + +func TestQueue_PushAndPop(t *testing.T) { + c := newClient() + q := c.Queue("test_queue") + j, err := q.Put("class", "data", PutJID("jid")) + assert.NoError(t, err) + fmt.Println(j) + job, err := q.PopOne() + assert.NoError(t, err) + assert.NotNil(t, job) + fmt.Println(job.Data()) +} diff --git a/resource.go b/resource.go new file mode 100644 index 0000000..1c6693d --- /dev/null +++ b/resource.go @@ -0,0 +1 @@ +package qless diff --git a/structs.go b/structs.go new file mode 100644 index 0000000..90f4c7f --- /dev/null +++ b/structs.go @@ -0,0 +1,22 @@ +package qless + +//easyjson:json +type jobData struct { + Jid string + Klass string + State string + Queue string + Worker string + Tracked bool + Priority int + Expires int64 + Retries int + Remaining int + Data interface{} + Tags StringSlice + History []History + Failure interface{} + Dependents StringSlice + Dependencies interface{} +} + diff --git a/worker.go b/worker.go old mode 100644 new mode 100755 index 081856b..0f672f5 --- a/worker.go +++ b/worker.go @@ -1,6 +1,6 @@ // This worker does not model the way qless does it. // I more or less modeled it after my own needs. -package goqless +package qless import ( "errors" @@ -17,8 +17,8 @@ func init() { log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) } -type JobFunc func(*Job) error -type JobCallback func(*Job) error +type JobFunc func(Job) error +type JobCallback func(Job) error type Worker struct { sync.Mutex @@ -34,7 +34,7 @@ type Worker struct { func NewWorker(queueAddr string, queueName string, interval int) (*Worker, error) { ipport := strings.Split(queueAddr, ":") - client, err := Dial(ipport[0], ipport[1]) + client, err := Dial(ipport[0], ipport[1], 0) if err != nil { log.Println("Dial err:", err) return nil, errors.New(err.Error()) @@ -54,7 +54,7 @@ func NewWorker(queueAddr string, queueName string, interval int) (*Worker, error return w, nil } -func heartbeatStart(job *Job, done chan bool, heartbeat int, l sync.Locker) { +func heartbeatStart(job Job, done chan bool, heartbeat int, l sync.Locker) { tick := time.NewTicker(time.Duration(heartbeat) * time.Duration(time.Second)) for { select { @@ -67,38 +67,37 @@ func heartbeatStart(job *Job, done chan bool, heartbeat int, l sync.Locker) { l.Unlock() if err != nil { log.Printf("failed HeartbeatWithNoData jid:%v, queue:%v, success:%v, error:%v", - job.Jid, job.Queue, success, err) + job.JID(), job.Queue(), success, err) } else { log.Printf("warning, slow, HeartbeatWithNoData jid:%v, queue:%v, success:%v", - job.Jid, job.Queue, success) + job.JID(), job.Queue(), success) } } } } //try our best to complete the job -func (w *Worker) tryCompleteJob(job *Job) error { +func (w *Worker) tryCompleteJob(job Job) error { for i := 0; i < 3; i++ { time.Sleep(time.Second) - log.Println("tryCompleteJob", job.Jid) + log.Println("tryCompleteJob", job.JID()) ipport := strings.Split(w.queueAddr, ":") - client, err := Dial(ipport[0], ipport[1]) + client, err := Dial(ipport[0], ipport[1], 0) if err != nil { log.Println("Dial err:", err) continue } - job.SetClient(client) if _, err := job.CompleteWithNoData(); err != nil { client.Close() - log.Println("tryCompleteJob", job.Jid, err) + log.Println("tryCompleteJob", job.JID(), err) } else { client.Close() return nil } } - return errors.New("tryCompleteJob " + job.Jid + " failed") + return errors.New("tryCompleteJob " + job.JID() + " failed") } func (w *Worker) Start() error { @@ -133,13 +132,13 @@ func (w *Worker) Start() error { } for i := 0; i < len(jobs); i++ { - if len(jobs[i].History) > 2 { + if len(jobs[i].History()) > 2 { log.Printf("warning, multiple processed exist %+v\n", jobs[i]) } done := make(chan bool) //todo: using seprate connection to send heartbeat go heartbeatStart(jobs[i], done, heartbeat, w) - f, ok := w.funcs[jobs[i].Klass] + f, ok := w.funcs[jobs[i].Class()] if !ok { //we got a job that not belongs to us done <- false log.Fatalf("got a message not belongs to us, queue %v, job %+v\n", q.Name, jobs[i]) @@ -149,14 +148,14 @@ func (w *Worker) Start() error { err := f(jobs[i]) if err != nil { // TODO: probably do something with this - log.Println("error: job failed, jid", jobs[i].Jid, "queue", jobs[i].Queue, err.Error()) + log.Println("error: job failed, jid", jobs[i].JID(), "queue", jobs[i].Queue(), err.Error()) w.Lock() success, err := jobs[i].Fail("fail", err.Error()) w.Unlock() done <- false if err != nil { log.Printf("fail job:%+v success:%v, error:%v", - jobs[i].Jid, success, err) + jobs[i].JID(), success, err) return err } } else { @@ -168,14 +167,14 @@ func (w *Worker) Start() error { err = w.tryCompleteJob(jobs[i]) if err != nil { log.Printf("fail job:%+v status:%v, error:%v", - jobs[i].Jid, status, err) + jobs[i].JID(), status, err) } else { - log.Println("retry complete job ", jobs[i].Jid, "ok") + log.Println("retry complete job ", jobs[i].JID(), "ok") } return errors.New("restart") } else { if status != "complete" { - log.Printf("job:%+v status:%v", jobs[i].Jid, status) + log.Printf("job:%+v status:%v", jobs[i].JID(), status) } } } @@ -203,7 +202,7 @@ func (w *Worker) AddService(name string, rcvr interface{}) error { val := reflect.ValueOf(rcvr) for i := 0; i < typ.NumMethod(); i++ { method := typ.Method(i) - w.AddFunc(name+"."+method.Name, func(job *Job) error { + w.AddFunc(name+"."+method.Name, func(job Job) error { ret := method.Func.Call([]reflect.Value{val, reflect.ValueOf(job)}) if len(ret) > 0 { if err, ok := ret[0].Interface().(error); ok {