performance monitor package #2046
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| # Perf | ||
|
|
||
| Perf is a performance test module that is scheduled to run and cache those tests results in redis which can be retrieved later over RMB call. | ||
|
|
||
| Perf tests are monitored by `noded` service from zos modules. | ||
|
|
||
| ### Usage Example | ||
|
|
||
| 1. Create a task that implement `Task` interface | ||
|
|
||
| ```go | ||
| type LoggingTask struct { | ||
| TaskID string | ||
| Schedule string // should be in cron syntax with seconds ("* 0 * * * *") | ||
| } | ||
|
|
||
| func (t *LoggingTask) ID() string { | ||
| return t.TaskID | ||
| } | ||
|
|
||
| func (t *LoggingTask) Cron() string { | ||
| return t.Schedule | ||
| } | ||
|
|
||
| // a simple task that returns a string with the current time | ||
| func (t *LoggingTask) Run(ctx context.Context) (interface{}, error) { | ||
| result := fmt.Sprintf("time is: %v", time.Now()) | ||
| return result, nil | ||
| } | ||
| ``` | ||
|
|
||
| 2. Add the created task to scheduler | ||
|
|
||
| ```go | ||
| perfMon.AddTask(&perf.LoggingTask{ | ||
| TaskID: "LoggingTask", | ||
| Schedule: "* 0 * * * *", // when minutes is 0 (aka: every hour) | ||
| }) | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,103 @@ | ||||||
| package perf | ||||||
|
|
||||||
| import ( | ||||||
| "context" | ||||||
| "encoding/json" | ||||||
| "fmt" | ||||||
|
|
||||||
| "github.com/garyburd/redigo/redis" | ||||||
| "github.com/pkg/errors" | ||||||
| ) | ||||||
|
|
||||||
| const ( | ||||||
| moduleName = "perf" | ||||||
| ) | ||||||
|
|
||||||
| // TaskResult the result test schema | ||||||
| type TaskResult struct { | ||||||
| TaskName string `json:"task_name"` | ||||||
|
There was a problem hiding this comment. The type is already name TaskResult, so I can perfectly call this
Suggested change
|
||||||
| Timestamp uint64 `json:"timestamp"` | ||||||
| Result interface{} `json:"result"` | ||||||
| } | ||||||
|
|
||||||
| func generateKey(taskName string) string { | ||||||
| return fmt.Sprintf("%s.%s", moduleName, taskName) | ||||||
| } | ||||||
|
|
||||||
| // setCache set result in redis | ||||||
| func (pm *PerformanceMonitor) setCache(ctx context.Context, result TaskResult) error { | ||||||
| data, err := json.Marshal(result) | ||||||
| if err != nil { | ||||||
| return errors.Wrap(err, "failed to marshal data to JSON") | ||||||
| } | ||||||
|
|
||||||
| conn := pm.pool.Get() | ||||||
| defer conn.Close() | ||||||
|
|
||||||
| _, err = conn.Do("SET", generateKey(result.TaskName), data) | ||||||
| return err | ||||||
| } | ||||||
|
|
||||||
| // Get gets data from redis | ||||||
| func (pm *PerformanceMonitor) Get(taskName string) (TaskResult, error) { | ||||||
| var res TaskResult | ||||||
|
|
||||||
| conn := pm.pool.Get() | ||||||
| defer conn.Close() | ||||||
|
|
||||||
| data, err := conn.Do("GET", generateKey(taskName)) | ||||||
| if err != nil { | ||||||
| return res, errors.Wrap(err, "failed to get the result") | ||||||
| } | ||||||
|
|
||||||
| err = json.Unmarshal(data.([]byte), &res) | ||||||
| if err != nil { | ||||||
| return res, errors.Wrap(err, "failed to unmarshal data from json") | ||||||
| } | ||||||
|
|
||||||
| return res, nil | ||||||
| } | ||||||
|
|
||||||
| func (pm *PerformanceMonitor) GetAll() ([]TaskResult, error) { | ||||||
| var res []TaskResult | ||||||
|
|
||||||
| conn := pm.pool.Get() | ||||||
| defer conn.Close() | ||||||
|
|
||||||
| var keys []string | ||||||
|
|
||||||
| cursor := 0 | ||||||
| for { | ||||||
| values, err := redis.Values(conn.Do("SCAN", cursor, "MATCH", generateKey("*"))) | ||||||
| if err != nil { | ||||||
| return nil, err | ||||||
| } | ||||||
|
|
||||||
| _, err = redis.Scan(values, &cursor, &keys) | ||||||
| if err != nil { | ||||||
| return nil, err | ||||||
| } | ||||||
|
|
||||||
| for _, key := range keys { | ||||||
| value, err := conn.Do("GET", key) | ||||||
| if err != nil { | ||||||
| continue | ||||||
| } | ||||||
|
|
||||||
| var result TaskResult | ||||||
|
|
||||||
| err = json.Unmarshal(value.([]byte), &result) | ||||||
| if err != nil { | ||||||
| return res, errors.Wrap(err, "failed to unmarshal data from json") | ||||||
| } | ||||||
|
|
||||||
|
|
||||||
| res = append(res, result) | ||||||
| } | ||||||
|
|
||||||
| if cursor == 0 { | ||||||
| break | ||||||
| } | ||||||
|
|
||||||
| } | ||||||
| return res, nil | ||||||
| } | ||||||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,69 @@ | ||||||
| package perf | ||||||
|
|
||||||
| import ( | ||||||
| "context" | ||||||
| "time" | ||||||
|
|
||||||
| "github.com/go-co-op/gocron" | ||||||
| "github.com/gomodule/redigo/redis" | ||||||
|
|
||||||
| "github.com/pkg/errors" | ||||||
| "github.com/threefoldtech/zos/pkg/utils" | ||||||
| ) | ||||||
|
|
||||||
| // PerformanceMonitor holds the module data | ||||||
| type PerformanceMonitor struct { | ||||||
| scheduler *gocron.Scheduler | ||||||
| pool redis.Pool | ||||||
|
There was a problem hiding this comment.
Suggested change
check the other comment to see why |
||||||
| tasks []Task | ||||||
| } | ||||||
|
|
||||||
| // NewPerformanceMonitor returns PerformanceMonitor instance | ||||||
| func NewPerformanceMonitor(redisAddr string) (*PerformanceMonitor, error) { | ||||||
| redisPool, err := utils.NewRedisPool(redisAddr) | ||||||
| if err != nil { | ||||||
| return nil, errors.Wrap(err, "failed creating new redis pool") | ||||||
| } | ||||||
|
|
||||||
| scheduler := gocron.NewScheduler(time.UTC) | ||||||
|
|
||||||
| return &PerformanceMonitor{ | ||||||
| scheduler: scheduler, | ||||||
| pool: *redisPool, | ||||||
|
There was a problem hiding this comment. You can't do that here, this fails already in CI. redisPool is already a pointer right? and that's is exactly what u need to keep on the PerfomanceMonitor structure There was a problem hiding this comment. The reason this is done because when u do a It's not caught by the compiler but by the CI tests |
||||||
| tasks: []Task{}, | ||||||
| }, nil | ||||||
| } | ||||||
|
|
||||||
| // AddTask a simple helper method to add new tasks | ||||||
| func (pm *PerformanceMonitor) AddTask(task Task) { | ||||||
| pm.tasks = append(pm.tasks, task) | ||||||
| } | ||||||
|
|
||||||
| // Run adds the tasks to the corn queue and start the scheduler | ||||||
| func (pm *PerformanceMonitor) Run(ctx context.Context) error { | ||||||
| for _, task := range pm.tasks { | ||||||
| _, err := pm.scheduler.CronWithSeconds(task.Cron()).Do(func() error { | ||||||
| res, err := task.Run(ctx) | ||||||
| if err != nil { | ||||||
| return errors.Wrapf(err, "failed to run task: %s", task.ID()) | ||||||
| } | ||||||
|
|
||||||
| err = pm.setCache(ctx, TaskResult{ | ||||||
| TaskName: task.ID(), | ||||||
| Timestamp: uint64(time.Now().Unix()), | ||||||
| Result: res, | ||||||
| }) | ||||||
| if err != nil { | ||||||
| return errors.Wrap(err, "failed to set cache") | ||||||
| } | ||||||
|
|
||||||
| return nil | ||||||
| }) | ||||||
| if err != nil { | ||||||
| return errors.Wrapf(err, "failed to schedule the task: %s", task.ID()) | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| pm.scheduler.StartAsync() | ||||||
| return nil | ||||||
| } | ||||||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| package perf | ||
|
|
||
| import ( | ||
| "context" | ||
| ) | ||
|
|
||
| type Task interface { | ||
| ID() string | ||
| Cron() string | ||
| Run(ctx context.Context) (interface{}, error) | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
While this will work perfectly will in Go . I don't like the fact that the output type changes based on the input (one returns a single object, the other returns a list) which makes it annoying to call from languages that supports static type.
This can be solved in 2 possible ways
zos.perf.getandzos.perf.get_allA 3rd solution would be always return ALL? and then client can filter the test on client side is also a possible option