Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add function to return named slots #13

Merged
merged 8 commits into from
May 10, 2024
Merged

feat: add function to return named slots #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c98d1d5
feat: add function to return named slots
vsoch May 6, 2024
b4da819
wip: named slots
vsoch May 7, 2024
cadb71e
test requires as an array
vsoch May 7, 2024
4e43ba6
remove extra code
vsoch May 8, 2024
6b9f996
ensure slot is represented with replicate
vsoch May 8, 2024
cfb3114
update generation to use replicas
vsoch May 8, 2024
4f0d6ea
ensure generated jobspec has slot at rack
vsoch May 9, 2024
a0e233c
allow attributes on the top level of jobspec
vsoch May 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions examples/nextgen/v1/example1/example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,4 +62,10 @@ func main() {
for _, slot := range slots {
fmt.Println(slot)
}

named := js.GetScheduledNamedSlots()
fmt.Printf("Found %d scheduled named slots\n", len(named))
for name := range named {
fmt.Println(name)
}
}
18 changes: 8 additions & 10 deletions pkg/nextgen/v1/convert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@ func NewSimpleJobspec(name, command string, nodes, tasks int32) (*Jobspec, error
Count: nodes,
}

// The slot is where we are doing an assessment for scheduling
slot := Resource{
Type: "slot",
Count: int32(1),
Label: name,
// But we put it under the slot of a rack
rackResource := Resource{
Type: "rack",
Replicas: 1,
Label: name,
}

// If tasks are defined, this is total tasks across the nodes
Expand All @@ -41,13 +41,11 @@ func NewSimpleJobspec(name, command string, nodes, tasks int32) (*Jobspec, error
Type: "core",
Count: tasks,
}
slot.With = []Resource{taskResource}
nodeResource.With = []Resource{taskResource}
}

// And then the entire resource spec is added to the top level node resource
nodeResource.With = []Resource{slot}

// Resource name matches resources to named set
rackResource.With = []Resource{nodeResource}
resourceName := "task-resources"

// Tasks reference the slot and command
Expand All @@ -63,6 +61,6 @@ func NewSimpleJobspec(name, command string, nodes, tasks int32) (*Jobspec, error
return &Jobspec{
Version: jobspecVersion,
Tasks: tasklist,
Resources: Resources{resourceName: nodeResource},
Resources: Resources{resourceName: rackResource},
}, nil
}
25 changes: 17 additions & 8 deletions pkg/nextgen/v1/jobspec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ func (js *Jobspec) GetSlots() []Resource {
// a slot at the top level. We wrap in a faux slot
fauxSlots = append(fauxSlots, generateFauxSlot(name, resource))

// Slot at the top level already!
if resource.Type == "slot" {
// If replicas > 0, we have a slot at the level already
if resource.Replicas > 0 {
slots = append(slots, resource)
}
for _, with := range resource.With {
Expand Down Expand Up @@ -88,15 +88,24 @@ func (js *Jobspec) GetScheduledSlots() []Resource {
return scheduled
}

// GetScheduledNamedSlots returns slots as a lookup by name
func (js *Jobspec) GetScheduledNamedSlots() map[string]Resource {

slots := js.GetScheduledSlots()
named := map[string]Resource{}
for _, slot := range slots {
named[slot.Label] = slot
}
return named
}

// A fauxSlot will only be use if we don't have any actual slots
func generateFauxSlot(name string, resource Resource) Resource {
return Resource{
Type: "slot",
Label: name,
Count: 1,
Schedule: resource.Schedule,
With: []Resource{resource},
resource.Replicas = 1
if resource.Label == "" {
resource.Label = name
}
return resource
}

// getSlots is a recursive helper function that takes resources explicitly
Expand Down
59 changes: 21 additions & 38 deletions pkg/nextgen/v1/schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,15 @@
"type": "integer",
"enum": [1]
},
"requires": {"$ref": "#/definitions/requires"},
"resources": {
"type": "object",
"patternProperties": {
"^([a-z]|[|]|&|[0-9]+)+$": {"$ref": "#/definitions/resources"}
}
},
"attributes": {"$ref": "#/definitions/attributes"},
"groups": {"type": "array", "items": {"$ref": "#/definitions/group"}},
"tasks": {"$ref": "#/definitions/tasks"},
"attributes": {"$ref": "#/definitions/attributes"},
"additionalProperties": false
},
"definitions": {
Expand All @@ -34,31 +33,28 @@
"environment": {"type": "object"}
}
},
"requires": {
"description": "compatibility requirements",
"type": "object"
},
"resources": {
"description": "requested resources",
"type": "object",
"required": ["type", "count"],
"required": ["type"],
"properties": {
"type": {"type": "string"},
"schedule": {"type": "boolean"},
"exclusive": {"type": "boolean"},
"count": {"type": "integer", "minimum": 1},
"unit": {"type": "string"},
"with": {"$ref": "#/definitions/with"}
},
"additionalProperties": false
},
"with": {
"type": "array",
"minItems": 1,
"items": {"$ref": "#/definitions/resources"}
"requires": {
"type": "array",
"items": {"type": "object"}
},
"attributes": {"$ref": "#/definitions/attributes"},
"schedule": {"type": "boolean"},
"with": {
"type": "array",
"minItems": 1,
"items": {"$ref": "#/definitions/resources"}
}
}
},
"steps": {
"type": "array",
"type": ["array"],
"items": {
"type": "object",
"properties": {
Expand All @@ -76,15 +72,9 @@
"items": {
"type": "object",
"properties": {
"requires": {"$ref": "#/definitions/requires"},
"resources": {
"oneOf": [
{"$ref": "#/definitions/resources"},
{"type": "string"}
]
},
"attributes": {"$ref": "#/definitions/attributes"},
"resources": {"type": "string"},
"group": {"type": "string"},
"local": {"type": "boolean"},
"name": {"type": "string"},
"depends_on": {"type": "array", "items": {"type": "string"}},
"replicas": {"type": "number", "minimum": 1, "default": 1},
Expand All @@ -102,18 +92,11 @@
"type": "object",
"properties": {
"name": {"type": "string"},
"requires": {"$ref": "#/definitions/requires"},
"resources": {
"oneOf": [
{"$ref": "#/definitions/resources"},
{"type": "string"}
]
},
"attributes": {"$ref": "#/definitions/attributes"},
"resources": {"type": "string"},
"depends_on": {"type": "array", "items": {"type": "string"}},
"tasks": {"$ref": "#/definitions/tasks"}
"tasks": {"$ref": "#/definitions/tasks"},
"groups": {"type": "array", "items": {"$ref": "#/definitions/group"}}
},
"additionalProperties": false
}
"additionalProperties": false }
}
}
52 changes: 29 additions & 23 deletions pkg/nextgen/v1/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,44 +4,50 @@ var (
jobspecVersion = 1
)

// The JobSpec is what the user writes to describe their work
type Jobspec struct {
Version int `json:"version" yaml:"version"`
Name string `json:"name,omitempty" yaml:"name,omitempty"`
Resources Resources `json:"resources,omitempty" yaml:"resources,omitempty"`
Tasks Tasks `json:"tasks,omitempty" yaml:"tasks,omitempty"`
Groups Groups `json:"groups,omitempty" yaml:"groups,omitempty"`
Requires map[string]string `json:"requires,omitempty" yaml:"requires,omitempty"`
Version int `json:"version" yaml:"version"`
Name string `json:"name,omitempty" yaml:"name,omitempty"`
Resources Resources `json:"resources,omitempty" yaml:"resources,omitempty"`
Tasks Tasks `json:"tasks,omitempty" yaml:"tasks,omitempty"`
Groups Groups `json:"groups,omitempty" yaml:"groups,omitempty"`
Requires []Requires `json:"requires,omitempty" yaml:"requires,omitempty"`
Attributes Attributes `json:"attributes,omitempty" yaml:"attributes,omitempty"`
}

type Environment map[string]string
type Resources map[string]Resource
type Requires map[string]string
type Tasks []Task
type Groups []Group

type Task struct {
Group string `json:"group,omitempty" yaml:"group,omitempty"`
Name string `json:"name,omitempty" yaml:"name,omitempty"`
Replicas int `json:"replicas,omitempty" yaml:"replicas,omitempty"`
Resources string `json:"resources,omitempty" yaml:"resources,omitempty"`
Command []string `json:"command,omitempty" yaml:"command,omitempty"`
Attributes Attributes `json:"attributes,omitempty" yaml:"attributes,omitempty"`
Group string `json:"group,omitempty" yaml:"group,omitempty"`
Name string `json:"name,omitempty" yaml:"name,omitempty"`
Replicas int `json:"replicas,omitempty" yaml:"replicas,omitempty"`
Resources string `json:"resources,omitempty" yaml:"resources,omitempty"`
Command []string `json:"command,omitempty" yaml:"command,omitempty"`
Requires []Requires `json:"requires,omitempty" yaml:"requires,omitempty"`
}

type Group struct {
Name string `json:"name,omitempty" yaml:"name,omitempty"`
Resources string `json:"resources,omitempty" yaml:"resources,omitempty"`
Tasks Tasks `json:"tasks,omitempty" yaml:"tasks,omitempty"`
Attributes Attributes `json:"attributes,omitempty" yaml:"attributes,omitempty"`
Name string `json:"name,omitempty" yaml:"name,omitempty"`
Tasks Tasks `json:"tasks,omitempty" yaml:"tasks,omitempty"`
Groups Groups `json:"groups,omitempty" yaml:"groups,omitempty"`
Resources string `json:"resources,omitempty" yaml:"resources,omitempty"`
}

type Resource struct {
Type string `yaml:"type,omitempty" json:"type,omitempty"`
Unit string `yaml:"unit,omitempty" json:"unit,omitempty"`
Count int32 `yaml:"count,omitempty" json:"count,omitempty"`
With []Resource `yaml:"with,omitempty" json:"with,omitempty"`
Label string `yaml:"label,omitempty" json:"label,omitempty"`
Exclusive bool `yaml:"exclusive,omitempty" json:"exclusive,omitempty"`
Schedule bool `yaml:"schedule,omitempty" json:"schedule,omitempty"`
Type string `yaml:"type,omitempty" json:"type,omitempty"`
Unit string `yaml:"unit,omitempty" json:"unit,omitempty"`
Replicas int32 `yaml:"replicas,omitempty" json:"replicas,omitempty"`
Count int32 `yaml:"count,omitempty" json:"count,omitempty"`
With []Resource `yaml:"with,omitempty" json:"with,omitempty"`
Label string `yaml:"label,omitempty" json:"label,omitempty"`
Exclusive bool `yaml:"exclusive,omitempty" json:"exclusive,omitempty"`
Requires []Requires `json:"requires,omitempty" yaml:"requires,omitempty"`
Attributes Attributes `json:"attributes,omitempty" yaml:"attributes,omitempty"`
Schedule bool `yaml:"schedule,omitempty" json:"schedule,omitempty"`
}

type Attributes struct {
Expand Down
Loading