Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

work in progress - remote run #3986

Draft
wants to merge 14 commits into
base: master
Choose a base branch
from
4 changes: 2 additions & 2 deletions paasta_tools/api/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,8 @@ def make_app(global_config=None):
"/v1/services/{service}/{instance}/tasks/{task_id}",
)
config.add_route(
"service.instance.remote_run",
"/v1/services/{service}/{instance}/remote_run",
"remote_run",
"/v1/remote_run/{service}/{instance}",
Qmando marked this conversation as resolved.
Show resolved Hide resolved
)
config.add_route("service.list", "/v1/services/{service}")
config.add_route("services", "/v1/services")
Expand Down
83 changes: 80 additions & 3 deletions paasta_tools/api/api_docs/oapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1627,9 +1627,9 @@ paths:
summary: Get mesos task of service_name.instance_name by task_id
tags:
- service
/services/{service}/{instance}/remote_run:
/remote_run/{service}/{instance}/start:
post:
operationId: remote_run
operationId: remote_run_start
requestBody:
content:
application/json:
Expand Down Expand Up @@ -1672,7 +1672,84 @@ paths:
description: Failure
summary: Launch a remote-run pod
tags:
- service
- remote_run
/remote_run/{service}/{instance}/stop:
post:
operationId: remote_run_sop
requestBody:
content:
application/json:
schema:
type: object
properties:
user:
type: string
required:
- user
required: true
parameters:
- description: Service name
in: path
name: service
required: true
schema:
type: string
- description: Instance name
in: path
name: instance
required: true
schema:
type: string
responses:
"200":
content:
application/json:
schema:
type: string
description: Remote run pod stopped
"404":
description: Deployment key not found
"500":
description: Failure
summary: Stop a remote-run pod
tags:
- remote_run
/remote_run/{service}/{instance}/token:
get:
operationId: remote_run_token
parameters:
- description: Service name
in: path
name: service
required: true
schema:
type: string
- description: Instance name
in: path
name: instance
required: true
schema:
type: string
- description: User name
in: query
name: user
schema:
type: string
required: true
responses:
"200":
content:
application/json:
schema:
type: string
description: Token generated successfully
"404":
description: Deployment key not found
"500":
description: Failure
summary: Get a short lived token for exec into remote-run pod
tags:
- remote_run
/version:
get:
operationId: showVersion
Expand Down
96 changes: 94 additions & 2 deletions paasta_tools/api/api_docs/swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -847,7 +847,7 @@
]
}
},
"/services/{service}/{instance}/remote_run": {
"/remote_run/{service}/{instance}/start": {
"post": {
"responses": {
"200": {
Expand All @@ -864,7 +864,53 @@
}
},
"summary": "Do a remote run",
"operationId": "remote_run",
"operationId": "remote_run_start",
"tags": [
"remote_run"
],
"parameters": [
{
"in": "path",
"description": "Service name",
"name": "service",
"required": true,
"type": "string"
},
{
"in": "path",
"description": "Instance name",
"name": "instance",
"required": true,
"type": "string"
},
{
"in": "query",
"description": "Username",
"name": "user",
"required": true,
"type": "string"
}
]
}
},
"/remote_run/{service}/{instance}/stop": {
"post": {
"responses": {
"200": {
"description": "It worked!",
"schema": {
"type": "string"
}
},
"404": {
"description": "Deployment key not found"
},
"500": {
"description": "Failure"
}
},
"summary": "Stop a remote run",
"operationId": "remote_run_stop",
"tags": [
"service"
],
Expand Down Expand Up @@ -892,6 +938,52 @@
}
]
}
},
"/remote_run/{service}/{instance}/token": {
"get": {
"responses": {
"200": {
"description": "It worked!",
"schema": {
"type": "string"
}
},
"404": {
"description": "Deployment key not found"
},
"500": {
"description": "Failure"
}
},
"summary": "Get a remote run token",
"operationId": "remote_run_token",
"tags": [
"remote_run"
],
"parameters": [
{
"in": "path",
"description": "Service name",
"name": "service",
"required": true,
"type": "string"
},
{
"in": "path",
"description": "Instance name",
"name": "instance",
"required": true,
"type": "string"
},
{
"in": "query",
"description": "Username",
"name": "user",
"required": true,
"type": "string"
}
]
}
}
},
"definitions": {
Expand Down
16 changes: 12 additions & 4 deletions paasta_tools/api/views/instance.py
Original file line number Diff line number Diff line change
Expand Up @@ -388,10 +388,18 @@ def instance_mesh_status(request):
return instance_mesh


@view_config(
route_name="service.instance.remote_run", request_method="POST", renderer="json"
)
def remote_run(request):
@view_config(route_name="remote_run.stop", request_method="POST", renderer="json")
def remote_run_stop(request):
pass


@view_config(route_name="remote_run.token", request_method="POST", renderer="json")
def remote_run_token(request):
pass


@view_config(route_name="remote_run.start", request_method="POST", renderer="json")
def remote_run_start(request):
service = request.swagger_data.get("service")
instance = request.swagger_data.get("instance")
user = request.swagger_data.get("user")
Expand Down
58 changes: 56 additions & 2 deletions paasta_tools/kubernetes/application/controller_wrappers.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,14 @@
from kubernetes.client import V1beta1PodDisruptionBudget
from kubernetes.client import V1DeleteOptions
from kubernetes.client import V1Deployment
from kubernetes.client import V1Job
from kubernetes.client import V1StatefulSet
from kubernetes.client.rest import ApiException

from paasta_tools.autoscaling.autoscaling_service_lib import autoscaling_is_paused
from paasta_tools.eks_tools import load_eks_service_config_no_cache
from paasta_tools.kubernetes_tools import create_deployment
from paasta_tools.kubernetes_tools import create_job
from paasta_tools.kubernetes_tools import create_pod_disruption_budget
from paasta_tools.kubernetes_tools import create_stateful_set
from paasta_tools.kubernetes_tools import ensure_service_account
Expand Down Expand Up @@ -51,7 +53,6 @@ def __init__(
"config_sha",
]
}

replicas = (
item.spec.replicas
if item.metadata.labels.get(paasta_prefixed("autoscaled"), "false")
Expand Down Expand Up @@ -411,14 +412,67 @@ def update(self, kube_client: KubeClient):
)


class JobWrapper(Application):
def __init__(
self,
item: V1Job,
logging=logging.getLogger(__name__),
):
item.spec.replicas = None
super().__init__(item, logging)

def deep_delete(self, kube_client: KubeClient) -> None:
"""
Remove all controllers, pods, and pod disruption budgets related to this application
:param kube_client:
"""
delete_options = V1DeleteOptions(propagation_policy="Foreground")
try:
kube_client.batchess.delete_namespaced_job(
Qmando marked this conversation as resolved.
Show resolved Hide resolved
self.item.metadata.name,
self.item.metadata.namespace,
body=delete_options,
)
except ApiException as e:
if e.status == 404:
# Job does not exist, nothing to delete but
# we can consider this a success.
self.logging.debug(
"not deleting nonexistent job/{} from namespace/{}".format(
self.item.metadata.name, self.item.metadata.namespace
)
)
else:
raise
else:
self.logging.info(
"deleted job/{} from namespace/{}".format(
self.item.metadata.name, self.item.metadata.namespace
)
)

def create(self, kube_client: KubeClient):
create_job(
kube_client=kube_client,
formatted_job=self.item,
namespace=self.soa_config.get_namespace(),
)

def update(self, kube_client: KubeClient):
# TODO: Is this needed?
Qmando marked this conversation as resolved.
Show resolved Hide resolved
pass


def get_application_wrapper(
formatted_application: Union[V1Deployment, V1StatefulSet]
formatted_application: Union[V1Deployment, V1StatefulSet, V1Job]
) -> Application:
app: Application
if isinstance(formatted_application, V1Deployment):
app = DeploymentWrapper(formatted_application)
elif isinstance(formatted_application, V1StatefulSet):
app = StatefulSetWrapper(formatted_application)
elif isinstance(formatted_application, V1Job):
app = JobWrapper(formatted_application)
else:
raise Exception("Unknown kubernetes object to update")

Expand Down
Loading
Loading