Skip to content

Commit

Permalink
feat: download tar.gz of workspace
Browse files Browse the repository at this point in the history
  • Loading branch information
amolenaar committed Sep 22, 2023
1 parent b8a15c6 commit 0c811fa
Showing 1 changed file with 108 additions and 7 deletions.
115 changes: 108 additions & 7 deletions backend/capellacollab/cli/ws.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
# SPDX-License-Identifier: Apache-2.0

import contextlib
import select
import time
import uuid

Check warning on line 7 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L4-L7

Added lines #L4 - L7 were not covered by tests

import typer
from kubernetes import client, config
from kubernetes import client, config, stream
from websocket import ABNF

Check warning on line 11 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L9-L11

Added lines #L9 - L11 were not covered by tests

app = typer.Typer()

Check warning on line 13 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L13

Added line #L13 was not covered by tests

Expand All @@ -21,12 +27,24 @@ def list(namespace: str = None):
@app.command()

Check warning on line 27 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L27

Added line #L27 was not covered by tests
def download(volume_name: str, namespace: str = None):
config.load_kube_config()
pod = create_temporary_pod(volume_name, namespace)
print(pod)
v1 = client.CoreV1Api()
mount_path = "/workspace"

Check warning on line 31 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L29-L31

Added lines #L29 - L31 were not covered by tests

with pod_for_volume(volume_name, namespace, mount_path, v1) as pod_name:
print(f"Downloading workspace volume to '{volume_name}.tar.gz'")

Check warning on line 34 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L34

Added line #L34 was not covered by tests

with open(f"{volume_name}.tar.gz", "wb") as outfile:
for data in stream_tar_from_pod(
pod_name, namespace, mount_path, v1
):
outfile.write(data)

Check warning on line 40 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L40

Added line #L40 was not covered by tests


def create_temporary_pod(volume_name: str, namespace: str):
name = f"download-{volume_name}"
@contextlib.contextmanager

Check warning on line 43 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L43

Added line #L43 was not covered by tests
def pod_for_volume(
volume_name: str, namespace: str, mount_path: str, v1: client.CoreV1Api
):
name = f"ws-download-{volume_name}-{uuid.uuid1()}"[:63]

Check warning on line 47 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L47

Added line #L47 was not covered by tests

containers = [

Check warning on line 49 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L49

Added line #L49 was not covered by tests
client.V1Container(
Expand All @@ -36,7 +54,7 @@ def create_temporary_pod(volume_name: str, namespace: str):
volume_mounts=[
client.V1VolumeMount(
name=volume_name,
mount_path="/mnt",
mount_path=mount_path,
read_only=True,
)
],
Expand Down Expand Up @@ -65,7 +83,15 @@ def create_temporary_pod(volume_name: str, namespace: str):
),
)

return client.CoreV1Api().create_namespaced_pod(namespace, pod)
v1.create_namespaced_pod(namespace, pod)

Check warning on line 86 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L86

Added line #L86 was not covered by tests

while not is_pod_ready(name, namespace, v1):
print("Waiting for pod to come online...")
time.sleep(2)

Check warning on line 90 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L89-L90

Added lines #L89 - L90 were not covered by tests

yield name

Check warning on line 92 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L92

Added line #L92 was not covered by tests

v1.delete_namespaced_pod(name, namespace)

Check warning on line 94 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L94

Added line #L94 was not covered by tests


def get_current_namespace():
Expand All @@ -74,3 +100,78 @@ def get_current_namespace():
return active_context["context"]["namespace"]
except KeyError:
return "default"

Check warning on line 102 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L97-L102

Added lines #L97 - L102 were not covered by tests


def is_pod_ready(pod_name, namespace, v1):
try:
pod_status = v1.read_namespaced_pod_status(pod_name, namespace)
return pod_status.status.phase == "Running"
except client.exceptions.ApiException as e:
print(

Check warning on line 110 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L105-L110

Added lines #L105 - L110 were not covered by tests
f"Exception when calling CoreV1Api->read_namespaced_pod_status: {e}"
)
return False

Check warning on line 113 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L113

Added line #L113 was not covered by tests


def stream_tar_from_pod(pod_name, namespace, source_path, v1):
exec_stream = stream.stream(

Check warning on line 117 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L116-L117

Added lines #L116 - L117 were not covered by tests
v1.connect_get_namespaced_pod_exec,
pod_name,
namespace,
command=["tar", "zcf", "-", source_path],
stderr=True,
stdin=True,
stdout=True,
tty=False,
_preload_content=False,
)

try:
reader = WSFileManager(exec_stream)
while True:
out, err, closed = reader.read_bytes()

Check warning on line 132 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L129-L132

Added lines #L129 - L132 were not covered by tests
if out:
yield out

Check warning on line 134 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L134

Added line #L134 was not covered by tests
elif err:
print(err.decode("utf-8", "replace"))

Check warning on line 136 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L136

Added line #L136 was not covered by tests
if closed:
break

Check warning on line 138 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L138

Added line #L138 was not covered by tests
finally:
exec_stream.close()

Check warning on line 140 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L140

Added line #L140 was not covered by tests


class WSFileManager:

Check warning on line 143 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L143

Added line #L143 was not covered by tests
"""WS wrapper to manage read and write bytes in K8s WSClient."""

def __init__(self, ws_client):
self.ws_client = ws_client

Check warning on line 147 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L146-L147

Added lines #L146 - L147 were not covered by tests

def read_bytes(self, timeout=0):
stdout_bytes = None
stderr_bytes = None

Check warning on line 151 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L149-L151

Added lines #L149 - L151 were not covered by tests

if not self.ws_client.is_open():
return stdout_bytes, stderr_bytes, not self.ws_client._connected

Check warning on line 154 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L154

Added line #L154 was not covered by tests

if not self.ws_client.sock.connected:
self.ws_client._connected = False
return stdout_bytes, stderr_bytes, not self.ws_client._connected

Check warning on line 158 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L157-L158

Added lines #L157 - L158 were not covered by tests

r, _, _ = select.select((self.ws_client.sock.sock,), (), (), timeout)

Check warning on line 160 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L160

Added line #L160 was not covered by tests
if not r:
return stdout_bytes, stderr_bytes, not self.ws_client._connected

Check warning on line 162 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L162

Added line #L162 was not covered by tests

op_code, frame = self.ws_client.sock.recv_data_frame(True)

Check warning on line 164 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L164

Added line #L164 was not covered by tests
if op_code == ABNF.OPCODE_CLOSE:
self.ws_client._connected = False

Check warning on line 166 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L166

Added line #L166 was not covered by tests
elif op_code in (ABNF.OPCODE_BINARY, ABNF.OPCODE_TEXT):
data = frame.data

Check warning on line 168 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L168

Added line #L168 was not covered by tests
if len(data) > 1:
channel = data[0]
data = data[1:]

Check warning on line 171 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L170-L171

Added lines #L170 - L171 were not covered by tests
if data:
if channel == stream.ws_client.STDOUT_CHANNEL:
stdout_bytes = data

Check warning on line 174 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L174

Added line #L174 was not covered by tests
elif channel == stream.ws_client.STDERR_CHANNEL:
stderr_bytes = data
return stdout_bytes, stderr_bytes, not self.ws_client._connected

Check warning on line 177 in backend/capellacollab/cli/ws.py

View check run for this annotation

Codecov / codecov/patch

backend/capellacollab/cli/ws.py#L176-L177

Added lines #L176 - L177 were not covered by tests

0 comments on commit 0c811fa

Please sign in to comment.