Skip to content

Commit

Permalink
Delete remaining Mesos code (#961)
Browse files Browse the repository at this point in the history 
We don't run Mesos clusters anymore (and haven't for a while), let's
clean up all this cruft.
  • Loading branch information
@nemacysts
nemacysts authored May 17, 2024
1 parent 39bbf9a commit 1f71d0f
Show file tree
Hide file tree
Showing 17 changed files with 38 additions and 2,213 deletions.
2 changes: 1 addition & 1 deletion requirements-minimal.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ pytimeparse
pytz
PyYAML>=5.1
requests
task_processing[mesos_executor,k8s]
task_processing[k8s]
Twisted>=19.7.0
urllib3>=1.24.2
Werkzeug>=0.15.3
3 changes: 0 additions & 3 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
addict==2.2.1
argcomplete==1.9.5
asttokens==2.2.1
attrs==19.3.0
Expand All @@ -25,7 +24,6 @@ ecdsa==0.13.3
executing==1.2.0
future==0.18.3
google-auth==1.23.0
http-parser==0.9.0
humanize==0.5.1
hyperlink==19.0.0
idna==2.8
Expand Down Expand Up @@ -62,7 +60,6 @@ pyasn1-modules==0.2.8
pycparser==2.19
pyformance==0.4
Pygments==2.13.0
pymesos==0.3.9
pyrsistent==0.15.4
pysensu-yelp==0.4.4
PyStaticConfiguration==0.10.4
Expand Down
138 changes: 9 additions & 129 deletions tests/config/config_parse_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,20 +101,6 @@ def make_node_pools():
}


def make_mesos_options():
return schema.ConfigMesos(
master_address=None,
master_port=5050,
secret_file=None,
role="*",
principal="tron",
enabled=False,
default_volumes=(),
dockercfg_location=None,
offer_timeout=300,
)


def make_k8s_options():
return schema.ConfigKubernetes(enabled=False, default_volumes=())

Expand Down Expand Up @@ -162,7 +148,6 @@ def make_job(**kwargs):
kwargs.setdefault("allow_overlap", False)
kwargs.setdefault("time_zone", None)
kwargs.setdefault("expected_runtime", datetime.timedelta(0, 3600))
kwargs.setdefault("use_k8s", False)
return schema.ConfigJob(**kwargs)


Expand Down Expand Up @@ -243,31 +228,6 @@ def make_master_jobs():
cleanup_action=None,
expected_runtime=datetime.timedelta(1),
),
"MASTER.test_job_mesos": make_job(
name="MASTER.test_job_mesos",
node="NodePool",
schedule=schedule_parse.ConfigDailyScheduler(
original="00:00:00 ",
hour=0,
minute=0,
second=0,
days=set(),
jitter=None,
),
actions={
"action_mesos": make_action(
name="action_mesos",
command="test_command_mesos",
executor=schema.ExecutorTypes.mesos.value,
cpus=0.1,
mem=100,
disk=600,
docker_image="container:latest",
),
},
cleanup_action=None,
expected_runtime=datetime.timedelta(1),
),
"MASTER.test_job_k8s": make_job(
name="MASTER.test_job_k8s",
node="NodePool",
Expand Down Expand Up @@ -408,22 +368,6 @@ class ConfigTestCase(TestCase):
enabled=False,
actions=[dict(name="action", command="command")],
),
dict(
name="test_job_mesos",
node="NodePool",
schedule="daily",
actions=[
dict(
name="action_mesos",
executor="mesos",
command="test_command_mesos",
cpus=0.1,
mem=100,
disk=600,
docker_image="container:latest",
),
],
),
dict(
name="test_job_k8s",
node="NodePool",
Expand Down Expand Up @@ -481,7 +425,7 @@ def test_attributes(self):
assert test_config.nodes == expected.nodes
assert test_config.node_pools == expected.node_pools
assert test_config.k8s_options == expected.k8s_options
for key in ["0", "1", "2", "_actions_dict", "4", "_mesos"]:
for key in ["0", "1", "2", "_actions_dict", "4", "_k8s"]:
job_name = f"MASTER.test_job{key}"
assert job_name in test_config.jobs, f"{job_name} in test_config.jobs"
assert job_name in expected.jobs, f"{job_name} in test_config.jobs"
Expand Down Expand Up @@ -1058,24 +1002,13 @@ def test_valid_jobs_success(self):
expected_runtime="20m",
),
dict(
name="action_mesos",
name="action_k8s",
command="command",
executor="mesos",
executor="kubernetes",
cpus=4,
mem=300,
disk=600,
constraints=[
dict(
attribute="pool",
operator="LIKE",
value="default",
),
],
docker_image="my_container:latest",
docker_parameters=[
dict(key="label", value="labelA"),
dict(key="label", value="labelB"),
],
env=dict(USER="batch"),
extra_volumes=[
dict(
Expand Down Expand Up @@ -1106,30 +1039,13 @@ def test_valid_jobs_success(self):
"action": make_action(
expected_runtime=datetime.timedelta(0, 1200),
),
"action_mesos": make_action(
name="action_mesos",
executor=schema.ExecutorTypes.mesos.value,
"action_k8s": make_action(
name="action_k8s",
executor=schema.ExecutorTypes.kubernetes.value,
cpus=4.0,
mem=300.0,
disk=600.0,
constraints=(
schema.ConfigConstraint(
attribute="pool",
operator="LIKE",
value="default",
),
),
docker_image="my_container:latest",
docker_parameters=(
schema.ConfigParameter(
key="label",
value="labelA",
),
schema.ConfigParameter(
key="label",
value="labelB",
),
),
env={"USER": "batch"},
extra_volumes=(
schema.ConfigVolume(
Expand Down Expand Up @@ -1161,12 +1077,12 @@ def test_valid_jobs_success(self):
assert expected_jobs == test_config["jobs"]


class TestValidMesosAction(TestCase):
class TestValidKubernetesAction(TestCase):
def test_missing_docker_image(self):
config = dict(
name="test_missing",
command="echo hello",
executor="mesos",
executor="kubernetes",
cpus=0.2,
mem=150,
disk=450,
Expand All @@ -1177,7 +1093,7 @@ def test_missing_docker_image(self):
def test_cleanup_missing_docker_image(self):
config = dict(
command="echo hello",
executor="mesos",
executor="kubernetes",
cpus=0.2,
mem=150,
disk=450,
Expand Down Expand Up @@ -1338,7 +1254,6 @@ def test_get_job_names(self):
"test_job_actions_dict",
"test_job2",
"test_job4",
"test_job_mesos",
"test_job_k8s",
]
assert_equal(set(job_names), set(expected))
Expand All @@ -1350,7 +1265,6 @@ def test_get_jobs(self):
"test_job_actions_dict",
"test_job2",
"test_job4",
"test_job_mesos",
"test_job_k8s",
]
assert_equal(set(expected), set(self.container.get_jobs().keys()))
Expand Down Expand Up @@ -1677,39 +1591,5 @@ def test_valid(self, config):
config_parse.valid_secret_volume(config, NullConfigContext)


class TestValidMasterAddress:
@pytest.fixture
def context(self):
return config_utils.NullConfigContext

@pytest.mark.parametrize(
"url",
[
"http://blah.com",
"http://blah.com/",
"blah.com",
"blah.com/",
],
)
def test_valid(self, url, context):
normalized = "http://blah.com"
result = config_parse.valid_master_address(url, context)
assert result == normalized

@pytest.mark.parametrize(
"url",
[
"https://blah.com",
"http://blah.com/something",
"blah.com/other",
"http://",
"blah.com?a=1",
],
)
def test_invalid(self, url, context):
with pytest.raises(ConfigError):
config_parse.valid_master_address(url, context)


if __name__ == "__main__":
run()
22 changes: 2 additions & 20 deletions tests/core/action_test.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import pytest

from tron.config.schema import ConfigAction
from tron.config.schema import ConfigConstraint
from tron.config.schema import ConfigParameter
from tron.config.schema import ConfigSecretSource
from tron.config.schema import ConfigSecretVolume
from tron.config.schema import ConfigSecretVolumeItem
Expand All @@ -17,24 +15,12 @@ def test_from_config_full(self, disk):
name="ted",
command="do something",
node="first",
executor="ssh",
executor="kubernetes",
cpus=1,
mem=100,
disk=disk, # default: 1024.0
constraints=[
ConfigConstraint(
attribute="pool",
operator="LIKE",
value="default",
),
],
node_selectors={"yelp.com/pool": "default"},
docker_image="fake-docker.com:400/image",
docker_parameters=[
ConfigParameter(
key="test",
value=123,
),
],
env={"TESTING": "true"},
secret_env={"TEST_SECRET": ConfigSecretSource(secret_name="tron-secret-svc-sec--A", key="sec_A")},
secret_volumes=[
Expand Down Expand Up @@ -68,9 +54,7 @@ def test_from_config_full(self, disk):
assert command_config.cpus == config.cpus
assert command_config.mem == config.mem
assert command_config.disk == (600.0 if disk else 1024.0)
assert command_config.constraints == {("pool", "LIKE", "default")}
assert command_config.docker_image == config.docker_image
assert command_config.docker_parameters == {("test", 123)}
assert command_config.env == config.env
assert command_config.secret_env == config.secret_env
# cant do direct tuple equality, since this is not hashable
Expand All @@ -89,9 +73,7 @@ def test_from_config_none_values(self):
assert new_action.executor == config.executor
command_config = new_action.command_config
assert command_config.command == config.command
assert command_config.constraints == set()
assert command_config.docker_image is None
assert command_config.docker_parameters == set()
assert command_config.env == {}
assert command_config.secret_env == {}
assert command_config.secret_volumes == []
Expand Down
Loading

0 comments on commit 1f71d0f

Please sign in to comment.