diff --git a/.gitignore b/.gitignore index 623fd05..ba61b97 100644 --- a/.gitignore +++ b/.gitignore @@ -103,7 +103,7 @@ venv.bak/ # generated by build coverage-*3*.xml -junit-*3*.xml +junit*.xml pylama-parent.ini tox-parent.ini adapt_template.sh diff --git a/netprobify/main.py b/netprobify/main.py index 6d52c58..03c54a4 100755 --- a/netprobify/main.py +++ b/netprobify/main.py @@ -39,11 +39,6 @@ ICMP_LOSS_RATIO, ICMP_ROUND_TRIP, ICMP_SENT, - IPERF_BANDWIDTH, - IPERF_LOSS, - IPERF_LOSS_RATIO, - IPERF_OUT_OF_ORDER, - IPERF_SENT, LIST_TARGET_MEASUREMENT_METRICS, LIST_TARGET_METRICS, NETPROBIFY_INFO, @@ -62,7 +57,6 @@ ) from netprobify.protocol.common.protocols import list_self_ips from netprobify.protocol.icmp_ping import ICMPping -from netprobify.protocol.iperf import Iperf from netprobify.protocol.target import Group from netprobify.protocol.tcpsyn import TCPsyn from netprobify.protocol.udp_unreachable import UDPunreachable @@ -106,9 +100,7 @@ def __init__(self, f="config.yaml"): self.config_file = f self.list_targets = [] self.list_target_name = [] - self.list_special_targets = [] self.list_dynamic_targets = [] - self.list_dynamic_special_targets = [] self.shared_dynamic_targets = {} self.shared_dynamic_targets_backup = {} self.list_groups = [] @@ -235,42 +227,12 @@ def load_target_conf(self, target, target_name, target_groups): creation_date=target.get("creation_date"), lifetime=target.get("lifetime"), ) - elif target["type"] == "iperf": - target = Iperf( - name=target_name, - active=True, - description=target.get("description", target_name), - destination=None, - config_destination=target["destination"], - address_family=target.get("address_family", DEFAULT_ADDRESS_FAMILY), - dst_port=target["dst_port"], - threshold=target.get("threshold"), - state=target.get("state"), - alert_level=target.get("alert_level", "no_alert"), - is_dynamic=target.get("is_dynamic", False), - # dns_update interval is global if not specified - dns_update_interval=target.get( - "dns_update_interval", self.global_vars.get("dns_update_interval", 0) - ), - groups=target_groups, - duration=target.get("iperf_parameters", {}).get("duration", 5), - bandwidth=target.get("iperf_parameters", {}).get("bandwidth_per_stream", "1M"), - protocol=target.get("iperf_parameters", {}).get("protocol", "udp"), - num_streams=target.get("iperf_parameters", {}).get("nb_parallel_streams", 1), - creation_date=target.get("creation_date"), - lifetime=target.get("lifetime"), - ) else: return # we put the target in the right list if target.is_dynamic: - if target.is_special: - self.list_dynamic_special_targets.append(target) - else: - self.list_dynamic_targets.append(target) - elif target.is_special: - self.list_special_targets.append(target) + self.list_dynamic_targets.append(target) else: self.list_targets.append(target) @@ -285,7 +247,6 @@ def load_conf(self, schema_file="schema_config.yaml"): # cleaning targets list self.list_groups = [] self.list_targets = [] - self.list_special_targets = [] self.list_target_name = [] self.global_vars = {} self.first_iter = True @@ -387,7 +348,7 @@ def load_conf(self, schema_file="schema_config.yaml"): log.debug("Target %s created", target_name) - if len(target_groups) == 0 and target["type"] != "iperf": + if len(target_groups) == 0: log.warning("Target %s disabled: not associated to any group", target_name) def update_hosts(self, force=False): @@ -399,9 +360,7 @@ def update_hosts(self, force=False): self_ips = {af: list_self_ips(af, scapyconf) for af in ("ipv4", "ipv6")} for target in itertools.chain( self.list_targets, - self.list_special_targets, self.list_dynamic_targets, - self.list_dynamic_special_targets, ): if len(target.groups): changed = False @@ -772,47 +731,6 @@ def get_metrics(self): destination=name, address_family=address_family, ).inc(port_mismatch) - elif res["probing_type"] == "iperf": - loss_ratio = res["loss"] / res["sent"] if res["sent"] != 0 else 0 - IPERF_SENT.labels( - probe_name=self.global_vars["probe_name"], - destination=name, - address_family=address_family, - state=res["state"], - group=grp.name, - ).set(res["sent"]) - - IPERF_LOSS.labels( - probe_name=self.global_vars["probe_name"], - destination=name, - address_family=address_family, - state=res["state"], - group=grp.name, - ).set(res["loss"]) - - IPERF_LOSS_RATIO.labels( - probe_name=self.global_vars["probe_name"], - destination=name, - address_family=address_family, - state=res["state"], - group=grp.name, - ).set(loss_ratio) - - IPERF_BANDWIDTH.labels( - probe_name=self.global_vars["probe_name"], - destination=name, - address_family=address_family, - state=res["state"], - group=grp.name, - ).set(res["bandwidth"]) - - IPERF_OUT_OF_ORDER.labels( - probe_name=self.global_vars["probe_name"], - destination=name, - address_family=address_family, - state=res["state"], - group=grp.name, - ).set(res["out_of_order"]) def reload_request(self, signum, frame): """Reload handler for SIGHUP. Will reload the configuration. @@ -879,7 +797,6 @@ def get_dynamic_targets(self): # we reset the targets self.list_dynamic_targets = [] - self.list_dynamic_special_targets = [] # get targets by inventory for inventory in self.shared_dynamic_targets.keys(): @@ -1075,10 +992,6 @@ def main(self): self.global_vars.get("timeout", 3600), ) - # start probing only for special targets which are not a priority - remaining_time = self.global_vars["interval"] - (time.time() - round_start) - self.start_processes(self.list_special_targets, remaining_time) - # the first iteration is done self.first_iter = False diff --git a/netprobify/metrics.py b/netprobify/metrics.py index f209373..d2879c4 100644 --- a/netprobify/metrics.py +++ b/netprobify/metrics.py @@ -88,33 +88,6 @@ ["probe_name", "destination", "address_family"], ) -# prometheus metrics - iperf probe -IPERF_SENT = Gauge( - "iperf_sent_total", - "number of sent packets reported by iperf.", - ["probe_name", "destination", "address_family", "state", "group"], -) -IPERF_LOSS = Gauge( - "iperf_loss_total", - "number of lost packets reported by iperf.", - ["probe_name", "destination", "address_family", "state", "group"], -) -IPERF_LOSS_RATIO = Gauge( - "iperf_loss_ratio", - "loss ratio reported by iperf.", - ["probe_name", "destination", "address_family", "state", "group"], -) -IPERF_BANDWIDTH = Gauge( - "iperf_bandwidth_bps", - "bandwidth reported by iperf.", - ["probe_name", "destination", "address_family", "state", "group"], -) -IPERF_OUT_OF_ORDER = Gauge( - "iperf_out_of_order_count", - "port source/destination mismatch.", - ["probe_name", "destination", "address_family", "state", "group"], -) - # prometheus metrics - common THRESHOLD = Gauge( "threshold", @@ -175,11 +148,6 @@ ICMP_LOSS, ICMP_ROUND_TRIP, ICMP_LOSS_RATIO, - IPERF_SENT, - IPERF_LOSS, - IPERF_LOSS_RATIO, - IPERF_BANDWIDTH, - IPERF_OUT_OF_ORDER, ] LIST_TARGET_METRICS = LIST_TARGET_HEALTH_METRICS + LIST_TARGET_MEASUREMENT_METRICS diff --git a/netprobify/protocol/iperf.py b/netprobify/protocol/iperf.py deleted file mode 100644 index 244665a..0000000 --- a/netprobify/protocol/iperf.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Module for Iperf bandwidth test.""" - -import logging -import subprocess - -from netprobify.protocol.target import Target - -log_iperf = logging.getLogger(__name__) - - -def get_sum_from_stream(iterable, index): - """Return sum from a column. - - Keyword arguments: - iterable -- lists to iter on - index -- column of the lists to sum up - """ - return sum([int(line.split(",")[index]) for line in iterable]) - - -class Iperf(Target): - """Iperf target. - - Only iperf2 is supported. Iperf3 servers are missing multiple client feature. - """ - - def __init__( - self, - name, - active, - description, - destination, - config_destination, - address_family, - dst_port, - threshold, - state, - alert_level, - is_dynamic, - dns_update_interval, - groups, - duration, - bandwidth, - protocol, - num_streams, - creation_date, - lifetime, - ): - """Initialize. - - Keyword arguments: - name -- name of the target - active -- state of the target - description -- short description of the target - destination -- hostname or ip address to target - config_destination -- original destination before DNS resolution - address_family -- address family to use for probing - duration -- duration of the iperf test - dst_port -- destination port for TCP request - bandwidth -- IP payload size to generate - protocol -- protocol used by iperf (udp or tcp) - is_dynamic -- if coming from a dynamic inventory. False if coming from the main config file. - dns_update_interval -- interval for DNS resolution - """ - Target.__init__( - self, - name, - active, - description, - destination, - config_destination, - address_family, - None, # unused (dont_fragment) - False, # unused (is_subnet) - None, # unused (nb_packets) - None, # unused (interval) - None, # unused (timeout) - None, # unused (ip_payload_size) - threshold, - state, - alert_level, - is_dynamic, - dns_update_interval, - groups, - creation_date, - lifetime, - ) - self.dst_port = dst_port - self.duration = duration - self.bandwidth = bandwidth - self.protocol = "-u" if protocol == "udp" else None - self.num_streams = num_streams - self.is_special = True - - def send_packets(self, res, logging_level, *args): - """Send the packets stored in self.packets. - - Keyword arguments: - res -- variable to store results (manager list) - logging_level -- logging level for targets class - args -- to catch unused variable - """ - try: - iperf_results = subprocess.check_output( - "iperf -c {0} -p {1} {2} -b {3} -t {4} \ - --reportexclude CDMS --reportstyle C -P {5}".format( - self.destination, - self.dst_port, - self.protocol, - self.bandwidth, - self.duration, - self.num_streams, - ), - shell=True, # noqa - stderr=subprocess.DEVNULL, - universal_newlines=True, - ) - except subprocess.SubprocessError as error: - log_iperf.error("iperf failed to proceed. Error is: {0}".format(error)) # noqa - return - - stream_results = [line for line in iperf_results.split("\n") if line] - # get the results from the output - try: - res.append( - { - "name": self.name, - "probing_type": "iperf", - "groups": self.groups, - "state": self.state, - "alert_level": self.alert_level, - "duration": self.duration, - "destination": self.destination, - "address_family": self.address_family, - "bandwidth": get_sum_from_stream(stream_results, 8), - "loss": get_sum_from_stream(stream_results, 10), - "sent": get_sum_from_stream(stream_results, 11), - "out_of_order": get_sum_from_stream(stream_results, 13), - } - ) - except (IndexError, ValueError) as error: - log_iperf.warning("Invalid output from iperf: {}".format(error)) # noqa diff --git a/netprobify/protocol/target.py b/netprobify/protocol/target.py index c1d37ac..828c18f 100644 --- a/netprobify/protocol/target.py +++ b/netprobify/protocol/target.py @@ -162,7 +162,6 @@ def __init__( self.time_to_refresh = time.time() self.packets = [] - self.is_special = False self.config_ip_payload_size = ip_payload_size self.proto_payload_size = None diff --git a/netprobify/protocol/tcpsyn.py b/netprobify/protocol/tcpsyn.py index 808d13f..44c37a0 100644 --- a/netprobify/protocol/tcpsyn.py +++ b/netprobify/protocol/tcpsyn.py @@ -326,7 +326,9 @@ def send_packets(self, res, logging_level, all_groups, verbose=0, force_raw_sock if seq_acked not in packets: log_tcpsyn.error( "No sent packet for response with ack=%s, destination=%s:%s", - pkt[1].ack, self.config_destination, self.dst_port, + pkt[1].ack, + self.config_destination, + self.dst_port, ) match_fail += 1 else: diff --git a/netprobify/schema_config.yaml b/netprobify/schema_config.yaml index 95a6395..a3dd69a 100644 --- a/netprobify/schema_config.yaml +++ b/netprobify/schema_config.yaml @@ -118,7 +118,7 @@ mapping: desc: description of the target type: type: str - enum: ["TCPsyn", "ICMPping", "UDPunreachable", "iperf"] + enum: ["TCPsyn", "ICMPping", "UDPunreachable"] desc: probe type required: true destination: @@ -186,22 +186,4 @@ mapping: sequence: - type: str pattern: ^([a-z0-9_\-\.])+$ - desc: group name - iperf_parameters: - type: map - desc: parameters for iperf targets - mapping: - duration: - type: int - desc: duration of the test in seconds - protocol: - type: str - enum: ["udp","tcp"] - desc: protocol used - bandwidth: - type: str - pattern: ^([0-9]+[k|m|g|K|M|G])$ - desc: bandwidth - num_streams: - type: int - desc: number of parallel client \ No newline at end of file + desc: group name \ No newline at end of file diff --git a/tests/netprobify/config/test_config.yaml b/tests/netprobify/config/test_config.yaml index d5537c4..daa6ba4 100644 --- a/tests/netprobify/config/test_config.yaml +++ b/tests/netprobify/config/test_config.yaml @@ -105,21 +105,6 @@ targets: destination: "127.0.1.1" dst_port: 80 timeout: 1 - 7_full_iperf: - type: iperf - destination: 127.0.0.1 - timeout: 30 - dst_port: 5000 - iperf_parameters: - duration: 30 - protocol: udp - bandwidth: 10m - num_streams: 2 - 8_minimal_iperf: - type: iperf - destination: 127.0.0.1 - timeout: 30 - dst_port: 5000 9_fake_group: type: "TCPsyn" destination: 127.0.0.1 diff --git a/tests/netprobify/test_iperf.py b/tests/netprobify/test_iperf.py deleted file mode 100644 index 0c35e98..0000000 --- a/tests/netprobify/test_iperf.py +++ /dev/null @@ -1,68 +0,0 @@ -from unittest import mock - -import pytest -from netprobify.protocol.iperf import Iperf - - -@pytest.fixture(name="test_target") -def create_target(): - """Create fixture for test target.""" - return Iperf( - "localhost", - active=True, - description="localhost", - destination="127.0.0.1", - config_destination="127.0.0.1", - address_family="ipv4", - dst_port=0, - threshold=1, - state="in production", - alert_level="paging", - is_dynamic=False, - dns_update_interval=0, - groups={"test"}, - duration=5, - bandwidth="1M", - protocol="udp", - num_streams="2", - lifetime={"days": "1"}, - creation_date=None, - ) - - -@mock.patch("netprobify.protocol.iperf.subprocess.check_output") -def test_send_packets(mock_check_output, test_target): - """Check iperf result is parsed properly.""" - mock_check_output.return_value = """ -20190322165127,127.0.0.1,5001,127.0.0.1,49583,3,0.0-5.0,657090,1051051,0.005,0,446,0.000,1 -20190322165127,127.0.0.1,5001,127.0.0.1,60162,4,0.0-5.0,654150,1046345,0.003,1,446,0.224,0 -""" - - result = [] - test_target.send_packets(result, "DEBUG") - - assert result == [ - { - "alert_level": "paging", - "bandwidth": 2097396, - "destination": "127.0.0.1", - "address_family": "ipv4", - "duration": 5, - "groups": {"test"}, - "loss": 1, - "name": "localhost", - "out_of_order": 1, - "probing_type": "iperf", - "sent": 892, - "state": "in production", - } - ] - - -@mock.patch("netprobify.protocol.iperf.subprocess.check_output") -def test_send_packets_invalid_result(mock_check_output, test_target): - """Check the module does not fail when output is invalid.""" - # invalid output - mock_check_output.return_value = "1" - result = [] - test_target.send_packets(result, "DEBUG") diff --git a/tests/netprobify/test_main.py b/tests/netprobify/test_main.py index 441f0cf..485b119 100755 --- a/tests/netprobify/test_main.py +++ b/tests/netprobify/test_main.py @@ -39,7 +39,6 @@ def test_load_target_conf(): target_tcp = {"type": "TCPsyn", "destination": "test", "dst_port": 8000} target_icmp = {"type": "ICMPping", "destination": "test"} target_udp = {"type": "UDPunreachable", "destination": "test", "dst_port": 8000} - target_iperf = {"type": "iperf", "destination": "test", "dst_port": 8000} target_fake = {"type": "fake", "destination": "test", "dst_port": 8000} @@ -52,16 +51,11 @@ def test_load_target_conf(): netprobify.load_target_conf(target_udp, "test", None) assert len(netprobify.list_targets) == 3 - netprobify.load_target_conf(target_iperf, "test", None) - assert len(netprobify.list_special_targets) == 1 - netprobify.load_target_conf(target_fake, "test", None) # we check we have the right number of targets in the lists assert len(netprobify.list_targets) == 3 - assert len(netprobify.list_special_targets) == 1 assert len(netprobify.list_dynamic_targets) == 0 - assert len(netprobify.list_dynamic_special_targets) == 0 def test_load_conf(): @@ -143,7 +137,6 @@ def test_load_conf(): "interval": 0, "ip_payload_size": 1000, "is_dynamic": False, - "is_special": False, "is_subnet": True, "max_seq": None, "min_seq": None, @@ -175,7 +168,6 @@ def test_load_conf(): "interval": 0, "ip_payload_size": 20, "is_dynamic": False, - "is_special": False, "is_subnet": False, "max_seq": None, "min_seq": None, @@ -207,7 +199,6 @@ def test_load_conf(): "interval": 0, "ip_payload_size": 1000, "is_dynamic": False, - "is_special": False, "is_subnet": True, "name": "3_full_icmp", "nb_packets": 1, @@ -235,7 +226,6 @@ def test_load_conf(): "interval": 0, "ip_payload_size": 8, "is_dynamic": False, - "is_special": False, "is_subnet": False, "name": "4_minimal_icmp", "nb_packets": 1, @@ -264,7 +254,6 @@ def test_load_conf(): "interval": 0, "ip_payload_size": 1000, "is_dynamic": False, - "is_special": False, "is_subnet": True, "max_seq": None, "min_seq": None, @@ -296,7 +285,6 @@ def test_load_conf(): "interval": 0, "ip_payload_size": 8, "is_dynamic": False, - "is_special": False, "is_subnet": False, "max_seq": None, "min_seq": None, @@ -328,7 +316,6 @@ def test_load_conf(): "interval": 0, "ip_payload_size": 20, "is_dynamic": False, - "is_special": False, "is_subnet": False, "max_seq": None, "min_seq": None, @@ -346,74 +333,6 @@ def test_load_conf(): "creation_date": None, } - assert netprobify.list_special_targets[0].__dict__ == { - "active": True, - "alert_level": "no_alert", - "config_ip_payload_size": None, - "bandwidth": "1M", - "config_destination": "127.0.0.1", - "address_family": "ipv4", - "description": "7_full_iperf", - "destination": None, - "dns_update_interval": 300, - "dont_fragment": None, - "dst_port": 5000, - "duration": 30, - "groups": {"group1"}, - "interval": None, - "ip_payload_size": None, - "is_dynamic": False, - "is_special": True, - "is_subnet": False, - "name": "7_full_iperf", - "nb_packets": None, - "num_streams": 1, - "packets": [], - "proto_payload_size": None, - "protocol": "-u", - "state": None, - "threshold": None, - # this one cannot be tested: - "time_to_refresh": netprobify.list_special_targets[0].time_to_refresh, - "timeout": None, - "lifetime": None, - "creation_date": None, - } - - assert netprobify.list_special_targets[1].__dict__ == { - "active": True, - "alert_level": "no_alert", - "config_ip_payload_size": None, - "bandwidth": "1M", - "config_destination": "127.0.0.1", - "address_family": "ipv4", - "description": "8_minimal_iperf", - "destination": None, - "dns_update_interval": 300, - "dont_fragment": None, - "dst_port": 5000, - "duration": 5, - "groups": {"group1"}, - "interval": None, - "ip_payload_size": None, - "is_dynamic": False, - "is_special": True, - "is_subnet": False, - "name": "8_minimal_iperf", - "nb_packets": None, - "num_streams": 1, - "packets": [], - "protocol": "-u", - "proto_payload_size": None, - "state": None, - "threshold": None, - # this one cannot be tested: - "time_to_refresh": netprobify.list_special_targets[1].time_to_refresh, - "timeout": None, - "lifetime": None, - "creation_date": None, - } - ## # targets not defined in yaml ## @@ -659,20 +578,6 @@ def test_get_metrics(): "match_fail": 0, "port_mismatch": 0, }, - { - "name": "4_pe01.paris", - "probing_type": "iperf", - "groups": {"group2"}, - "state": "in production", - "alert_level": "paging", - "duration": 10, - "destination": "169.254.0.1", - "address_family": "ipv4", - "bandwidth": 100, - "loss": 0, - "sent": 1000, - "out_of_order": 0, - }, ] # evaluate metrics and set Prometheus metrics @@ -726,22 +631,6 @@ def test_get_metrics(): "UDP_UNREACHABLE_PORT_MISTMATCH" ).__dict__["_metrics"] - assert ("test", "4_pe01.paris", "ipv4", "in production", "group2") in netprobify.getter( - "IPERF_SENT" - ).__dict__["_metrics"] - assert ("test", "4_pe01.paris", "ipv4", "in production", "group2") in netprobify.getter( - "IPERF_LOSS" - ).__dict__["_metrics"] - assert ("test", "4_pe01.paris", "ipv4", "in production", "group2") in netprobify.getter( - "IPERF_LOSS_RATIO" - ).__dict__["_metrics"] - assert ("test", "4_pe01.paris", "ipv4", "in production", "group2") in netprobify.getter( - "IPERF_BANDWIDTH" - ).__dict__["_metrics"] - assert ("test", "4_pe01.paris", "ipv4", "in production", "group2") in netprobify.getter( - "IPERF_OUT_OF_ORDER" - ).__dict__["_metrics"] - @mock.patch("netprobify.main.os._exit", sys.exit) def test_signal_handlers(): @@ -1169,16 +1058,6 @@ def test_get_dynamic_targets(): "lifetime": None, "creation_date": None, }, - { - "hostname": "pe02.paris", - "destination": "169.254.0.1", - "address_family": "ipv4", - "type": "iperf", - "dst_port": 22, - "groups": {"group2"}, - "lifetime": None, - "creation_date": None, - }, { "hostname": "to_expire", "destination": "169.254.0.100", @@ -1229,7 +1108,6 @@ def test_get_dynamic_targets(): "destination": "169.254.0.1", "dns_update_interval": 300, "dont_fragment": True, - "is_special": False, "max_seq": 1083, "min_seq": 984, "packets": netprobify.list_dynamic_targets[0].packets, @@ -1261,7 +1139,6 @@ def test_get_dynamic_targets(): "destination": "169.254.0.1", "dns_update_interval": 300, "dont_fragment": True, - "is_special": False, "max_seq": 1183, "min_seq": 1084, "packets": netprobify.list_dynamic_targets[1].packets, @@ -1272,39 +1149,6 @@ def test_get_dynamic_targets(): "creation_date": None, } - assert netprobify.list_dynamic_special_targets[0].__dict__ == { - "active": True, - "config_ip_payload_size": None, - "alert_level": "no_alert", - "bandwidth": "1M", - "config_destination": "169.254.0.1", - "address_family": "ipv4", - "description": "from_DCv3Lan", - "destination": "169.254.0.1", - "dns_update_interval": 300, - "dont_fragment": None, - "dst_port": 22, - "duration": 5, - "groups": {"group2"}, - "interval": None, - "ip_payload_size": None, - "is_dynamic": True, - "is_special": True, - "is_subnet": False, - "name": "DCv3Lan_pe02.paris", - "nb_packets": None, - "num_streams": 1, - "packets": [], - "protocol": "-u", - "proto_payload_size": None, - "state": None, - "threshold": None, - "time_to_refresh": netprobify.list_dynamic_special_targets[0].time_to_refresh, - "timeout": None, - "lifetime": None, - "creation_date": None, - } - # fake results of probing netprobify.result = [ {