Merge branch 'release/7.2.0'

.gitignore

@@ -8,7 +8,6 @@ Products/ZenUI3/node_modules
 dist
 build
 Zenoss.egg-info
-install-zenoss.mk
 lib/python2.7/site-packages/Zenoss-nspkg.pth
 lib/python2.7/site-packages/Zenoss.egg-link
 

Products/DataCollector/SnmpClient.py

@@ -50,7 +50,7 @@ def __init__(
         datacollector=None,
         plugins=[],
     ):
-        BaseClient.__init__(self, device, datacollector)
+        super(SnmpClient, self).__init__(device, datacollector)
         global defaultTries, defaultTimeout
         self.hostname = hostname
         self.device = device
@@ -266,7 +266,8 @@ def clientFinished(self, result):
 
             if isinstance(result.value, error.TimeoutError):
                 log.error(
-                    "Device %s timed out: are " "your SNMP settings correct?",
+                    "Device %s timed out: are "
+                    "your SNMP settings correct?",
                     self.hostname,
                 )
                 summary = "SNMP agent down - no response received"
@@ -286,13 +287,14 @@ def clientFinished(self, result):
             self._sendStatusEvent(summary, eventKey="agent_down")
         else:
             self._sendStatusEvent(
-                "SNMP agent up", eventKey="agent_down", severity=Event.Clear
+                "SNMP agent up",
+                eventKey="agent_down",
+                severity=Event.Clear,
             )
         try:
             self.proxy.close()
         except AttributeError:
-            log.info("Caught AttributeError closing SNMP connection.")
-        # Tell the datacollector that we are all done
+            log.info("caught AttributeError closing SNMP connection.")
         if self.datacollector:
             self.datacollector.clientFinished(self)
         else:

Products/DataCollector/plugins/CollectorPlugin.py

@@ -126,13 +126,13 @@ def copyDataToProxy(self, device, proxy):
 
     def asdate(self, val):
         """Convert a byte string to the date string 'YYYY/MM/DD HH:MM:SS'"""
-        datear = (1968, 1, 8, 10, 15, 00)
+        datear = (1968, 1, 8, 10, 15, 0)
         try:
             datear = struct.unpack("!h5B", val[0:7])
         except Exception:
             pass
         if datear[0] == 0:
-            datear = (1968, 1, 8, 10, 15, 00)
+            datear = (1968, 1, 8, 10, 15, 0)
         return "%d/%02d/%02d %02d:%02d:%02d" % datear[:6]
 
 

Products/DataCollector/zendisc.py

@@ -72,6 +72,14 @@ def _partitionPingResults(results):
     return (good, bad)
 
 
+# From ZenRRD/zenprocess.py; should move into ZenUtils somewhere
+def chunk(lst, n):
+    """
+    Break lst into n-sized chunks
+    """
+    return (lst[i : i + n] for i in range(0, len(lst), n))
+
+
 class ZenDisc(ZenModeler):
     """
     Scan networks and routes looking for devices to add to the ZODB
@@ -134,7 +142,13 @@ def discoverIps(self, nets):
                 )
                 continue
             self.log.info("Discover network '%s'", net.getNetworkName())
-            results = yield self.pingMany(net.fullIpList())
+
+            full_ip_list = net.fullIpList()
+            if self.options.removeInterfaceIps:
+                full_ip_list = yield self.config().callRemote(
+                    "removeInterfaces", net)
+
+            results = yield self.pingMany(full_ip_list)
             goodips, badips = _partitionPingResults(results)
             self.log.debug(
                 "Found %d good IPs and %d bad IPs", len(goodips), len(badips)
@@ -178,27 +192,29 @@ def discoverRanges(self):
         defer.returnValue(devices)
 
     @defer.inlineCallbacks
-    def discoverRouters(self, rootdev, seenips=None):
+    def discoverRouters(self, rootdev):
         """
         Discover all default routers based on DMD configuration.
 
         @param rootdev {device class} device root in DMD
-        @param seenips {list} list of IP addresses
         """
-        if not seenips:
-            seenips = []
-        ips = yield self.config().callRemote("followNextHopIps", rootdev.id)
-        for ip in ips:
-            if ip in seenips:
-                continue
-            self.log.info("Device '%s' next hop '%s'", rootdev.id, ip)
-            seenips.append(ip)
-            router = yield self.discoverDevice(
-                ip, devicepath="/Network/Router"
-            )
-            if not router:
-                continue
-            yield self.discoverRouters(router, seenips)
+        observedIps = set()
+        routers = [rootdev]
+        while routers:
+            router = routers.pop(0)
+            ips = yield self.config().callRemote("followNextHopIps", router.id)
+            unknownIps = (ip for ip in ips if ip not in observedIps)
+            for batch_of_ips in chunk(unknownIps, self.options.parallel):
+                results = yield defer.DeferredList(
+                    self.discoverDevice(ip, devicepath="/Network/Router")
+                    for ip in batch_of_ips
+                )
+                observedIps.update(batch_of_ips)
+                routers.extend(
+                    foundRouter
+                    for _, foundRouter in results
+                    if foundRouter is not None
+                )
 
     def sendDiscoveredEvent(self, ip, dev=None, sev=2):
         """
@@ -245,12 +261,21 @@ def discoverDevices(self, ips, devicepath=None, prodState=None):
             devicepath = self.options.deviceclass
         if prodState is None:
             prodState = self.options.productionState
-        devices = []
-        for ip in ips:
-            device = yield self.discoverDevice(ip, devicepath, prodState)
-            if device is not None:
-                devices.append(device)
-        defer.returnValue(devices)
+        foundDevices = []
+        for batch_of_ips in chunk(ips, self.options.parallel):
+            results = yield defer.DeferredList(
+                self.discoverDevice(ip, devicepath, prodState)
+                for ip in batch_of_ips
+            )
+            discovered = [dev for _, dev in results if dev is not None]
+            if self.log.isEnabledFor(logging.DEBUG):
+                self.log.debug(
+                    "Discovered %s devices: %s",
+                    len(discovered),
+                    ", ".join(dev.getId() for dev in discovered)
+                )
+            foundDevices.extend(discovered)
+        defer.returnValue(foundDevices)
 
     @defer.inlineCallbacks
     def findRemoteDeviceInfo(self, ip, devicePath, deviceSnmpCommunities=None):
@@ -561,15 +586,13 @@ def collectNet(self):
         # --net 10.0.0.0 --net 192.168.0.1
         if isinstance(network, (list, tuple)) and "," in network[0]:
             network = [n.strip() for n in network[0].split(",")]
-        count = 0
-        devices = []
         if not network:
             network = yield self.config().callRemote("getDefaultNetworks")
-
         if not network:
             self.log.warning("No networks configured")
-            defer.returnValue(None)
+            defer.returnValue([])
 
+        foundIps = []
         for net in network:
             try:
                 nets = yield self.config().callRemote(
@@ -578,22 +601,18 @@ def collectNet(self):
                 if not nets:
                     self.log.warning("No networks found for %s", net)
                     continue
-                ips = yield self.discoverIps(nets)
-                devices += ips
-                count += len(ips)
+                foundIps += yield self.discoverIps(nets)
             except Exception as ex:
                 self.log.exception(
                     "Error performing net discovery on %s: %s", net, ex
                 )
-        self.log.info("Working on devices: %s", devices)
+        self.log.info("Working on devices: %s", foundIps)
 
-        foundDevices = []
-        for device in devices:
-            result = yield self.discoverDevice(
-                device, self.options.deviceclass, self.options.productionState
-            )
-            if result is not None:
-                foundDevices.append(result)
+        foundDevices = yield self.discoverDevices(
+            foundIps,
+            devicepath=self.options.deviceclass,
+            prodState=self.options.productionState
+        )
         defer.returnValue(foundDevices)
 
     @defer.inlineCallbacks
@@ -641,7 +660,7 @@ def walkDiscovery(self):
         self.log.debug("My hostname = %s", myname)
         myip = None
         try:
-            myip = getHostByName(myname)
+            myip = getHostByName(myname)  # not async; blocks reactor!
             self.log.debug("My IP address = %s", myip)
         except (socket.error, DNSNameError):
             raise SystemExit("Failed lookup of my IP for name %s" % myname)
@@ -863,6 +882,15 @@ def buildOptions(self):
             default=False,
             help="Prefer SNMP name to DNS name when modeling via SNMP.",
         )
+        self.parser.add_option(
+            "--remove-interface-ips",
+            dest="removeInterfaceIps",
+            action="store_true",
+            default=False,
+            help="Skip discovery on IPs already assigned to interfaces "
+            "(device components).",
+        )
+
         # --job: a development-only option that jobs will use to communicate
         # their existence to zendisc. Not for users, so help is suppressed.
         self.parser.add_option("--job", dest="job", help=SUPPRESS_HELP)