From 7ac3eb8ce14c3ef686fdb05dec4c245016f6af75 Mon Sep 17 00:00:00 2001 From: Tomas Gonzalez Date: Thu, 9 Aug 2018 16:49:04 -0500 Subject: [PATCH] Moved common interop code, adding missing html/refs --- RedfishInteropValidator.py | 653 ++++--------------------------------- commonInterop.py | 481 +++++++++++++++++++++++++++ tohtml.py | 53 +-- 3 files changed, 575 insertions(+), 612 deletions(-) create mode 100644 commonInterop.py diff --git a/RedfishInteropValidator.py b/RedfishInteropValidator.py index af3a97b..3a6a739 100644 --- a/RedfishInteropValidator.py +++ b/RedfishInteropValidator.py @@ -3,7 +3,6 @@ # Copyright 2016 Distributed Management Task Force, Inc. All rights reserved. # License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/Redfish-Interop-Validator/blob/master/LICENSE.md -import io import os import sys import re @@ -12,480 +11,26 @@ import logging import json import traverseService as rst -import jsonschema import argparse -from enum import Enum from io import StringIO from commonProfile import getProfiles, checkProfileAgainstSchema from traverseService import AuthenticationError from tohtml import renderHtml, writeHtml +from metadata import setup_schema_pack +import commonInterop rsvLogger = rst.getLogger() -config = {'WarnRecommended': False} - VERBO_NUM = 15 logging.addLevelName(VERBO_NUM, "VERBO") + + def verboseout(self, message, *args, **kws): if self.isEnabledFor(VERBO_NUM): self._log(VERBO_NUM, message, args, **kws) logging.Logger.verboseout = verboseout -class sEnum(Enum): - FAIL = 'FAIL' - PASS = 'PASS' - WARN = 'WARN' - -class msgInterop: - def __init__(self, name, entry, expected, actual, success): - self.name = name - self.entry = entry - self.expected = expected - self.actual = actual - if isinstance(success, bool): - self.success = sEnum.PASS if success else sEnum.FAIL - else: - self.success = success - self.parent = None - - -def validateRequirement(entry, decodeditem, conditional=False): - """ - Validates Requirement entry - """ - propDoesNotExist = (decodeditem == 'DNE') - rsvLogger.info('Testing ReadRequirement \n\texpected:' + str(entry) + ', exists: ' + str(not propDoesNotExist)) - # If we're not mandatory, pass automatically, else fail - # However, we have other entries "IfImplemented" and "Conditional" - # note: Mandatory is default!! if present in the profile. Make sure this is made sure. - originalentry = entry - if entry == "IfImplemented" or (entry == "Conditional" and conditional): - entry = "Mandatory" - paramPass = not entry == "Mandatory" or \ - entry == "Mandatory" and not propDoesNotExist - if entry == "Recommended" and propDoesNotExist: - rsvLogger.info('\tItem is recommended but does not exist') - if config['WarnRecommended']: - rsvLogger.error('\tItem is recommended but does not exist, escalating to WARN') - paramPass = sEnum.WARN - - rsvLogger.info('\tpass ' + str(paramPass)) - if not paramPass: - rsvLogger.error('\tNoPass') - return msgInterop('ReadRequirement', originalentry, 'Must Exist' if entry == "Mandatory" else 'Any', 'Exists' if not propDoesNotExist else 'DNE', paramPass),\ - paramPass - - -def isPropertyValid(profilePropName, rObj): - for prop in rObj.getResourceProperties(): - if profilePropName == prop.propChild: - return None, True - rsvLogger.error('{} - Does not exist in ResourceType Schema, please consult profile provided'.format(profilePropName)) - return msgInterop('PropertyValidity', profilePropName, 'Should Exist', 'in ResourceType Schema', False), False - - -def validateMinCount(alist, length, annotation=0): - """ - Validates Mincount annotation - """ - rsvLogger.info('Testing minCount \n\texpected:' + str(length) + ', val:' + str(annotation)) - paramPass = len(alist) >= length or annotation >= length - rsvLogger.info('\tpass ' + str(paramPass)) - if not paramPass: - rsvLogger.error('\tNoPass') - return msgInterop('MinCount', length, '<=', annotation if annotation > len(alist) else len(alist), paramPass),\ - paramPass - - -def validateSupportedValues(enumlist, annotation): - """ - Validates SupportedVals annotation - """ - rsvLogger.info('Testing supportedValues \n\t:' + str(enumlist) + ', exists:' + str(annotation)) - for item in enumlist: - paramPass = item in annotation - if not paramPass: - break - rsvLogger.info('\tpass ' + str(paramPass)) - if not paramPass: - rsvLogger.error('\tNoPass') - return msgInterop('SupportedValues', enumlist, 'included in...', annotation, paramPass),\ - paramPass - - -def findPropItemforString(propObj, itemname): - """ - Finds an appropriate object for an item - """ - for prop in propObj.getResourceProperties(): - decodedName = prop.name.split(':')[-1] - if itemname == decodedName: - return prop - return None - - -def validateWriteRequirement(propObj, entry, itemname): - """ - Validates if a property is WriteRequirement or not - """ - rsvLogger.info('writeable \n\t' + str(entry)) - permission = 'Read' - expected = "OData.Permission/ReadWrite" if entry else "Any" - if entry: - targetProp = findPropItemforString(propObj, itemname.replace('#', '')) - propAttr = None - if targetProp is not None: - propAttr = targetProp.propDict.get('OData.Permissions') - if propAttr is not None: - permission = propAttr.get('EnumMember', 'Read') - paramPass = permission \ - == "OData.Permission/ReadWrite" - else: - paramPass = False - else: - paramPass = True - - rsvLogger.info('\tpass ' + str(paramPass)) - if not paramPass: - rsvLogger.error('\tNoPass') - return msgInterop('WriteRequirement', entry, expected, permission, paramPass),\ - paramPass - - -def checkComparison(val, compareType, target): - """ - Validate a given comparison option, given a value and a target set - """ - rsvLogger.info('Testing a comparison \n\t' + str((val, compareType, target))) - vallist = val if isinstance(val, list) else [val] - paramPass = False - if compareType == "AnyOf": - for item in vallist: - paramPass = item in target - if paramPass: - break - else: - continue - - if compareType == "AllOf": - alltarget = set() - for item in vallist: - paramPass = item in target and item not in alltarget - if paramPass: - alltarget.add(item) - if len(alltarget) == len(target): - break - else: - continue - paramPass = len(alltarget) == len(target) - if compareType == "LinkToResource": - vallink = val.get('@odata.id') - success, decoded, code, elapsed = rst.callResourceURI(vallink) - if success: - ourType = decoded.get('@odata.type') - if ourType is not None: - SchemaType = rst.getType(ourType) - paramPass = SchemaType in target - else: - paramPass = False - else: - paramPass = False - - if compareType == "Equal": - paramPass = val == target - if compareType == "NotEqual": - paramPass = val != target - if compareType == "GreaterThan": - paramPass = val > target - if compareType == "GreaterThanOrEqual": - paramPass = val >= target - if compareType == "LessThan": - paramPass = val < target - if compareType == "LessThanOrEqual": - paramPass = val <= target - if compareType == "Absent": - paramPass = val == 'DNE' - if compareType == "Present": - paramPass = val != 'DNE' - rsvLogger.info('\tpass ' + str(paramPass)) - if not paramPass: - rsvLogger.error('\tNoPass') - return msgInterop('Comparison', target, compareType, val, paramPass),\ - paramPass - - -def validateMembers(members, entry, annotation): - """ - Validate an entry of Members and its count annotation - """ - rsvLogger.info('Testing members \n\t' + str((members, entry, annotation))) - if not validateRequirement('Mandatory', members): - return False - if "MinCount" in entry: - mincount, mincountpass = validateMinCount(members, entry["MinCount"], annotation) - mincount.name = 'MembersMinCount' - return mincount, mincountpass - - -def validateMinVersion(fulltype, entry): - """ - Checks for the minimum version of a resource's type - """ - fulltype = fulltype.replace('#', '') - rsvLogger.info('Testing minVersion \n\t' + str((fulltype, entry))) - # If fulltype doesn't contain version as is, try it as v#_#_# - versionSplit = entry.split('.') - versionNew = 'v' - for x in versionSplit: - versionNew = versionNew + x + '_' - versionNew = versionNew[:-1] - # get version from payload - v_payload = rst.getNamespace(fulltype).split('.', 1)[-1] - # use string comparison, given version numbering is accurate to regex - paramPass = v_payload >= (versionNew if 'v' in v_payload else entry) - rsvLogger.info('\tpass ' + str(paramPass)) - if not paramPass: - rsvLogger.error('\tNo Pass') - return msgInterop('MinVersion', '{} ({})'.format(entry, versionNew), '<=', fulltype, paramPass),\ - paramPass - - -def checkConditionalRequirement(propResourceObj, entry, decodedtuple, itemname): - """ - Returns boolean if entry's conditional is true or false - """ - rsvLogger.info('Evaluating conditionalRequirements') - if "SubordinateToResource" in entry: - isSubordinate = False - # iterate through parents via resourceObj - # list must be reversed to work backwards - resourceParent = propResourceObj.parent - for expectedParent in reversed(entry["SubordinateToResource"]): - if resourceParent is not None: - parentType = resourceParent.typeobj.stype - isSubordinate = parentType == expectedParent - rsvLogger.info('\tsubordinance ' + - str(parentType) + ' ' + str(isSubordinate)) - resourceParent = resourceParent.parent - else: - rsvLogger.info('no parent') - isSubordinate = False - return isSubordinate - if "CompareProperty" in entry: - decodeditem, decoded = decodedtuple - # find property in json payload by working backwards thru objects - # decoded tuple is designed just for this piece, since there is - # no parent in dictionaries - comparePropName = entry["CompareProperty"] - while comparePropName not in decodeditem and decoded is not None: - decodeditem, decoded = decoded - compareProp = decodeditem.get(comparePropName, 'DNE') - return checkComparison(compareProp, entry["Comparison"], entry.get("CompareValues", []))[1] - - -def validatePropertyRequirement(propResourceObj, entry, decodedtuple, itemname, chkCondition=False): - """ - Validate PropertyRequirements - """ - msgs = [] - counts = Counter() - decodeditem, decoded = decodedtuple - if entry is None or len(entry) == 0: - rsvLogger.debug('there are no requirements for this prop') - else: - rsvLogger.info('propRequirement with value: ' + str(decodeditem if not isinstance( - decodeditem, dict) else 'dict')) - # If we're working with a list, then consider MinCount, Comparisons, then execute on each item - # list based comparisons include AnyOf and AllOf - if isinstance(decodeditem, list): - rsvLogger.info("inside of a list: " + itemname) - if "MinCount" in entry: - msg, success = validateMinCount(decodeditem, entry["MinCount"], - decoded[0].get(itemname.split('.')[-1] + '@odata.count', 0)) - msgs.append(msg) - msg.name = itemname + '.' + msg.name - for k, v in entry.get('PropertyRequirements', {}).items(): - # default to AnyOf if Comparison is not present but Values is - comparisonValue = v.get("Comparison", "AnyOf") if v.get("Values") is not None else None - if comparisonValue in ["AllOf", "AnyOf"]: - msg, success = (checkComparison([val.get(k, 'DNE') for val in decodeditem], - comparisonValue, v["Values"])) - msgs.append(msg) - msg.name = itemname + '.' + msg.name - cnt = 0 - for item in decodeditem: - listmsgs, listcounts = validatePropertyRequirement( - propResourceObj, entry, (item, decoded), itemname + '#' + str(cnt)) - counts.update(listcounts) - msgs.extend(listmsgs) - cnt += 1 - - else: - # consider requirement before anything else - # problem: if dne, skip? - - # Read Requirement is default mandatory if not present - msg, success = validateRequirement(entry.get('ReadRequirement', 'Mandatory'), decodeditem) - msgs.append(msg) - msg.name = itemname + '.' + msg.name - - if "WriteRequirement" in entry: - msg, success = validateWriteRequirement(propResourceObj, entry["WriteRequirement"], itemname) - msgs.append(msg) - msg.name = itemname + '.' + msg.name - if "ConditionalRequirements" in entry: - innerList = entry["ConditionalRequirements"] - for item in innerList: - if checkConditionalRequirement(propResourceObj, item, decodedtuple, itemname): - rsvLogger.info("\tCondition DOES apply") - conditionalMsgs, conditionalCounts = validatePropertyRequirement( - propResourceObj, item, decodedtuple, itemname, chkCondition = True) - counts.update(conditionalCounts) - for item in conditionalMsgs: - item.name = item.name.replace('.', '.Conditional.', 1) - msgs.extend(conditionalMsgs) - else: - rsvLogger.info("\tCondition does not apply") - if "MinSupportValues" in entry: - msg, success = validateSupportedValues( - decodeditem, entry["MinSupportValues"], - decoded[0].get(itemname.split('.')[-1] + '@Redfish.AllowableValues', [])) - msgs.append(msg) - msg.name = itemname + '.' + msg.name - if "Comparison" in entry and not chkCondition and\ - entry["Comparison"] not in ["AnyOf", "AllOf"]: - msg, success = checkComparison(decodeditem, entry["Comparison"], entry.get("Values",[])) - msgs.append(msg) - msg.name = itemname + '.' + msg.name - if "PropertyRequirements" in entry: - innerDict = entry["PropertyRequirements"] - if isinstance(decodeditem, dict): - for item in innerDict: - rsvLogger.info('inside complex ' + itemname + '.' + item) - complexMsgs, complexCounts = validatePropertyRequirement( - propResourceObj, innerDict[item], (decodeditem.get(item, 'DNE'), decodedtuple), item) - msgs.extend(complexMsgs) - counts.update(complexCounts) - else: - rsvLogger.info('complex {} is missing or not a dictionary'.format(itemname + '.' + item, None)) - return msgs, counts - - -def validateActionRequirement(propResourceObj, entry, decodedtuple, actionname): - """ - Validate Requirements for one action - """ - decodeditem, decoded = decodedtuple - counts = Counter() - msgs = [] - rsvLogger.info('actionRequirement \n\tval: ' + str(decodeditem if not isinstance( - decodeditem, dict) else 'dict') + ' ' + str(entry)) - if "ReadRequirement" in entry: - # problem: if dne, skip - msg, success = validateRequirement(entry.get('ReadRequirement', "Mandatory"), decodeditem) - msgs.append(msg) - msg.name = actionname + '.' + msg.name - propDoesNotExist = (decodeditem == 'DNE') - if propDoesNotExist: - return msgs, counts - # problem: if dne, skip - if "Parameters" in entry: - innerDict = entry["Parameters"] - for k in innerDict: - item = innerDict[k] - annotation = decodeditem.get(str(k) + '@Redfish.AllowableValues', 'DNE') - # problem: if dne, skip - # assume mandatory - msg, success = validateRequirement(item.get('ReadRequirement', "Mandatory"), annotation) - msgs.append(msg) - msg.name = actionname + '.Parameters.' + msg.name - if annotation == 'DNE': - continue - if "ParameterValues" in item: - msg, success = validateSupportedValues( - item["ParameterValues"], annotation) - msgs.append(msg) - msg.name = actionname + '.' + msg.name - if "RecommendedValues" in item: - msg, success = validateSupportedValues( - item["RecommendedValues"], annotation) - msg.name = msg.name.replace('Supported', 'Recommended') - if config['WarnRecommended'] and not success: - rsvLogger.error('\tRecommended parameters do not all exist, escalating to WARN') - msg.success = sEnum.WARN - elif not success: - rsvLogger.error('\tRecommended parameters do not all exist, but are not Mandatory') - msg.success = sEnum.PASS - - msgs.append(msg) - msg.name = actionname + '.' + msg.name - # consider requirement before anything else, what if action - # if the action doesn't exist, you can't check parameters - # if it doesn't exist, what should not be checked for action - return msgs, counts - - -def validateInteropResource(propResourceObj, interopDict, decoded): - """ - Base function that validates a single Interop Resource by its entry - """ - msgs = [] - rsvLogger.info('### Validating an InteropResource') - rsvLogger.debug(str(interopDict)) - counts = Counter() - # decodedtuple provides the chain of dicts containing dicts, needed for CompareProperty - decodedtuple = (decoded, None) - if "MinVersion" in interopDict: - msg, success = validateMinVersion(propResourceObj.typeobj.fulltype, interopDict['MinVersion']) - msgs.append(msg) - if "PropertyRequirements" in interopDict: - # problem, unlisted in 0.9.9a - innerDict = interopDict["PropertyRequirements"] - for item in innerDict: - vmsg, isvalid = isPropertyValid(item, propResourceObj) - if not isvalid: - msgs.append(vmsg) - vmsg.name = '{}.{}'.format(item, vmsg.name) - continue - rsvLogger.info('### Validating PropertyRequirements for {}'.format(item)) - pmsgs, pcounts = validatePropertyRequirement( - propResourceObj, innerDict[item], (decoded.get(item, 'DNE'), decodedtuple), item) - rsvLogger.info(pcounts) - counts.update(pcounts) - msgs.extend(pmsgs) - if "ActionRequirements" in interopDict: - innerDict = interopDict["ActionRequirements"] - actionsJson = decoded.get('Actions', {}) - decodedInnerTuple = (actionsJson, decodedtuple) - for item in innerDict: - actionName = '#' + propResourceObj.typeobj.stype + '.' + item - rsvLogger.info(actionName) - amsgs, acounts = validateActionRequirement(propResourceObj, innerDict[item], (actionsJson.get( - actionName, 'DNE'), decodedInnerTuple), actionName) - rsvLogger.info(acounts) - counts.update(acounts) - msgs.extend(amsgs) - if "CreateResource" in interopDict: - rsvLogger.info('Skipping CreateResource') - pass - if "DeleteResource" in interopDict: - rsvLogger.info('Skipping DeleteResource') - pass - if "UpdateResource" in interopDict: - rsvLogger.info('Skipping UpdateResource') - pass - - for item in msgs: - if item.success == sEnum.WARN: - counts['warn'] += 1 - elif item.success == sEnum.PASS: - counts['pass'] += 1 - elif item.success == sEnum.FAIL: - counts['fail.{}'.format(item.name)] += 1 - rsvLogger.info(counts) - return msgs, counts - def checkPayloadConformance(uri, decoded): """ @@ -524,6 +69,7 @@ def checkPayloadConformance(uri, decoded): 'PASS' if paramPass else 'FAIL') return success, messages + def setupLoggingCaptures(): class WarnFilter(logging.Filter): def filter(self, rec): @@ -541,13 +87,13 @@ def filter(self, rec): warnh.addFilter(WarnFilter()) warnh.setFormatter(fmt) - rsvLogger.addHandler(errh) # Printout FORMAT - rsvLogger.addHandler(warnh) # Printout FORMAT + rsvLogger.addHandler(errh) + rsvLogger.addHandler(warnh) yield - rsvLogger.removeHandler(errh) # Printout FORMAT - rsvLogger.removeHandler(warnh) # Printout FORMAT + rsvLogger.removeHandler(errh) + rsvLogger.removeHandler(warnh) warnstrings = warnMessages.getvalue() warnMessages.close() errorstrings = errorMessages.getvalue() @@ -570,11 +116,10 @@ def validateSingleURI(URI, profile, uriName='', expectedType=None, expectedSchem counts = Counter() results = OrderedDict() messages = [] - success = True - results[uriName] = {'uri':URI, 'success':False, 'counts':counts,\ - 'messages':messages, 'errors':'', 'warns': '',\ - 'rtime':'', 'context':'', 'fulltype':''} + results[uriName] = {'uri': URI, 'success': False, 'counts': counts, + 'messages': messages, 'errors': '', 'warns': '', + 'rtime': '', 'context': '', 'fulltype': ''} # check for @odata mandatory stuff # check for version numbering problems @@ -596,7 +141,7 @@ def validateSingleURI(URI, profile, uriName='', expectedType=None, expectedSchem if not successPayload: counts['failPayloadError'] += 1 - rsvLogger.error(str(URI) + ': payload error, @odata property non-conformant',) # Printout FORMAT + rsvLogger.error(str(URI) + ': payload error, @odata property non-conformant',) # Generate dictionary of property info try: @@ -606,10 +151,10 @@ def validateSingleURI(URI, profile, uriName='', expectedType=None, expectedSchem counts['problemResource'] += 1 results[uriName]['warns'], results[uriName]['errors'] = next(lc) return False, counts, results, None, None - except AuthenticationError as e: + except AuthenticationError: raise # re-raise exception - except Exception as e: - rsvLogger.exception("") # Printout FORMAT + except Exception: + rsvLogger.exception("") counts['exceptionResource'] += 1 results[uriName]['warns'], results[uriName]['errors'] = next(lc) return False, counts, results, None, None @@ -627,7 +172,7 @@ def validateSingleURI(URI, profile, uriName='', expectedType=None, expectedSchem results[uriName]['fulltype'] = propResourceObj.typeobj.fulltype results[uriName]['success'] = True - rsvLogger.info("\t Type (%s), GET SUCCESS (time: %s)", propResourceObj.typeobj.stype, propResourceObj.rtime) # Printout FORMAT + rsvLogger.info("\t URI {}, Type ({}), GET SUCCESS (time: {})".format(URI, propResourceObj.typeobj.stype, propResourceObj.rtime)) uriName, SchemaFullType, jsondata = propResourceObj.name, propResourceObj.typeobj.fulltype, propResourceObj.jsondata SchemaNamespace, SchemaType = rst.getNamespace( @@ -636,7 +181,7 @@ def validateSingleURI(URI, profile, uriName='', expectedType=None, expectedSchem objRes = profile.get('Resources') if SchemaType not in objRes: - rsvLogger.info( + rsvLogger.debug( '\nNo Such Type in sample {} {}.{}, skipping'.format(URI, SchemaNamespace, SchemaType)) else: rsvLogger.info("\n*** %s, %s", uriName, URI) @@ -645,11 +190,11 @@ def validateSingleURI(URI, profile, uriName='', expectedType=None, expectedSchem objRes = objRes.get(SchemaType) rsvLogger.info(SchemaType) try: - propMessages, propCounts = validateInteropResource( + propMessages, propCounts = commonInterop.validateInteropResource( propResourceObj, objRes, jsondata) messages = messages.extend(propMessages) counts.update(propCounts) - except Exception as ex: + except Exception: rsvLogger.exception("Something went wrong") rsvLogger.error( 'Could not finish validation check on this payload') @@ -676,7 +221,6 @@ def validateURITree(URI, uriName, profile, expectedType=None, expectedSchema=Non # Resource level validation rcounts = Counter() rmessages = [] - rsuccess = True rerror = StringIO() objRes = dict(profile.get('Resources')) @@ -693,23 +237,23 @@ def validateURITree(URI, uriName, profile, expectedType=None, expectedSchema=Non serviceVersion = profile.get("Protocol") if serviceVersion is not None: serviceVersion = serviceVersion.get('MinVersion', '1.0.0') - msg, mpss = validateMinVersion(thisobj.jsondata.get("RedfishVersion", "0"), serviceVersion) + msg, mpss = commonInterop.validateMinVersion(thisobj.jsondata.get("RedfishVersion", "0"), serviceVersion) rmessages.append(msg) currentLinks = [(l, links[l], thisobj) for l in links] + # todo : churning a lot of links, causing possible slowdown even with set checks while len(currentLinks) > 0: newLinks = list() for linkName, link, parent in currentLinks: - if refLinks is not currentLinks and ('Links' in linkName.split('.', 1)[0] or 'RelatedItem' in linkName.split('.', 1)[0] or 'Redundancy' in linkName.split('.', 1)[0]): - refLinks.append((linkName, link, parent)) - continue - linkURI, autoExpand, linkType, linkSchema, innerJson = link if linkURI in allLinks or linkType == 'Resource.Item': continue - print('PARENT', parent.uri) + if refLinks is not currentLinks and ('Links' in linkName.split('.') or 'RelatedItem' in linkName.split('.') or 'Redundancy' in linkName.split('.')): + refLinks.append((linkName, link, parent)) + continue + if autoExpand and linkType is not None: linkSuccess, linkCounts, linkResults, innerLinks, linkobj = \ validateSingleURI(linkURI, profile, "{} -> {}".format(uriName, linkName), linkType, linkSchema, innerJson, parent=parent) @@ -731,8 +275,8 @@ def validateURITree(URI, uriName, profile, expectedType=None, expectedSchema=Non if SchemaType in objRes: traverseLogger.info("Checking service requirement for {}".format(SchemaType)) req = objRes[SchemaType].get("ReadRequirement", "Mandatory") - msg, pss = validateRequirement(req, None) - if pss and objRes[SchemaType].get('mark', False) == False: + msg, pss = commonInterop.validateRequirement(req, None) + if pss and not objRes[SchemaType].get('mark', False): rmessages.append(msg) msg.name = SchemaType + '.' + msg.name objRes[SchemaType]['mark'] = True @@ -741,9 +285,9 @@ def validateURITree(URI, uriName, profile, expectedType=None, expectedSchema=Non innerList = objRes[SchemaType]["ConditionalRequirements"] newList = list() for condreq in innerList: - condtrue = checkConditionalRequirement(linkobj, condreq, (linkobj.jsondata, None), None) + condtrue = commonInterop.checkConditionalRequirement(linkobj, condreq, (linkobj.jsondata, None), None) if condtrue: - msg, cpss = validateRequirement(condreq.get("ReadRequirement", "Mandatory"), None) + msg, cpss = commonInterop.validateRequirement(condreq.get("ReadRequirement", "Mandatory"), None) if cpss: rmessages.append(msg) msg.name = SchemaType + '.Conditional.' + msg.name @@ -753,43 +297,44 @@ def validateURITree(URI, uriName, profile, expectedType=None, expectedSchema=Non newList.append(condreq) objRes[SchemaType]["ConditionalRequirements"] = newList - currentLinks = newLinks - if len(currentLinks) == 0 and len(refLinks) > 0: - refLinks = OrderedDict() + if refLinks is not currentLinks and len(newLinks) == 0 and len(refLinks) > 0: currentLinks = refLinks + else: + currentLinks = newLinks # interop service level checks finalResults = OrderedDict() for left in objRes: - resultEnum = sEnum.FAIL + resultEnum = commonInterop.sEnum.FAIL if URI != "/redfish/v1": - resultEnum = sEnum.WARN + resultEnum = commonInterop.sEnum.WARN traverseLogger.info("We are not validating root, warn only") if not objRes[left].get('mark', False): req = objRes[left].get("ReadRequirement", "Mandatory") rmessages.append( - msgInterop(left + '.ReadRequirement', req, 'Must Exist' if req == "Mandatory" else 'Any', 'DNE', resultEnum)) + commonInterop.msgInterop(left + '.ReadRequirement', req, 'Must Exist' if req == "Mandatory" else 'Any', 'DNE', resultEnum)) if "ConditionalRequirements" in objRes[left]: innerList = objRes[left]["ConditionalRequirements"] for condreq in innerList: req = condreq.get("ReadRequirement", "Mandatory") rmessages.append( - msgInterop(left + '.Conditional.ReadRequirement', req, 'Must Exist' if req == "Mandatory" else 'Any', 'DNE', resultEnum)) + commonInterop.msgInterop(left + '.Conditional.ReadRequirement', req, 'Must Exist' if req == "Mandatory" else 'Any', 'DNE', resultEnum)) for item in rmessages: - if item.success == sEnum.WARN: + if item.success == commonInterop.sEnum.WARN: rcounts['warn'] += 1 - elif item.success == sEnum.PASS: + elif item.success == commonInterop.sEnum.PASS: rcounts['pass'] += 1 - elif item.success == sEnum.FAIL: + elif item.success == commonInterop.sEnum.FAIL: rcounts['fail.{}'.format(item.name)] += 1 finalResults['n/a'] = {'uri': "Service Level Requirements", 'success':rcounts.get('fail', 0) == 0,\ 'counts':rcounts,\ 'messages':rmessages, 'errors':rerror.getvalue(), 'warns': '',\ 'rtime':'', 'context':'', 'fulltype':''} - for l in allLinks: - print (l) + for l in sorted(allLinks): + print(l) + print(len(allLinks)) finalResults.update(results) rerror.close() @@ -858,7 +403,7 @@ def main(arglist=None, direct_parser=None): cdict = rst.convertConfigParserToDict(direct_parser) rst.setConfig(cdict) except Exception as ex: - rsvLogger.exception("Something went wrong") # Printout FORMAT + rsvLogger.exception("Something went wrong") return 1, None, 'Config Parser Exception' elif args.config is None and args.ip is None: rsvLogger.info('No ip or config specified.') @@ -868,13 +413,14 @@ def main(arglist=None, direct_parser=None): try: rst.setByArgparse(args) except Exception: - rsvLogger.exception("Something went wrong") # Printout FORMAT + rsvLogger.exception("Something went wrong") return 1, None, 'Config Exception' config = rst.config # Set interop config items config['WarnRecommended'] = rst.config.get('warnrecommended', args.warnrecommended) + commonInterop.config['WarnRecommended'] = config['WarnRecommended'] config['profile'] = args.profile config['schema'] = args.schema @@ -893,16 +439,16 @@ def main(arglist=None, direct_parser=None): if not os.path.isdir(logpath): os.makedirs(logpath) fmt = logging.Formatter('%(levelname)s - %(message)s') - fh = logging.FileHandler(datetime.strftime(startTick, os.path.join(logpath, "ConformanceLog_%m_%d_%Y_%H%M%S.txt"))) + fh = logging.FileHandler(datetime.strftime(startTick, os.path.join(logpath, "InteropLog_%m_%d_%Y_%H%M%S.txt"))) fh.setLevel(min(args.debug_logging, args.verbose_checks)) fh.setFormatter(fmt) - rsvLogger.addHandler(fh) # Printout FORMAT + rsvLogger.addHandler(fh) # Then start service try: currentService = rst.startService() except Exception as ex: - rsvLogger.error("Service could not be started: {}".format(ex)) # Printout FORMAT + rsvLogger.error("Service could not be started: {}".format(ex)) return 1, None, 'Service Exception' metadata = currentService.metadata @@ -910,10 +456,11 @@ def main(arglist=None, direct_parser=None): # start printing rsvLogger.info('ConfigURI: ' + ConfigURI) - rsvLogger.info('System Info: ' + sysDescription) # Printout FORMAT + rsvLogger.info('System Info: ' + sysDescription) + rsvLogger.info('Profile:' + config['profile']) rsvLogger.info('\n'.join( - ['{}: {}'.format(x, config[x]) for x in sorted(list(config.keys() - set(['systeminfo', 'targetip', 'password', 'description'])))])) - rsvLogger.info('Start time: ' + startTick.strftime('%x - %X')) # Printout FORMAT + ['{}: {}'.format(x, config[x]) for x in sorted(list(config.keys() - set(['systeminfo', 'targetip', 'password', 'description', 'profile'])))])) + rsvLogger.info('Start time: ' + startTick.strftime('%x - %X')) # Interop Profile handling profile = schema = None @@ -966,16 +513,12 @@ def main(arglist=None, direct_parser=None): for x in resultsNew[item]['messages']: x.name = profileName + ' -- ' + x.name results[item]['messages'].extend(resultsNew[item]['messages']) - else: - newKey = profileName + '...' + key - input(newKey) - results[newKey] = resultsNew[key] #resultsNew = {profileName+key: resultsNew[key] for key in resultsNew if key in results} #results.update(resultsNew) finalCounts = Counter() nowTick = datetime.now() - rsvLogger.info('Elapsed time: {}'.format(str(nowTick-startTick).rsplit('.', 1)[0])) # Printout FORMAT + rsvLogger.info('Elapsed time: {}'.format(str(nowTick-startTick).rsplit('.', 1)[0])) finalCounts.update(metadata.get_counter()) for item in results: @@ -984,9 +527,13 @@ def main(arglist=None, direct_parser=None): # detect if there are error messages for this resource, but no failure counts; if so, add one to the innerCounts counters_all_pass = True for countType in sorted(innerCounts.keys()): + if innerCounts.get(countType) == 0: + continue if any(x in countType for x in ['problem', 'fail', 'bad', 'exception']): counters_all_pass = False - break + if 'fail' in countType or 'exception' in countType: + rsvLogger.error('{} {} errors in {}'.format(innerCounts[countType], countType, results[item]['uri'])) + innerCounts[countType] += 0 error_messages_present = False if results[item]['errors'] is not None and len(results[item]['errors']) > 0: error_messages_present = True @@ -1009,7 +556,7 @@ def main(arglist=None, direct_parser=None): html_str = renderHtml(results, finalCounts, tool_version, startTick, nowTick) - lastResultsPage = datetime.strftime(startTick, os.path.join(logpath, "ConformanceHtmlLog_%m_%d_%Y_%H%M%S.html")) + lastResultsPage = datetime.strftime(startTick, os.path.join(logpath, "InteropHtmlLog%m_%d_%Y_%H%M%S.html")) writeHtml(html_str, lastResultsPage) @@ -1024,86 +571,6 @@ def main(arglist=None, direct_parser=None): return status_code, lastResultsPage, 'Validation done' - """ - rsvLogger.info(len(results)) - for cnt, item in enumerate(results): - printPayload = False - innerCounts = results[item][2] - finalCounts.update(innerCounts) - if results[item][3] is not None and len(results[item][3]) == 0: - continue - htmlStr += '' - htmlStr += ''.format(results[item][0], cnt, cnt) - htmlStr += ''.format(item, results[item][5], results[item][6]) - htmlStr += '' - htmlStr += '' - htmlStr += '
{}
\ -
Show results
\ -
URI: {}
XML: {}
type: {}
 GET Success' if results[item] - [1] else 'class="fail"> GET Failure') + '' - - for countType in sorted(innerCounts.keys()): - if innerCounts.get(countType) == 0: - continue - if 'fail' in countType or 'exception' in countType: - rsvLogger.error('{} {} errors in {}'.format(innerCounts[countType], countType, results[item][0].split(' ')[0])) - innerCounts[countType] += 0 - htmlStr += '
{p}: {q}
'.format( - p=countType, - q=innerCounts.get(countType, 0), - style='class="fail log"' if 'fail' in countType or 'exception' in countType else 'class="warn log"' if 'warn' in countType.lower() else 'class=log') - htmlStr += '
' - htmlStr += '' - if results[item][4] is not None: - htmlStr += '' - htmlStr += "
'.format(cnt) - if results[item][3] is not None: - for i in results[item][3]: - htmlStr += '' - htmlStr += '' - htmlStr += '' - htmlStr += '' - htmlStr += '' - htmlStr += ''.format(str(i.success.value).lower(), str(i.success.value)) - htmlStr += '' - htmlStr += '
Name Entry Value must be Service Value Success
' + str(i.name) + '' + str(i.entry) + '' + str(i.expected) + '' + str(i.actual) + '{}
' + str(results[item][4].getvalue()).replace('\n', '
') + '
Payload\ -

{}

\ -
".format(json.dumps(results[item][7], - indent=4, separators=(',', ': ')) if len(results[item]) >= 8 else "n/a") - - htmlStr += '' - - htmlStrTotal = '
Final counts: ' - for countType in sorted(finalCounts.keys()): - if finalCounts.get(countType) == 0: - continue - htmlStrTotal += '{p}: {q}, '.format(p=countType, q=finalCounts.get(countType, 0)) - htmlStrTotal += '
Expand All
' - htmlStrTotal += '
Collapse All
' - - htmlPage = htmlStrTop + htmlStrBodyHeader + htmlStrTotal + htmlStr - - with open(datetime.strftime(startTick, os.path.join(logpath, "ConformanceHtmlLog_%m_%d_%Y_%H%M%S.html")), 'w') as f: - f.write(htmlPage) - """ - - fails = 0 - for key in finalCounts: - if 'problem' in key or 'fail' in key or 'exception' in key: - fails += finalCounts[key] - - success = success and not (fails > 0) - rsvLogger.info(finalCounts) - - if not success: - rsvLogger.info("Validation has failed: %d problems found", fails) - else: - rsvLogger.info("Validation has succeeded.") - status_code = 0 - - return status_code - if __name__ == '__main__': status_code, lastResultsPage, exit_string = main() diff --git a/commonInterop.py b/commonInterop.py new file mode 100644 index 0000000..0c34e51 --- /dev/null +++ b/commonInterop.py @@ -0,0 +1,481 @@ + +# Copyright Notice: +# Copyright 2016 Distributed Management Task Force, Inc. All rights reserved. +# License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/Redfish-Interop-Validator/blob/master/LICENSE.md + +import io +import os +import sys +import re +from datetime import datetime +from collections import Counter, OrderedDict +import logging +import json +import traverseService as rst +import jsonschema +import argparse +from enum import Enum +from io import StringIO + +from commonProfile import getProfiles, checkProfileAgainstSchema +from traverseService import AuthenticationError +from tohtml import renderHtml, writeHtml + +rsvLogger = rst.getLogger() + +config = {'WarnRecommended': False} + +class sEnum(Enum): + FAIL = 'FAIL' + PASS = 'PASS' + WARN = 'WARN' + +class msgInterop: + def __init__(self, name, entry, expected, actual, success): + self.name = name + self.entry = entry + self.expected = expected + self.actual = actual + if isinstance(success, bool): + self.success = sEnum.PASS if success else sEnum.FAIL + else: + self.success = success + self.parent = None + + +def validateRequirement(entry, decodeditem, conditional=False): + """ + Validates Requirement entry + """ + propDoesNotExist = (decodeditem == 'DNE') + rsvLogger.info('Testing ReadRequirement \n\texpected:' + str(entry) + ', exists: ' + str(not propDoesNotExist)) + # If we're not mandatory, pass automatically, else fail + # However, we have other entries "IfImplemented" and "Conditional" + # note: Mandatory is default!! if present in the profile. Make sure this is made sure. + originalentry = entry + if entry == "IfImplemented" or (entry == "Conditional" and conditional): + entry = "Mandatory" + paramPass = not entry == "Mandatory" or \ + entry == "Mandatory" and not propDoesNotExist + if entry == "Recommended" and propDoesNotExist: + rsvLogger.info('\tItem is recommended but does not exist') + if config['WarnRecommended']: + rsvLogger.error('\tItem is recommended but does not exist, escalating to WARN') + paramPass = sEnum.WARN + + rsvLogger.info('\tpass ' + str(paramPass)) + if not paramPass: + rsvLogger.error('\tNoPass') + return msgInterop('ReadRequirement', originalentry, 'Must Exist' if entry == "Mandatory" else 'Any', 'Exists' if not propDoesNotExist else 'DNE', paramPass),\ + paramPass + + +def isPropertyValid(profilePropName, rObj): + for prop in rObj.getResourceProperties(): + if profilePropName == prop.propChild: + return None, True + rsvLogger.error('{} - Does not exist in ResourceType Schema, please consult profile provided'.format(profilePropName)) + return msgInterop('PropertyValidity', profilePropName, 'Should Exist', 'in ResourceType Schema', False), False + + +def validateMinCount(alist, length, annotation=0): + """ + Validates Mincount annotation + """ + rsvLogger.info('Testing minCount \n\texpected:' + str(length) + ', val:' + str(annotation)) + paramPass = len(alist) >= length or annotation >= length + rsvLogger.info('\tpass ' + str(paramPass)) + if not paramPass: + rsvLogger.error('\tNoPass') + return msgInterop('MinCount', length, '<=', annotation if annotation > len(alist) else len(alist), paramPass),\ + paramPass + + +def validateSupportedValues(enumlist, annotation): + """ + Validates SupportedVals annotation + """ + rsvLogger.info('Testing supportedValues \n\t:' + str(enumlist) + ', exists:' + str(annotation)) + for item in enumlist: + paramPass = item in annotation + if not paramPass: + break + rsvLogger.info('\tpass ' + str(paramPass)) + if not paramPass: + rsvLogger.error('\tNoPass') + return msgInterop('SupportedValues', enumlist, 'included in...', annotation, paramPass),\ + paramPass + + +def findPropItemforString(propObj, itemname): + """ + Finds an appropriate object for an item + """ + for prop in propObj.getResourceProperties(): + decodedName = prop.name.split(':')[-1] + if itemname == decodedName: + return prop + return None + + +def validateWriteRequirement(propObj, entry, itemname): + """ + Validates if a property is WriteRequirement or not + """ + rsvLogger.info('writeable \n\t' + str(entry)) + permission = 'Read' + expected = "OData.Permission/ReadWrite" if entry else "Any" + if entry: + targetProp = findPropItemforString(propObj, itemname.replace('#', '')) + propAttr = None + if targetProp is not None: + propAttr = targetProp.propDict.get('OData.Permissions') + if propAttr is not None: + permission = propAttr.get('EnumMember', 'Read') + paramPass = permission \ + == "OData.Permission/ReadWrite" + else: + paramPass = False + else: + paramPass = True + + rsvLogger.info('\tpass ' + str(paramPass)) + if not paramPass: + rsvLogger.error('\tNoPass') + return msgInterop('WriteRequirement', entry, expected, permission, paramPass),\ + paramPass + + +def checkComparison(val, compareType, target): + """ + Validate a given comparison option, given a value and a target set + """ + rsvLogger.info('Testing a comparison \n\t' + str((val, compareType, target))) + vallist = val if isinstance(val, list) else [val] + paramPass = False + if compareType == "AnyOf": + for item in vallist: + paramPass = item in target + if paramPass: + break + else: + continue + + if compareType == "AllOf": + alltarget = set() + for item in vallist: + paramPass = item in target and item not in alltarget + if paramPass: + alltarget.add(item) + if len(alltarget) == len(target): + break + else: + continue + paramPass = len(alltarget) == len(target) + if compareType == "LinkToResource": + vallink = val.get('@odata.id') + success, decoded, code, elapsed = rst.callResourceURI(vallink) + if success: + ourType = decoded.get('@odata.type') + if ourType is not None: + SchemaType = rst.getType(ourType) + paramPass = SchemaType in target + else: + paramPass = False + else: + paramPass = False + + if compareType == "Equal": + paramPass = val == target + if compareType == "NotEqual": + paramPass = val != target + if compareType == "GreaterThan": + paramPass = val > target + if compareType == "GreaterThanOrEqual": + paramPass = val >= target + if compareType == "LessThan": + paramPass = val < target + if compareType == "LessThanOrEqual": + paramPass = val <= target + if compareType == "Absent": + paramPass = val == 'DNE' + if compareType == "Present": + paramPass = val != 'DNE' + rsvLogger.info('\tpass ' + str(paramPass)) + if not paramPass: + rsvLogger.error('\tNoPass') + return msgInterop('Comparison', target, compareType, val, paramPass),\ + paramPass + + +def validateMembers(members, entry, annotation): + """ + Validate an entry of Members and its count annotation + """ + rsvLogger.info('Testing members \n\t' + str((members, entry, annotation))) + if not validateRequirement('Mandatory', members): + return False + if "MinCount" in entry: + mincount, mincountpass = validateMinCount(members, entry["MinCount"], annotation) + mincount.name = 'MembersMinCount' + return mincount, mincountpass + + +def validateMinVersion(fulltype, entry): + """ + Checks for the minimum version of a resource's type + """ + fulltype = fulltype.replace('#', '') + rsvLogger.info('Testing minVersion \n\t' + str((fulltype, entry))) + # If fulltype doesn't contain version as is, try it as v#_#_# + versionSplit = entry.split('.') + versionNew = 'v' + for x in versionSplit: + versionNew = versionNew + x + '_' + versionNew = versionNew[:-1] + # get version from payload + v_payload = rst.getNamespace(fulltype).split('.', 1)[-1] + # use string comparison, given version numbering is accurate to regex + paramPass = v_payload >= (versionNew if 'v' in v_payload else entry) + rsvLogger.info('\tpass ' + str(paramPass)) + if not paramPass: + rsvLogger.error('\tNo Pass') + return msgInterop('MinVersion', '{} ({})'.format(entry, versionNew), '<=', fulltype, paramPass),\ + paramPass + + +def checkConditionalRequirement(propResourceObj, entry, decodedtuple, itemname): + """ + Returns boolean if entry's conditional is true or false + """ + rsvLogger.info('Evaluating conditionalRequirements') + if "SubordinateToResource" in entry: + isSubordinate = False + # iterate through parents via resourceObj + # list must be reversed to work backwards + resourceParent = propResourceObj.parent + for expectedParent in reversed(entry["SubordinateToResource"]): + if resourceParent is not None: + parentType = resourceParent.typeobj.stype + isSubordinate = parentType == expectedParent + rsvLogger.info('\tsubordinance ' + + str(parentType) + ' ' + str(isSubordinate)) + resourceParent = resourceParent.parent + else: + rsvLogger.info('no parent') + isSubordinate = False + return isSubordinate + if "CompareProperty" in entry: + decodeditem, decoded = decodedtuple + # find property in json payload by working backwards thru objects + # decoded tuple is designed just for this piece, since there is + # no parent in dictionaries + comparePropName = entry["CompareProperty"] + while comparePropName not in decodeditem and decoded is not None: + decodeditem, decoded = decoded + compareProp = decodeditem.get(comparePropName, 'DNE') + return checkComparison(compareProp, entry["Comparison"], entry.get("CompareValues", []))[1] + + +def validatePropertyRequirement(propResourceObj, entry, decodedtuple, itemname, chkCondition=False): + """ + Validate PropertyRequirements + """ + msgs = [] + counts = Counter() + decodeditem, decoded = decodedtuple + if entry is None or len(entry) == 0: + rsvLogger.debug('there are no requirements for this prop') + else: + rsvLogger.info('propRequirement with value: ' + str(decodeditem if not isinstance( + decodeditem, dict) else 'dict')) + # If we're working with a list, then consider MinCount, Comparisons, then execute on each item + # list based comparisons include AnyOf and AllOf + if isinstance(decodeditem, list): + rsvLogger.info("inside of a list: " + itemname) + if "MinCount" in entry: + msg, success = validateMinCount(decodeditem, entry["MinCount"], + decoded[0].get(itemname.split('.')[-1] + '@odata.count', 0)) + msgs.append(msg) + msg.name = itemname + '.' + msg.name + for k, v in entry.get('PropertyRequirements', {}).items(): + # default to AnyOf if Comparison is not present but Values is + comparisonValue = v.get("Comparison", "AnyOf") if v.get("Values") is not None else None + if comparisonValue in ["AllOf", "AnyOf"]: + msg, success = (checkComparison([val.get(k, 'DNE') for val in decodeditem], + comparisonValue, v["Values"])) + msgs.append(msg) + msg.name = itemname + '.' + msg.name + cnt = 0 + for item in decodeditem: + listmsgs, listcounts = validatePropertyRequirement( + propResourceObj, entry, (item, decoded), itemname + '#' + str(cnt)) + counts.update(listcounts) + msgs.extend(listmsgs) + cnt += 1 + + else: + # consider requirement before anything else + # problem: if dne, skip? + + # Read Requirement is default mandatory if not present + msg, success = validateRequirement(entry.get('ReadRequirement', 'Mandatory'), decodeditem) + msgs.append(msg) + msg.name = itemname + '.' + msg.name + + if "WriteRequirement" in entry: + msg, success = validateWriteRequirement(propResourceObj, entry["WriteRequirement"], itemname) + msgs.append(msg) + msg.name = itemname + '.' + msg.name + if "ConditionalRequirements" in entry: + innerList = entry["ConditionalRequirements"] + for item in innerList: + if checkConditionalRequirement(propResourceObj, item, decodedtuple, itemname): + rsvLogger.info("\tCondition DOES apply") + conditionalMsgs, conditionalCounts = validatePropertyRequirement( + propResourceObj, item, decodedtuple, itemname, chkCondition = True) + counts.update(conditionalCounts) + for item in conditionalMsgs: + item.name = item.name.replace('.', '.Conditional.', 1) + msgs.extend(conditionalMsgs) + else: + rsvLogger.info("\tCondition does not apply") + if "MinSupportValues" in entry: + msg, success = validateSupportedValues( + decodeditem, entry["MinSupportValues"], + decoded[0].get(itemname.split('.')[-1] + '@Redfish.AllowableValues', [])) + msgs.append(msg) + msg.name = itemname + '.' + msg.name + if "Comparison" in entry and not chkCondition and\ + entry["Comparison"] not in ["AnyOf", "AllOf"]: + msg, success = checkComparison(decodeditem, entry["Comparison"], entry.get("Values",[])) + msgs.append(msg) + msg.name = itemname + '.' + msg.name + if "PropertyRequirements" in entry: + innerDict = entry["PropertyRequirements"] + if isinstance(decodeditem, dict): + for item in innerDict: + rsvLogger.info('inside complex ' + itemname + '.' + item) + complexMsgs, complexCounts = validatePropertyRequirement( + propResourceObj, innerDict[item], (decodeditem.get(item, 'DNE'), decodedtuple), item) + msgs.extend(complexMsgs) + counts.update(complexCounts) + else: + rsvLogger.info('complex {} is missing or not a dictionary'.format(itemname + '.' + item, None)) + return msgs, counts + + +def validateActionRequirement(propResourceObj, entry, decodedtuple, actionname): + """ + Validate Requirements for one action + """ + decodeditem, decoded = decodedtuple + counts = Counter() + msgs = [] + rsvLogger.info('actionRequirement \n\tval: ' + str(decodeditem if not isinstance( + decodeditem, dict) else 'dict') + ' ' + str(entry)) + if "ReadRequirement" in entry: + # problem: if dne, skip + msg, success = validateRequirement(entry.get('ReadRequirement', "Mandatory"), decodeditem) + msgs.append(msg) + msg.name = actionname + '.' + msg.name + propDoesNotExist = (decodeditem == 'DNE') + if propDoesNotExist: + return msgs, counts + # problem: if dne, skip + if "Parameters" in entry: + innerDict = entry["Parameters"] + for k in innerDict: + item = innerDict[k] + annotation = decodeditem.get(str(k) + '@Redfish.AllowableValues', 'DNE') + # problem: if dne, skip + # assume mandatory + msg, success = validateRequirement(item.get('ReadRequirement', "Mandatory"), annotation) + msgs.append(msg) + msg.name = actionname + '.Parameters.' + msg.name + if annotation == 'DNE': + continue + if "ParameterValues" in item: + msg, success = validateSupportedValues( + item["ParameterValues"], annotation) + msgs.append(msg) + msg.name = actionname + '.' + msg.name + if "RecommendedValues" in item: + msg, success = validateSupportedValues( + item["RecommendedValues"], annotation) + msg.name = msg.name.replace('Supported', 'Recommended') + if config['WarnRecommended'] and not success: + rsvLogger.error('\tRecommended parameters do not all exist, escalating to WARN') + msg.success = sEnum.WARN + elif not success: + rsvLogger.error('\tRecommended parameters do not all exist, but are not Mandatory') + msg.success = sEnum.PASS + + msgs.append(msg) + msg.name = actionname + '.' + msg.name + # consider requirement before anything else, what if action + # if the action doesn't exist, you can't check parameters + # if it doesn't exist, what should not be checked for action + return msgs, counts + + +def validateInteropResource(propResourceObj, interopDict, decoded): + """ + Base function that validates a single Interop Resource by its entry + """ + msgs = [] + rsvLogger.info('### Validating an InteropResource') + rsvLogger.debug(str(interopDict)) + counts = Counter() + # decodedtuple provides the chain of dicts containing dicts, needed for CompareProperty + decodedtuple = (decoded, None) + if "MinVersion" in interopDict: + msg, success = validateMinVersion(propResourceObj.typeobj.fulltype, interopDict['MinVersion']) + msgs.append(msg) + if "PropertyRequirements" in interopDict: + # problem, unlisted in 0.9.9a + innerDict = interopDict["PropertyRequirements"] + for item in innerDict: + vmsg, isvalid = isPropertyValid(item, propResourceObj) + if not isvalid: + msgs.append(vmsg) + vmsg.name = '{}.{}'.format(item, vmsg.name) + continue + rsvLogger.info('### Validating PropertyRequirements for {}'.format(item)) + pmsgs, pcounts = validatePropertyRequirement( + propResourceObj, innerDict[item], (decoded.get(item, 'DNE'), decodedtuple), item) + rsvLogger.info(pcounts) + counts.update(pcounts) + msgs.extend(pmsgs) + if "ActionRequirements" in interopDict: + innerDict = interopDict["ActionRequirements"] + actionsJson = decoded.get('Actions', {}) + decodedInnerTuple = (actionsJson, decodedtuple) + for item in innerDict: + actionName = '#' + propResourceObj.typeobj.stype + '.' + item + rsvLogger.info(actionName) + amsgs, acounts = validateActionRequirement(propResourceObj, innerDict[item], (actionsJson.get( + actionName, 'DNE'), decodedInnerTuple), actionName) + rsvLogger.info(acounts) + counts.update(acounts) + msgs.extend(amsgs) + if "CreateResource" in interopDict: + rsvLogger.info('Skipping CreateResource') + pass + if "DeleteResource" in interopDict: + rsvLogger.info('Skipping DeleteResource') + pass + if "UpdateResource" in interopDict: + rsvLogger.info('Skipping UpdateResource') + pass + + for item in msgs: + if item.success == sEnum.WARN: + counts['warn'] += 1 + elif item.success == sEnum.PASS: + counts['pass'] += 1 + elif item.success == sEnum.FAIL: + counts['fail.{}'.format(item.name)] += 1 + rsvLogger.info(counts) + return msgs, counts + diff --git a/tohtml.py b/tohtml.py index cd7b8e7..03043c3 100644 --- a/tohtml.py +++ b/tohtml.py @@ -9,7 +9,6 @@ import html - def wrapTag(string, tag='div', attr=None): string = str(string) ltag, rtag = '<{}>'.format(tag), ''.format(tag) @@ -96,43 +95,46 @@ def renderHtml(results, finalCounts, tool_version, startTick, nowTick): .titletable {width:100%}\ \ ' - htmlStrBodyHeader = \ - '' \ - '

System: ' + ConfigURI + ' Description: ' + sysDescription + '

' \ - '' \ - '' \ - '

Configuration:

' \ - '

' + str(config_str.replace('\n', '
')) + '

' \ - '' \ - '' htmlStrBodyHeader = '' # Logo and logname infos = [wrapTag('##### Redfish Conformance Test Report #####', 'h2')] infos.append(wrapTag('DMTF Redfish Logo', 'h4')) - infos.append('

' - 'https://github.com/DMTF/Redfish-Service-Validator

') + infos.append('

' + 'https://github.com/DMTF/Redfish-Interop-Validator

') infos.append('Tool Version: {}'.format(tool_version)) infos.append(startTick.strftime('%c')) infos.append('(Run time: {})'.format( str(nowTick-startTick).rsplit('.', 1)[0])) infos.append('

This tool is provided and maintained by the DMTF. ' 'For feedback, please open issues
in the tool\'s Github repository: ' - '' - 'https://github.com/DMTF/Redfish-Service-Validator/issues

') + '' + 'https://github.com/DMTF/Redfish-Interop-Validator/issues') htmlStrBodyHeader += tr(th(infoBlock(infos))) infos = {'System': ConfigURI, 'Description': sysDescription} htmlStrBodyHeader += tr(th(infoBlock(infos))) - infos = {x: config[x] for x in config if x not in ['systeminfo', 'targetip', 'password', 'description']} + infos = {'Profile': config['profile'], 'Schema': config['schema']} + htmlStrBodyHeader += tr(th(infoBlock(infos))) + + infos = {x: config[x] for x in config if x not in ['systeminfo', 'targetip', 'password', 'description', 'profile', 'schema']} block = tr(th(infoBlock(infos, '|||'))) for num, block in enumerate(block.split('|||'), 1): sep = '
' if num % 4 == 0 else ', ' sep = '' if num == len(infos) else sep htmlStrBodyHeader += block + sep + htmlStrTotal = '
Final counts: ' + for countType in sorted(finalCounts.keys()): + if finalCounts.get(countType) == 0: + continue + htmlStrTotal += '{p}: {q}, '.format(p=countType, q=finalCounts.get(countType, 0)) + htmlStrTotal += '
Expand All
' + htmlStrTotal += '
Collapse All
' + + htmlStrBodyHeader += tr(td(htmlStrTotal)) htmlPage = rst.currentService.metadata.to_html() for cnt, item in enumerate(results): @@ -140,6 +142,9 @@ def renderHtml(results, finalCounts, tool_version, startTick, nowTick): val = results[item] rtime = '(response time: {})'.format(val['rtime']) + if len(val['messages']) == 0: + continue + # uri block prop_type = val['fulltype'] if prop_type is not None: @@ -176,10 +181,21 @@ def renderHtml(results, finalCounts, tool_version, startTick, nowTick): rhead = wrapTag(''.join(rhead), *x) entry.append(rhead) - """ + htmlStr = '' + # actual table - rows = [[m] + list(val['messages'][m]) for m in val['messages']] - titles = ['Property Name', 'Value', 'Type', 'Exists', 'Result'] + rows = [(i.name, i.entry, i.expected, i.actual, str(i.success.value)) for i in val['messages']] + titles = ['Property Name', 'Value', 'Expected', 'Actual', 'Result'] widths = ['15','30','30','10','15'] tableHeader = tableBlock(rows, titles, widths, ffunc=applySuccessColor) @@ -205,7 +221,6 @@ def renderHtml(results, finalCounts, tool_version, startTick, nowTick): tableHeader = td(tableHeader, 'class="results" id=\'resNum{}\''.format(cnt)) entry.append(tableHeader) - """ # append htmlPage += ''.join([tr(x) for x in entry])
'.format(cnt) + if results[item]['messages'] is not None: + for i in results[item]['messages']: + htmlStr += '' + htmlStr += '' + htmlStr += '' + htmlStr += '' + htmlStr += '' + htmlStr += ''.format(str(i.success.value).lower(), str(i.success.value)) + htmlStr += '' + htmlStr += '
Name Entry Value must be Service Value Success
' + str(i.name) + '' + str(i.entry) + '' + str(i.expected) + '' + str(i.actual) + '{}