diff --git a/commonRedfish.py b/commonRedfish.py index 6a55bd1..ccae0b1 100644 --- a/commonRedfish.py +++ b/commonRedfish.py @@ -1,5 +1,10 @@ +# Copyright Notice: +# Copyright 2016-2018 DMTF. All rights reserved. +# License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/Redfish-Interop-Validator/blob/master/LICENSE.md import re +import traverseService as rst + """ Power.1.1.1.Power , Power.v1_0_0.Power @@ -9,6 +14,7 @@ def navigateJsonFragment(decoded, URILink): + traverseLogger = rst.getLogger() if '#' in URILink: URILink, frag = tuple(URILink.rsplit('#', 1)) fragNavigate = frag.split('/') @@ -41,6 +47,7 @@ def getNamespace(string: str): string = string.rsplit('#', 1)[1] return string.rsplit('.', 1)[0] + def getVersion(string: str): """getVersion diff --git a/metadata.py b/metadata.py index 7c3267e..c0b3cc8 100644 --- a/metadata.py +++ b/metadata.py @@ -19,7 +19,7 @@ EDMX_TAGS = ['DataServices', 'Edmx', 'Include', 'Reference'] -live_zip_uri = 'http://redfish.dmtf.org/schemas/DSP8010_2018.1.zip' +live_zip_uri = 'http://redfish.dmtf.org/schemas/DSP8010_2018.3.zip' def setup_schema_pack(uri, local_dir, proxies, timeout): @@ -27,6 +27,8 @@ def setup_schema_pack(uri, local_dir, proxies, timeout): if uri == 'latest': uri = live_zip_uri try: + if not os.path.isdir(local_dir): + os.makedirs(local_dir) response = requests.get(uri, timeout=timeout, proxies=proxies) expCode = [200] elapsed = response.elapsed.total_seconds() diff --git a/rfSchema.py b/rfSchema.py index f5d3eab..dc53e84 100644 --- a/rfSchema.py +++ b/rfSchema.py @@ -1,3 +1,6 @@ +# Copyright Notice: +# Copyright 2016-2018 DMTF. All rights reserved. +# License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/Redfish-Interop-Validator/blob/master/LICENSE.md from collections import namedtuple from bs4 import BeautifulSoup @@ -5,6 +8,7 @@ from collections import OrderedDict import re import difflib +import os.path from commonRedfish import getType, getNamespace, getNamespaceUnversioned, getVersion import traverseService as rst @@ -12,6 +16,34 @@ config = [] + +def storeSchemaToLocal(xml_data, origin): + """storeSchemaToLocal + + Moves data pulled from service/online to local schema storage + + Does NOT do so if preferonline is specified + + :param xml_data: data being transferred + :param origin: origin of xml pulled + """ + config = rst.config + SchemaLocation = config['metadatafilepath'] + if not config['preferonline']: + if not os.path.isdir(SchemaLocation): + os.makedirs(SchemaLocation) + if 'localFile' not in origin and '$metadata' not in origin: + __, xml_name = origin.rsplit('/', 1) + new_file = os.path.join(SchemaLocation, xml_name) + if not os.path.isfile(new_file): + with open(new_file, "w") as filehandle: + filehandle.write(xml_data) + rst.traverseLogger.info('Writing online XML to file: {}'.format(xml_name)) + else: + rst.traverseLogger.info('NOT writing online XML to file: {}'.format(xml_name)) + else: + pass + @lru_cache(maxsize=64) def getSchemaDetails(SchemaType, SchemaURI): """ @@ -40,6 +72,8 @@ def getSchemaDetails(SchemaType, SchemaURI): if success: return success, soup, origin + xml_suffix = currentService.config['schemasuffix'] + config = rst.currentService.config LocalOnly, SchemaLocation, ServiceOnly = config['localonlymode'], config['metadatafilepath'], config['servicemode'] @@ -76,10 +110,11 @@ def getSchemaDetails(SchemaType, SchemaURI): rst.traverseLogger.error( "SchemaURI missing reference link {} inside {}".format(frag, base_schema_uri)) # error reported; assume likely schema uri to allow continued validation - uri = 'http://redfish.dmtf.org/schemas/v1/{}_v1.xml'.format(frag) + uri = 'http://redfish.dmtf.org/schemas/v1/{}{}'.format(frag, xml_suffix) rst.traverseLogger.info("Continue assuming schema URI for {} is {}".format(SchemaType, uri)) return getSchemaDetails(SchemaType, uri) else: + storeSchemaToLocal(data, base_schema_uri) return True, soup, base_schema_uri if not inService and ServiceOnly: rst.traverseLogger.debug("Nonservice URI skipped: {}".format(base_schema_uri)) @@ -89,10 +124,9 @@ def getSchemaDetails(SchemaType, SchemaURI): rst.traverseLogger.debug("This program is currently LOCAL ONLY") if ServiceOnly: rst.traverseLogger.debug("This program is currently SERVICE ONLY") - if not LocalOnly and not ServiceOnly and not inService and config['preferonline']: + if not LocalOnly and not ServiceOnly or (not inService and config['preferonline']): rst.traverseLogger.warning("SchemaURI {} was unable to be called, defaulting to local storage in {}".format(SchemaURI, SchemaLocation)) - return getSchemaDetailsLocal(SchemaType, SchemaURI) - return False, None, None + return getSchemaDetailsLocal(SchemaType, SchemaURI) def getSchemaDetailsLocal(SchemaType, SchemaURI): @@ -141,10 +175,10 @@ def getSchemaDetailsLocal(SchemaType, SchemaURI): rst.traverseLogger.error('Could not find item in $metadata {}'.format(frag)) return False, None, None else: - return True, soup, "local" + SchemaLocation + '/' + filestring + return True, soup, "localFile:" + SchemaLocation + '/' + filestring if FoundAlias in Alias: - return True, soup, "local" + SchemaLocation + '/' + filestring + return True, soup, "localFile:" + SchemaLocation + '/' + filestring except FileNotFoundError: # if we're looking for $metadata locally... ditch looking for it, go straight to file @@ -254,13 +288,13 @@ def getTypeTagInSchema(self, currentType, tagType=['EntityType', 'ComplexType']) pnamespace, ptype = getNamespace(currentType), getType(currentType) soup = self.soup - currentSchema = soup.find( # BS4 line + currentSchema = soup.find( 'Schema', attrs={'Namespace': pnamespace}) if currentSchema is None: return None - currentEntity = currentSchema.find(tagType, attrs={'Name': ptype}, recursive=False) # BS4 line + currentEntity = currentSchema.find(tagType, attrs={'Name': ptype}, recursive=False) return currentEntity @@ -346,6 +380,7 @@ def getSchemaObject(typename, uri, metadata=None): class PropType: robjcache = {} + def __init__(self, typename, schemaObj): # if we've generated this type, use it, else generate type self.initiated = False @@ -444,18 +479,18 @@ def getActions(self): def compareURI(self, uri, my_id): expected_uris = self.expectedURI - uri = uri.rstrip('/') if expected_uris is not None: regex = re.compile(r"{.*?}") for e in expected_uris: e_left, e_right = tuple(e.rsplit('/', 1)) + _uri_left, uri_right = tuple(uri.rsplit('/', 1)) e_left = regex.sub('[a-zA-Z0-9_.-]+', e_left) if regex.match(e_right): if my_id is None: rst.traverseLogger.warn('No Id provided by payload') e_right = str(my_id) e_compare_to = '/'.join([e_left, e_right]) - success = re.match(e_compare_to, uri) is not None + success = re.fullmatch(e_compare_to, uri) is not None if success: break else: @@ -540,7 +575,7 @@ def getTypeDetails(schemaObj, SchemaAlias): if uriElement is not None: try: all_strings = uriElement.find('Collection').find_all('String') - expectedURI = [e.contents[0].rstrip('/') for e in all_strings] + expectedURI = [e.contents[0] for e in all_strings] except Exception as e: rst.traverseLogger.debug('Exception caught while checking URI', exc_info=1) rst.traverseLogger.warn('Could not gather info from Redfish.Uris annotation') @@ -575,7 +610,6 @@ def getTypeDetails(schemaObj, SchemaAlias): def getTypeObject(typename, schemaObj): idtag = (typename, schemaObj.origin) if idtag in PropType.robjcache: - # print('getTypeObject: cache hit', idtag) return PropType.robjcache[idtag] typename = typename.strip('#') @@ -662,7 +696,7 @@ def getPropertyDetails(schemaObj, propertyOwner, propertyName, val, topVersion=N # Get Entity of Owner, then the property of the Property we're targeting ownerEntity = ownerSchema.find( - ['EntityType', 'ComplexType'], attrs={'Name': OwnerType}, recursive=False) # BS4 line + ['EntityType', 'ComplexType'], attrs={'Name': OwnerType}, recursive=False) # check if this property is a nav property # Checks if this prop is an annotation @@ -671,20 +705,25 @@ def getPropertyDetails(schemaObj, propertyOwner, propertyName, val, topVersion=N if '@' not in propertyName: propEntry['isTerm'] = False # not an @ annotation propertyTag = ownerEntity.find( - ['NavigationProperty', 'Property'], attrs={'Name': propertyName}, recursive=False) # BS4 line + ['NavigationProperty', 'Property'], attrs={'Name': propertyName}, recursive=False) # start adding attrs and props together - propertyInnerTags = propertyTag.find_all() # BS4 line + propertyInnerTags = propertyTag.find_all(recursive=False) for tag in propertyInnerTags: - propEntry[tag['Term']] = tag.attrs + if(not tag.get('Term')): + rst.traverseLogger.warn(tag, 'does not contain a Term name') + elif (tag.get('Term') == 'Redfish.Revisions'): + propEntry[tag['Term']] = tag.find_all('Record') + else: + propEntry[tag['Term']] = tag.attrs propertyFullType = propertyTag.get('Type') else: propEntry['isTerm'] = True ownerEntity = ownerSchema.find( - ['Term'], attrs={'Name': OwnerType}, recursive=False) # BS4 line + ['Term'], attrs={'Name': OwnerType}, recursive=False) if ownerEntity is None: ownerEntity = ownerSchema.find( - ['EntityType', 'ComplexType'], attrs={'Name': OwnerType}, recursive=False) # BS4 line + ['EntityType', 'ComplexType'], attrs={'Name': OwnerType}, recursive=False) propertyTag = ownerEntity propertyFullType = propertyTag.get('Type', propertyOwner) @@ -699,7 +738,6 @@ def getPropertyDetails(schemaObj, propertyOwner, propertyName, val, topVersion=N propEntry['realtype'] = 'none' propEntry['attrs'] = dict() propEntry['attrs']['Type'] = customType - metadata = rst.currentService.metadata serviceRefs = rst.currentService.metadata.get_service_refs() serviceSchemaSoup = rst.currentService.metadata.get_soup() success, propertySoup, propertyRefs, propertyFullType = True, serviceSchemaSoup, serviceRefs, customType @@ -720,7 +758,7 @@ def getPropertyDetails(schemaObj, propertyOwner, propertyName, val, topVersion=N propEntry['isCollection'] = propertyFullType continue else: - if val is not None and isinstance(val, list) and propEntry.get('isCollection') is None : + if val is not None and isinstance(val, list) and propEntry.get('isCollection') is None: raise TypeError('This item should not be a List') # If basic, just pass itself @@ -733,6 +771,7 @@ def getPropertyDetails(schemaObj, propertyOwner, propertyName, val, topVersion=N schemaObj = schemaObj.getSchemaFromReference(PropertyNamespace) success = schemaObj is not None if success: + uri = schemaObj.origin propertySoup = schemaObj.soup propertyRefs = schemaObj.refs else: @@ -760,14 +799,14 @@ def getPropertyDetails(schemaObj, propertyOwner, propertyName, val, topVersion=N # this is a unique deprecated enum, labeled as Edm.String propertyFullType = propertyTypeTag.get('UnderlyingType') - isEnum = propertyTypeTag.find( # BS4 line + isEnum = propertyTypeTag.find( 'Annotation', attrs={'Term': 'Redfish.Enumeration'}, recursive=False) if propertyFullType == 'Edm.String' and isEnum is not None: propEntry['realtype'] = 'deprecatedEnum' propEntry['typeprops'] = list() - memberList = isEnum.find( # BS4 line - 'Collection').find_all('PropertyValue') # BS4 line + memberList = isEnum.find( + 'Collection').find_all('PropertyValue') for member in memberList: propEntry['typeprops'].append(member.get('String')) @@ -800,7 +839,7 @@ def getPropertyDetails(schemaObj, propertyOwner, propertyName, val, topVersion=N break elif topVersion is not None and (topVersion > OwnerNamespace): currentVersion = topVersion - currentSchema = baseSoup.find( # BS4 line + currentSchema = baseSoup.find( 'Schema', attrs={'Namespace': currentVersion}) # Working backwards from topVersion schematag, # created expectedType, check if currentTypeTag exists @@ -808,18 +847,18 @@ def getPropertyDetails(schemaObj, propertyOwner, propertyName, val, topVersion=N # until we exhaust all schematags in file while currentSchema is not None: expectedType = currentVersion + '.' + PropertyType - currentTypeTag = currentSchema.find( # BS4 line + currentTypeTag = currentSchema.find( 'ComplexType', attrs={'Name': PropertyType}) if currentTypeTag is not None: baseType = expectedType - rst.traverseLogger.debug('new type: ' + baseType) # Printout FORMAT + rst.traverseLogger.debug('new type: ' + baseType) break else: - nextEntity = currentSchema.find( # BS4 line + nextEntity = currentSchema.find( ['EntityType', 'ComplexType'], attrs={'Name': OwnerType}) nextType = nextEntity.get('BaseType') currentVersion = getNamespace(nextType) - currentSchema = baseSoup.find( # BS4 line + currentSchema = baseSoup.find( 'Schema', attrs={'Namespace': currentVersion}) continue propEntry['realtype'] = 'complex' @@ -830,14 +869,14 @@ def getPropertyDetails(schemaObj, propertyOwner, propertyName, val, topVersion=N propEntry['typeprops'] = [rst.createResourceObject(propertyName, 'complex', item, context=schemaObj.context, typename=baseType, isComplex=True) for item in val] break - elif nameOfTag == 'EnumType': # If enum, get all members + elif nameOfTag == 'EnumType': # If enum, get all members propEntry['realtype'] = 'enum' propEntry['typeprops'] = list() - for MemberName in propertyTypeTag.find_all('Member'): # BS4 line + for MemberName in propertyTypeTag.find_all('Member'): propEntry['typeprops'].append(MemberName['Name']) break - elif nameOfTag == 'EntityType': # If entity, do nothing special (it's a reference link) + elif nameOfTag == 'EntityType': # If entity, do nothing special (it's a reference link) propEntry['realtype'] = 'entity' if val is not None: if propEntry.get('isCollection') is None: @@ -862,5 +901,3 @@ def getPropertyDetails(schemaObj, propertyOwner, propertyName, val, topVersion=N break return propEntry - - diff --git a/tohtml.py b/tohtml.py index 983cdae..049c15f 100644 --- a/tohtml.py +++ b/tohtml.py @@ -1,7 +1,7 @@ # Copyright Notice: # Copyright 2016 DMTF. All rights reserved. -# License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/Redfish-Service-Validator/blob/master/LICENSE.md +# License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/Redfish-Interop-Validator/blob/master/LICENSE.md import traverseService as rst from commonRedfish import * diff --git a/traverseService.py b/traverseService.py index 7e8c69a..d0c647d 100644 --- a/traverseService.py +++ b/traverseService.py @@ -1,6 +1,6 @@ # Copyright Notice: # Copyright 2016-2018 DMTF. All rights reserved. -# License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/Redfish-Service-Validator/blob/master/LICENSE.md +# License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/Redfish-Interop-Validator/blob/master/LICENSE.md import requests import sys @@ -18,7 +18,7 @@ from urllib.parse import urlparse, urlunparse import metadata as md -from commonRedfish import createContext, getNamespace, getNamespaceUnversioned, getType, getVersion, navigateJsonFragment +from commonRedfish import createContext, getNamespace, getNamespaceUnversioned, getType, navigateJsonFragment import rfSchema traverseLogger = logging.getLogger(__name__) @@ -86,6 +86,7 @@ def getLogger(): 'preferonline': False, 'linklimit': {'LogEntry': 20}, 'sample': 0, + 'usessl': True, 'timeout': 30, 'schema_pack': None, 'forceauth': False, @@ -106,6 +107,7 @@ def getLogger(): config = dict(defaultconfig) + def startService(config, defaulted=[]): """startService @@ -297,6 +299,8 @@ def __init__(self, config, default_entries=[]): # with Version, get default and compare to user defined values default_config_target = defaultconfig_by_version.get(target_version, dict()) override_with = {k: default_config_target[k] for k in default_config_target if k in default_entries} + if len(override_with) > 0: + traverseLogger.info('CONFIG: RedfishVersion {} has augmented these tool defaults {}'.format(target_version, override_with)) self.config.update(override_with) self.active = True @@ -401,6 +405,7 @@ def callResourceURI(self, URILink): # rs-assertion: must have application/json or application/xml traverseLogger.debug('callingResourceURI {}with authtype {} and ssl {}: {} {}'.format( 'out of service ' if not inService else '', AuthType, UseSSL, URILink, headers)) + response = None try: if payload is not None and CacheMode == 'Prefer': return True, payload, -1, 0 @@ -459,22 +464,24 @@ def callResourceURI(self, URILink): .format(URILink, statusCode, responses[statusCode], cred_type, AuthType)) except requests.exceptions.SSLError as e: - traverseLogger.error("SSLError on {}".format(URILink)) + traverseLogger.error("SSLError on {}: {}".format(URILink, repr(e))) traverseLogger.debug("output: ", exc_info=True) except requests.exceptions.ConnectionError as e: - traverseLogger.error("ConnectionError on {}".format(URILink)) + traverseLogger.error("ConnectionError on {}: {}".format(URILink, repr(e))) traverseLogger.debug("output: ", exc_info=True) except requests.exceptions.Timeout as e: traverseLogger.error("Request has timed out ({}s) on resource {}".format(timeout, URILink)) traverseLogger.debug("output: ", exc_info=True) except requests.exceptions.RequestException as e: - traverseLogger.error("Request has encounted a problem when getting resource {}".format(URILink)) - traverseLogger.warning("output: ", exc_info=True) + traverseLogger.error("Request has encounted a problem when getting resource {}: {}".format(URILink, repr(e))) + traverseLogger.debug("output: ", exc_info=True) except AuthenticationError as e: raise e # re-raise exception - except Exception: - traverseLogger.error("A problem when getting resource has occurred {}".format(URILink)) - traverseLogger.warning("output: ", exc_info=True) + except Exception as e: + traverseLogger.error("A problem when getting resource {} has occurred: {}".format(URILink, repr(e))) + traverseLogger.debug("output: ", exc_info=True) + if response and response.text: + traverseLogger.debug("payload: {}".format(response.text)) if payload is not None and CacheMode == 'Fallback': return True, payload, -1, 0 @@ -507,13 +514,14 @@ def createResourceObject(name, uri, jsondata=None, typename=None, context=None, '{}: URI could not be acquired: {}'.format(uri, status)) return None else: - jsondata, rtime = jsondata, 0 + success, jsondata, status, rtime = True, jsondata, -1, 0 if not isinstance(jsondata, dict): if not isComplex: traverseLogger.error("Resource no longer a dictionary...") else: traverseLogger.debug("ComplexType does not have val") + return success, None, status return None acquiredtype = jsondata.get('@odata.type', typename) @@ -535,7 +543,7 @@ def createResourceObject(name, uri, jsondata=None, typename=None, context=None, # Get Schema object schemaObj = rfSchema.getSchemaObject(acquiredtype, context) if schemaObj is None: - traverseLogger.error("ResourceObject creation: No schema XML for {} {} {}".format(typename, acquiredtype, context)) + traverseLogger.error("ResourceObject creation: No schema XML for {} {}".format(acquiredtype, context)) return None forceType = False @@ -577,7 +585,9 @@ def createResourceObject(name, uri, jsondata=None, typename=None, context=None, else: traverseLogger.debug('Acquired resource thru AutoExpanded means {}'.format(uri_item)) traverseLogger.info('Regetting resource from URI {}'.format(uri_item)) - return createResourceObject(name, uri_item, None, typename, context, parent, isComplex) + new_payload = createResourceObject(name, uri_item, None, typename, context, parent, isComplex) + if new_payload is None: + traverseLogger.warn('Could not acquire resource, reverting to original payload...') else: if original_jsondata is None: traverseLogger.warn('Acquired Resource.Resource type with fragment, could cause issues {}'.format(uri_item)) @@ -588,7 +598,6 @@ def createResourceObject(name, uri, jsondata=None, typename=None, context=None, else: traverseLogger.warn('@odata.id should not have a fragment'.format(odata_id)) - elif 'Resource.ReferenceableMember' in allTypes: if fragment is not '': pass @@ -599,9 +608,9 @@ def createResourceObject(name, uri, jsondata=None, typename=None, context=None, else: traverseLogger.warn('@odata.id should have a fragment'.format(odata_id)) - newResource = ResourceObj(name, uri, jsondata, typename, original_context, parent, isComplex, forceType=forceType) newResource.rtime = rtime + newResource.status = status return newResource @@ -612,6 +621,7 @@ def __init__(self, name: str, uri: str, jsondata: dict, typename: str, context: self.parent = parent self.uri, self.name = uri, name self.rtime = 0 + self.status = -1 self.isRegistry = False self.errorIndex = { } @@ -719,7 +729,7 @@ def __init__(self, name: str, uri: str, jsondata: dict, typename: str, context: prop_type = propTypeObj.propPattern.get('Type', 'Resource.OemObject') regex = re.compile(prop_pattern) - for key in [k for k in self.jsondata if k not in propertyList and regex.match(k)]: + for key in [k for k in self.jsondata if k not in propertyList and regex.fullmatch(k)]: val = self.jsondata.get(key) value_obj = rfSchema.PropItem(propTypeObj.schemaObj, propTypeObj.fulltype, key, val, customType=prop_type) self.additionalList.append(value_obj) @@ -731,11 +741,15 @@ def __init__(self, name: str, uri: str, jsondata: dict, typename: str, context: if self.errorIndex['bad_uri_schema_uri']: traverseLogger.error('{}: URI not in Redfish.Uris: {}'.format(uri, self.typename)) + if my_id != uri.rsplit('/', 1)[-1]: + traverseLogger.error('Id {} in payload doesn\'t seem to match URI'.format(my_id)) else: traverseLogger.debug('{} in Redfish.Uris: {}'.format(uri, self.typename)) if self.errorIndex['bad_uri_schema_odata']: traverseLogger.error('{}: odata_id not in Redfish.Uris: {}'.format(odata_id, self.typename)) + if my_id != uri.rsplit('/', 1)[-1]: + traverseLogger.error('Id {} in payload doesn\'t seem to match URI'.format(my_id)) else: traverseLogger.debug('{} in Redfish.Uris: {}'.format(odata_id, self.typename))