From 12de71fd54dac6f490ed43758c029a93c50f9a1b Mon Sep 17 00:00:00 2001 From: Tomas Date: Fri, 26 Jul 2024 09:43:04 -0500 Subject: [PATCH] Changes to cache hit to use URIs without fragments Signed-off-by: Tomas --- .../RedfishServiceValidator.py | 2 +- redfish_service_validator/traverse.py | 72 +++++++++++-------- redfish_service_validator/validateRedfish.py | 5 +- 3 files changed, 46 insertions(+), 33 deletions(-) diff --git a/redfish_service_validator/RedfishServiceValidator.py b/redfish_service_validator/RedfishServiceValidator.py index a83d326..7c1449a 100755 --- a/redfish_service_validator/RedfishServiceValidator.py +++ b/redfish_service_validator/RedfishServiceValidator.py @@ -241,7 +241,7 @@ def validate(argslist=None, configfile=None): # dump cache info to debug log my_logger.debug('getSchemaDetails() -> {}'.format(getSchemaDetails.cache_info())) - my_logger.debug('callResourceURI() -> {}'.format(currentService.callResourceURI.cache_info())) + my_logger.debug('callResourceURI() -> {}'.format(currentService.cache_order)) if not success: my_logger.error("Validation has failed: {} problems found".format(fails)) diff --git a/redfish_service_validator/traverse.py b/redfish_service_validator/traverse.py index fe40c21..45e6d6d 100644 --- a/redfish_service_validator/traverse.py +++ b/redfish_service_validator/traverse.py @@ -20,6 +20,8 @@ my_logger = logging.getLogger(__name__) traverseLogger = my_logger +CACHE_SIZE = 128 + # dictionary to hold sampling notation strings for URIs class AuthenticationError(Exception): """Exception used for failed basic auth or token auth""" @@ -37,6 +39,8 @@ def __init__(self, config): traverseLogger.info('Setting up service...') self.active, self.config = False, config self.logger = getLogger() + + self.cache, self.cache_order = {}, [] self.config['configuri'] = self.config['ip'] self.config['metadatafilepath'] = self.config['schema_directory'] @@ -109,12 +113,10 @@ def __init__(self, config): self.active = True - def close(self): self.active = False - - @lru_cache(maxsize=128) - def callResourceURI(self, URILink): + + def callResourceURI(self, link_uri): traverseLogger = my_logger """ Makes a call to a given URI or URL @@ -126,21 +128,21 @@ def callResourceURI(self, URILink): # rs-assertion: handle redirects? and target permissions # rs-assertion: require no auth for serviceroot calls # TODO: Written with "success" values, should replace with Exception and catches - if URILink is None: + if link_uri is None: traverseLogger.warning("This URI is empty!") return False, None, None, 0 - + config = self.config # proxies = self.proxies ConfigIP, UseSSL, AuthType, ChkCert, ChkCertBundle, timeout, Token = config['configuri'], config['usessl'], config['authtype'], \ config['certificatecheck'], config['certificatebundle'], config['timeout'], config['token'] - scheme, netloc, path, params, query, fragment = urlparse(URILink) + scheme, netloc, path, params, query, fragment = urlparse(link_uri) inService = scheme == '' and netloc == '' if inService: - URLDest = urlunparse((scheme, netloc, path, '', '', '')) #URILink + my_destination = urlunparse((scheme, netloc, path, '', '', '')) # URILink else: - URLDest = urlunparse((scheme, netloc, path, params, query, fragment)) + my_destination = urlunparse((scheme, netloc, path, params, query, fragment)) payload, statusCode, elapsed, auth, noauthchk = None, '', 0, None, True @@ -151,8 +153,8 @@ def callResourceURI(self, URILink): # determine if we need to Auth... if inService: - noauthchk = URILink in ['/redfish', '/redfish/v1', '/redfish/v1/odata'] or\ - '/redfish/v1/$metadata' in URILink + noauthchk = link_uri in ['/redfish', '/redfish/v1', '/redfish/v1/odata'] or\ + '/redfish/v1/$metadata' in link_uri auth = None if noauthchk else (config.get('username'), config.get('password')) traverseLogger.debug('dont chkauth' if noauthchk else 'chkauth') @@ -169,22 +171,32 @@ def callResourceURI(self, URILink): # rs-assertion: must have application/json or application/xml traverseLogger.debug('callingResourceURI {}with authtype {} and ssl {}: {} {}'.format( - 'out of service ' if not inService else '', AuthType, UseSSL, URILink, headers)) + 'out of service ' if not inService else '', AuthType, UseSSL, link_uri, headers)) response = None try: startTick = datetime.now() - mockup_file_path = os.path.join(config['mockup'], URLDest.replace('/redfish/v1/', '', 1).strip('/'), 'index.json') - if not inService: - req = requests.get(URLDest, proxies=self.ext_proxies, verify=False) - content = req.json if not isXML else req.text - response = rf.rest.v1.StaticRestResponse(Status=req.status_code, Headers={x:req.headers[x] for x in req.headers}, Content=req.text) - elif config['mockup'] != '' and os.path.isfile(mockup_file_path): - content = {} - with open(mockup_file_path) as mockup_file: - content = json.load(mockup_file) - response = rf.rest.v1.StaticRestResponse(Status=200, Headers={'Content-Type': 'application/json', 'X-Redfish-Mockup': 'true'}, Content=content) + if my_destination not in self.cache: + mockup_file_path = os.path.join(config['mockup'], my_destination.replace('/redfish/v1/', '', 1).strip('/'), 'index.json') + if not inService: + req = requests.get(my_destination, proxies=self.ext_proxies, verify=False) + content = req.json if not isXML else req.text + response = rf.rest.v1.StaticRestResponse(Status=req.status_code, Headers={x:req.headers[x] for x in req.headers}, Content=req.text) + elif config['mockup'] != '' and os.path.isfile(mockup_file_path): + content = {} + with open(mockup_file_path) as mockup_file: + content = json.load(mockup_file) + response = rf.rest.v1.StaticRestResponse(Status=200, Headers={'Content-Type': 'application/json', 'X-Redfish-Mockup': 'true'}, Content=content) + else: + response = self.context.get(my_destination, headers=headers) + self.cache[my_destination] = response + self.cache_order.append(my_destination) + if len(self.cache) > CACHE_SIZE: + del self.cache[self.cache_order.pop(0)] else: - response = self.context.get(URLDest, headers=headers) + traverseLogger.debug("CACHE HIT {} {}".format(my_destination, link_uri)) + + response = self.cache[my_destination] + elapsed = datetime.now() - startTick statusCode = response.status @@ -192,28 +204,28 @@ def callResourceURI(self, URILink): if statusCode in [200]: contenttype = response.getheader('content-type') if contenttype is None: - traverseLogger.error("Content-type not found in header: {}".format(URILink)) + traverseLogger.error("Content-type not found in header: {}".format(link_uri)) contenttype = '' if 'application/json' in contenttype: traverseLogger.debug("This is a JSON response") decoded = response.dict # navigate fragment - decoded = navigateJsonFragment(decoded, URILink) + decoded = navigateJsonFragment(decoded, link_uri) if decoded is None: traverseLogger.error( - "The JSON pointer in the fragment of this URI is not constructed properly: {}".format(URILink)) + "The JSON pointer in the fragment of this URI is not constructed properly: {}".format(link_uri)) elif 'application/xml' in contenttype: decoded = response.text elif 'text/xml' in contenttype: # non-service schemas can use "text/xml" Content-Type if inService: traverseLogger.warning( - "Incorrect content type 'text/xml' for file within service {}".format(URILink)) + "Incorrect content type 'text/xml' for file within service {}".format(link_uri)) decoded = response.text else: traverseLogger.error( - "This URI did NOT return XML or Json contenttype, is this not a Redfish resource (is this redirected?): {}".format(URILink)) + "This URI did NOT return XML or Json contenttype, is this not a Redfish resource (is this redirected?): {}".format(link_uri)) decoded = None if isXML: traverseLogger.info('Attempting to interpret as XML') @@ -234,12 +246,12 @@ def callResourceURI(self, URILink): else: cred_type = 'username and password' raise AuthenticationError('Error accessing URI {}. Status code "{} {}". Check {} supplied for "{}" authentication.' - .format(URILink, statusCode, responses[statusCode], cred_type, AuthType)) + .format(link_uri, statusCode, responses[statusCode], cred_type, AuthType)) except AuthenticationError as e: raise e # re-raise exception except Exception as e: - traverseLogger.error("A problem when getting resource {} has occurred: {}".format(URILink, repr(e))) + traverseLogger.error("A problem when getting resource {} has occurred: {}".format(link_uri, repr(e))) traverseLogger.debug("output: ", exc_info=True) if response and response.text: traverseLogger.debug("payload: {}".format(response.text)) diff --git a/redfish_service_validator/validateRedfish.py b/redfish_service_validator/validateRedfish.py index 6c074fd..3c2acae 100644 --- a/redfish_service_validator/validateRedfish.py +++ b/redfish_service_validator/validateRedfish.py @@ -121,6 +121,9 @@ def validateEntity(service, prop, val, parentURI=""): my_logger.error("{}: EntityType resource does not contain required @odata.id property, attempting default {}".format(name, uri)) if parentURI == "": return False + else: + # Don't need to verify an excerpt's entity this way + return True # check if the entity is truly what it's supposed to be # if not autoexpand, we must grab the resource @@ -581,8 +584,6 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ if propRealType == 'entity': paramPass = validateEntity(service, prop, val) - - # Render our result my_type = prop.Type.fulltype