From f89cb6a102174900984553ab169a7c17b4a1d8c4 Mon Sep 17 00:00:00 2001 From: davidt99 Date: Wed, 8 Jun 2022 14:23:31 +0300 Subject: [PATCH] Fix/propegate url analysis additional parameters (#50) * fix: propagate additional parameters to URL analysis * fea: add from_analysis_id to sub_analysis * add analysis summary meadata function * fix: remove gene count software types on malicious summary report Co-authored-by: Matan Yechiel --- CHANGES | 7 +++ intezer_sdk/__init__.py | 2 +- intezer_sdk/analysis.py | 4 +- intezer_sdk/api.py | 10 ++-- intezer_sdk/base_analysis.py | 6 +-- intezer_sdk/errors.py | 59 ++++++++++++++++++----- intezer_sdk/index.py | 4 +- intezer_sdk/operation.py | 2 +- intezer_sdk/sub_analysis.py | 55 ++++++++++++++++++++-- intezer_sdk/util.py | 48 ++++++++++++++++++- tests/unit/test_analysis.py | 91 ++++++++++++++++++++++++++++++++---- tests/unit/test_index.py | 2 +- 12 files changed, 251 insertions(+), 39 deletions(-) diff --git a/CHANGES b/CHANGES index 84f2b47..0a00ad9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,10 @@ +1.9.0 +------- +- Rename exception to have Error suffix +- Add `SubAnalysis.from_analysis_id` to properly initialize SubAnalysis without the composed analysis +- Fix URL analysis additional parameters propagation +- Add File analysis summary metadata function + 1.8.3 ------- - add extraction info to sub analysis diff --git a/intezer_sdk/__init__.py b/intezer_sdk/__init__.py index cfe6447..e5102d3 100644 --- a/intezer_sdk/__init__.py +++ b/intezer_sdk/__init__.py @@ -1 +1 @@ -__version__ = '1.8.3' +__version__ = '1.9.0' diff --git a/intezer_sdk/analysis.py b/intezer_sdk/analysis.py index f825a53..a66e68c 100644 --- a/intezer_sdk/analysis.py +++ b/intezer_sdk/analysis.py @@ -226,7 +226,7 @@ def _query_status_from_api(self) -> Response: return self._api.get_url_analysis_response(self.analysis_id, False) def _send_analyze_to_api(self, **additional_parameters) -> str: - return self._api.analyze_url(self.url) + return self._api.analyze_url(self.url, **additional_parameters) @property def downloaded_file_analysis(self) -> Optional[FileAnalysis]: @@ -251,6 +251,6 @@ def get_url_analysis_by_id(analysis_id: str, api: IntezerApi = None) -> Optional def _assert_analysis_status(response: dict): if response['status'] in (consts.AnalysisStatusCode.IN_PROGRESS.value, consts.AnalysisStatusCode.QUEUED.value): - raise errors.AnalysisIsStillRunning() + raise errors.AnalysisIsStillRunningError() if response['status'] == consts.AnalysisStatusCode.FAILED.value: raise errors.AnalysisFailedError() diff --git a/intezer_sdk/api.py b/intezer_sdk/api.py index 5fb3a1a..eeedbc0 100644 --- a/intezer_sdk/api.py +++ b/intezer_sdk/api.py @@ -391,7 +391,7 @@ def _set_access_token(self, api_key: str): verify=self._verify_ssl) if response.status_code in (HTTPStatus.UNAUTHORIZED, HTTPStatus.BAD_REQUEST): - raise errors.InvalidApiKey(response) + raise errors.InvalidApiKeyError(response) if response.status_code != HTTPStatus.OK: raise_for_status(response) @@ -445,9 +445,9 @@ def _assert_analysis_response_status_code(response: Response): if response.status_code == HTTPStatus.NOT_FOUND: raise errors.HashDoesNotExistError(response) elif response.status_code == HTTPStatus.CONFLICT: - raise errors.AnalysisIsAlreadyRunning(response) + raise errors.AnalysisIsAlreadyRunningError(response) elif response.status_code == HTTPStatus.FORBIDDEN: - raise errors.InsufficientQuota(response) + raise errors.InsufficientQuotaError(response) elif response.status_code == HTTPStatus.BAD_REQUEST: data = response.json() error = data.get('error', '') @@ -472,14 +472,14 @@ def _get_index_id_from_response(response: Response): def assert_on_premise_above_v21_11(self): if self.on_premise_version and self.on_premise_version <= OnPremiseVersion.V21_11: - raise errors.UnsupportedOnPremiseVersion('This endpoint is not available yet on this on premise') + raise errors.UnsupportedOnPremiseVersionError('This endpoint is not available yet on this on premise') def get_global_api() -> IntezerApi: global _global_api if not _global_api: - raise errors.GlobalApiIsNotInitialized() + raise errors.GlobalApiIsNotInitializedError() return _global_api diff --git a/intezer_sdk/base_analysis.py b/intezer_sdk/base_analysis.py index fd603d7..f5daf79 100644 --- a/intezer_sdk/base_analysis.py +++ b/intezer_sdk/base_analysis.py @@ -68,7 +68,7 @@ def send(self, wait_timeout: Optional[datetime.timedelta] = None, **additional_parameters) -> None: if self.analysis_id: - raise errors.AnalysisHasAlreadyBeenSent() + raise errors.AnalysisHasAlreadyBeenSentError() self.analysis_id = self._send_analyze_to_api(**additional_parameters) @@ -101,7 +101,7 @@ def check_status(self) -> consts.AnalysisStatusCode: def result(self) -> dict: if self._is_analysis_running(): - raise errors.AnalysisIsStillRunning() + raise errors.AnalysisIsStillRunningError() if not self._report: raise errors.ReportDoesNotExistError() @@ -117,6 +117,6 @@ def set_report(self, report: dict): def _assert_analysis_finished(self): if self._is_analysis_running(): - raise errors.AnalysisIsStillRunning() + raise errors.AnalysisIsStillRunningError() if self.status != consts.AnalysisStatusCode.FINISH: raise errors.IntezerError('Analysis not finished successfully') diff --git a/intezer_sdk/errors.py b/intezer_sdk/errors.py index 6292d14..6fe87ef 100644 --- a/intezer_sdk/errors.py +++ b/intezer_sdk/errors.py @@ -13,10 +13,13 @@ class IntezerError(Exception): pass -class UnsupportedOnPremiseVersion(IntezerError): +class UnsupportedOnPremiseVersionError(IntezerError): pass +UnsupportedOnPremiseVersion = UnsupportedOnPremiseVersionError + + class ServerError(IntezerError): def __init__(self, message: str, response: requests.Response): self.response = response @@ -26,16 +29,22 @@ def __init__(self, message: str, response: requests.Response): super().__init__(message) -class AnalysisHasAlreadyBeenSent(IntezerError): +class AnalysisHasAlreadyBeenSentError(IntezerError): def __init__(self): - super(AnalysisHasAlreadyBeenSent, self).__init__('Analysis already been sent') + super().__init__('Analysis already been sent') + +AnalysisHasAlreadyBeenSent = AnalysisHasAlreadyBeenSentError -class IndexHasAlreadyBeenSent(IntezerError): + +class IndexHasAlreadyBeenSentError(IntezerError): def __init__(self): super().__init__('Index already been sent') +IndexHasAlreadyBeenSent = IndexHasAlreadyBeenSentError + + class FamilyNotFoundError(IntezerError): def __init__(self, family_id: str): super().__init__('Family not found: {}'.format(family_id)) @@ -51,41 +60,67 @@ def __init__(self): super().__init__('Report was not found') -class AnalysisIsAlreadyRunning(ServerError): +class AnalysisIsAlreadyRunningError(ServerError): def __init__(self, response: requests.Response): super().__init__('Analysis already running', response) -class InsufficientQuota(ServerError): +AnalysisIsAlreadyRunning = AnalysisIsAlreadyRunningError + + +class InsufficientQuotaError(ServerError): def __init__(self, response: requests.Response): super().__init__('Insufficient quota', response) -class GlobalApiIsNotInitialized(IntezerError): +InsufficientQuota = InsufficientQuotaError + + +class GlobalApiIsNotInitializedError(IntezerError): def __init__(self): super().__init__('Global API is not initialized') -class AnalysisIsStillRunning(IntezerError): +GlobalApiIsNotInitialized = GlobalApiIsNotInitializedError + + +class AnalysisIsStillRunningError(IntezerError): def __init__(self): super().__init__('Analysis is still running') +AnalysisIsStillRunning = AnalysisIsStillRunningError + + class AnalysisFailedError(IntezerError): def __init__(self): super().__init__('Analysis failed') -class InvalidApiKey(ServerError): +class InvalidApiKeyError(ServerError): def __init__(self, response: requests.Response): super().__init__('Invalid api key', response) -class IndexFailed(ServerError): +InvalidApiKey = InvalidApiKeyError + + +class IndexFailedError(ServerError): def __init__(self, response: requests.Response): super().__init__('Index operation failed', response) -class SubAnalysisOperationStillRunning(IntezerError): +IndexFailed = IndexFailedError + + +class SubAnalysisOperationStillRunningError(IntezerError): def __init__(self, operation): - super(SubAnalysisOperationStillRunning, self).__init__('{} is still running'.format(operation)) + super().__init__('{} is still running'.format(operation)) + + +SubAnalysisOperationStillRunning = SubAnalysisOperationStillRunningError + + +class SubAnalysisNotFoundError(IntezerError): + def __init__(self, analysis_id: str): + super().__init__('analysis {} is not found'.format(analysis_id)) diff --git a/intezer_sdk/index.py b/intezer_sdk/index.py index 512fe5c..7d0528c 100644 --- a/intezer_sdk/index.py +++ b/intezer_sdk/index.py @@ -31,7 +31,7 @@ def __init__(self, def send(self, wait: typing.Union[bool, int] = False): if self.index_id: - raise errors.IndexHasAlreadyBeenSent() + raise errors.IndexHasAlreadyBeenSentError() if self._sha256: self.index_id = self._api.index_by_sha256(self._sha256, self._index_as, self._family_name) @@ -70,7 +70,7 @@ def check_status(self): response = self._api.get_index_response(self.index_id) if response.status_code == HTTPStatus.OK: if response.json()['status'] == 'failed': - raise errors.IndexFailed(response) + raise errors.IndexFailedError(response) else: self.status = consts.IndexStatusCode.FINISH elif response.status_code == HTTPStatus.ACCEPTED: diff --git a/intezer_sdk/operation.py b/intezer_sdk/operation.py index d4298b3..7c8db92 100644 --- a/intezer_sdk/operation.py +++ b/intezer_sdk/operation.py @@ -28,7 +28,7 @@ def get_result(self): self.result = operation_result.json()['result'] self.status = AnalysisStatusCode.FINISH else: - raise errors.SubAnalysisOperationStillRunning('operation') + raise errors.SubAnalysisOperationStillRunningError('operation') return self.result def wait_for_completion(self, diff --git a/intezer_sdk/sub_analysis.py b/intezer_sdk/sub_analysis.py index 021dda1..de6904e 100644 --- a/intezer_sdk/sub_analysis.py +++ b/intezer_sdk/sub_analysis.py @@ -2,6 +2,7 @@ from typing import Optional from typing import Union +from intezer_sdk import errors from intezer_sdk.api import IntezerApi from intezer_sdk.api import get_global_api from intezer_sdk.consts import AnalysisStatusCode @@ -18,14 +19,50 @@ def __init__(self, api: IntezerApi = None): self.composed_analysis_id = composed_analysis_id self.analysis_id = analysis_id - self.sha256 = sha256 - self.source = source - self.extraction_info = extraction_info + self._sha256 = sha256 + self._source = source + self._extraction_info = extraction_info self._api = api or get_global_api() self._code_reuse = None self._metadata = None self._operations = {} + @classmethod + def from_analysis_id(cls, + analysis_id: str, + composed_analysis_id: str, + lazy_load=True, + api: IntezerApi = None) -> Optional['SubAnalysis']: + sub_analysis = cls(analysis_id, composed_analysis_id, '', '', None, api) + if not lazy_load: + try: + sub_analysis._init_sub_analysis_from_parent() + except errors.SubAnalysisNotFoundError: + return None + return sub_analysis + + @property + def sha256(self) -> str: + if not self._sha256: + self._init_sub_analysis_from_parent() + + return self._sha256 + + @property + def source(self) -> str: + if not self._source: + self._init_sub_analysis_from_parent() + + return self._source + + @property + def extraction_info(self) -> Optional[dict]: + # Since extraction_info could be none, we check if the sha256 was provided, signaling we already fetch it + if not self._sha256: + self._init_sub_analysis_from_parent() + + return self._extraction_info + @property def code_reuse(self): if self._code_reuse is None: @@ -38,6 +75,18 @@ def metadata(self): self._metadata = self._api.get_sub_analysis_metadata_by_id(self.composed_analysis_id, self.analysis_id) return self._metadata + def _init_sub_analysis_from_parent(self): + sub_analyses = self._api.get_sub_analyses_by_id(self.composed_analysis_id) + sub_analysis = next(( + sub_analysis for sub_analysis in sub_analyses if sub_analysis['sub_analysis_id'] == self.analysis_id), + None) + if not sub_analysis: + raise errors.SubAnalysisNotFoundError(self.analysis_id) + + self._sha256 = sub_analysis['sha256'] + self._source = sub_analysis['source'] + self._extraction_info = sub_analysis.get('extraction_info') + def find_related_files(self, family_id: str, wait: Union[bool, int] = False, diff --git a/intezer_sdk/util.py b/intezer_sdk/util.py index 4275639..6d23e94 100644 --- a/intezer_sdk/util.py +++ b/intezer_sdk/util.py @@ -1,5 +1,6 @@ import collections import itertools +from typing import Dict from typing import List from typing import Optional from typing import Tuple @@ -25,6 +26,51 @@ def _get_title(short: bool) -> str: '=========================\n\n') +def get_analysis_summary_metadata(analysis: FileAnalysis, use_hash_link=False) -> Dict[str, any]: + result = analysis.result() + verdict = result['verdict'].lower() + sub_verdict = result['sub_verdict'].lower() + analysis_url = f"{ANALYZE_URL}/files/{result['sha256']}?private=true" if use_hash_link else result['analysis_url'] + main_family = None + gene_count = None + iocs = None + dynamic_ttps = None + related_samples_unique_count = None + + software_type_priorities_by_verdict = { + 'malicious': [], + 'trusted': ['application', 'library', 'interpreter', 'installer'], + 'suspicious': ['administration_tool', 'packer'] + } + + software_type_priorities = software_type_priorities_by_verdict.get(verdict) + if software_type_priorities: + main_family, gene_count = get_analysis_family(analysis, software_type_priorities) + + if verdict in ('malicious', 'suspicious'): + iocs = analysis.iocs + dynamic_ttps = analysis.dynamic_ttps + + related_samples = [sub_analysis.get_account_related_samples(wait=True) for sub_analysis in + analysis.get_sub_analyses()] + if related_samples: + related_samples_unique_count = len({analysis['analysis']['sha256'] for analysis in + itertools.chain.from_iterable( + sample.result['related_samples'] for sample in related_samples + if sample is not None)}) + + return { + 'verdict': verdict, + 'sub_verdict': sub_verdict, + 'analysis_url': analysis_url, + 'main_family': main_family, + 'gene_count': gene_count, + 'iocs': iocs, + 'dynamic_ttps': dynamic_ttps, + 'related_samples_unique_count': related_samples_unique_count + } + + def get_analysis_summary(analysis: FileAnalysis, no_emojis: bool = False, short: bool = False, @@ -42,7 +88,7 @@ def get_analysis_summary(analysis: FileAnalysis, emoji = get_emoji(verdict) if verdict == 'malicious': - main_family, gene_count = get_analysis_family(analysis, ['malware', 'malicious_packer']) + main_family, gene_count = get_analysis_family(analysis, []) elif verdict == 'trusted': main_family, gene_count = get_analysis_family(analysis, ['application', 'library', 'interpreter', 'installer']) elif verdict == 'suspicious': diff --git a/tests/unit/test_analysis.py b/tests/unit/test_analysis.py index d6635a7..dacc597 100644 --- a/tests/unit/test_analysis.py +++ b/tests/unit/test_analysis.py @@ -259,7 +259,7 @@ def test_get_dynamic_ttps_raises_when_on_premise_on_21_11(self): get_global_api().on_premise_version = OnPremiseVersion.V21_11 # Act and Assert - with self.assertRaises(errors.UnsupportedOnPremiseVersion): + with self.assertRaises(errors.UnsupportedOnPremiseVersionError): _ = analysis.dynamic_ttps def test_send_analysis_by_file_and_get_dynamic_ttps_handle_no_ttps(self): @@ -517,7 +517,7 @@ def test_send_analysis_while_running_raise_error(self): json={'result_url': 'a/sd/asd'}) analysis = FileAnalysis(file_hash='a' * 64) # Act + Assert - with self.assertRaises(errors.AnalysisHasAlreadyBeenSent): + with self.assertRaises(errors.AnalysisHasAlreadyBeenSentError): analysis.send() analysis.send() @@ -614,6 +614,81 @@ def test_send_analysis_and_sub_analyses_metadata_and_code_reuse(self): self.assertIsNotNone(analysis.get_root_analysis().code_reuse) self.assertIsNotNone(analysis.get_root_analysis().metadata) + def test_sub_analysis_from_id_takes_parameters_from_composed_analysis_lazy_load_is_false(self): + # Arrange + analysis_id = str(uuid.uuid4()) + composed_analysis_id = str(uuid.uuid4()) + sha256 = 'axaxaxax' + source = 'root' + with responses.RequestsMock() as mock: + mock.add('GET', + url=f'{self.full_url}/analyses/{composed_analysis_id}/sub-analyses', + status=HTTPStatus.OK, + json={'sub_analyses': [{'source': source, 'sub_analysis_id': analysis_id, 'sha256': sha256}, + {'source': 'static_extraction', 'sub_analysis_id': 'ac', 'sha256': 'ba'}]}) + + # Act + sub_analysis = SubAnalysis.from_analysis_id(analysis_id, composed_analysis_id, lazy_load=False) + + # Assert + self.assertEqual(sub_analysis.sha256, sha256) + self.assertEqual(sub_analysis.source, source) + self.assertIsNone(sub_analysis.extraction_info) + + def test_sub_analysis_from_id_takes_parameters_from_composed_analysis_lazy_load_is_true(self): + # Arrange + analysis_id = str(uuid.uuid4()) + composed_analysis_id = str(uuid.uuid4()) + sha256 = 'axaxaxax' + source = 'root' + with responses.RequestsMock() as mock: + mock.add('GET', + url=f'{self.full_url}/analyses/{composed_analysis_id}/sub-analyses', + status=HTTPStatus.OK, + json={'sub_analyses': [{'source': source, 'sub_analysis_id': analysis_id, 'sha256': sha256}, + {'source': 'static_extraction', 'sub_analysis_id': 'ac', 'sha256': 'ba'}]}) + + # Act + sub_analysis = SubAnalysis.from_analysis_id(analysis_id, composed_analysis_id) + + # Assert + self.assertEqual(sub_analysis.sha256, sha256) + self.assertEqual(sub_analysis.source, source) + self.assertIsNone(sub_analysis.extraction_info) + + def test_sub_analysis_from_id_return_none_when_analysis_not_found_on_composed(self): + # Arrange + analysis_id = str(uuid.uuid4()) + composed_analysis_id = str(uuid.uuid4()) + with responses.RequestsMock() as mock: + mock.add('GET', + url=f'{self.full_url}/analyses/{composed_analysis_id}/sub-analyses', + status=HTTPStatus.OK, + json={'sub_analyses': []}) + + # Act + sub_analysis = SubAnalysis.from_analysis_id(analysis_id, composed_analysis_id, lazy_load=False) + + # Assert + self.assertIsNone(sub_analysis) + + def test_sub_analysis_raises_when_getting_sha256_and_analysis_not_found_on_compose(self): + # Arrange + analysis_id = str(uuid.uuid4()) + composed_analysis_id = str(uuid.uuid4()) + sha256 = 'axaxaxax' + source = 'root' + with responses.RequestsMock() as mock: + mock.add('GET', + url=f'{self.full_url}/analyses/{composed_analysis_id}/sub-analyses', + status=HTTPStatus.OK, + json={'sub_analyses': []}) + + sub_analysis = SubAnalysis.from_analysis_id(analysis_id, composed_analysis_id) + # Act + with self.assertRaises(errors.SubAnalysisNotFoundError): + _ = sub_analysis.sha256 + def test_sub_analysis_operations(self): # Arrange with responses.RequestsMock() as mock: @@ -685,7 +760,7 @@ def test_capabilities_raises_when_on_premise_21_11(self): get_global_api().on_premise_version = OnPremiseVersion.V21_11 # Act and Assert - with self.assertRaises(errors.UnsupportedOnPremiseVersion): + with self.assertRaises(errors.UnsupportedOnPremiseVersionError): _ = sub_analysis.get_capabilities() def test_send_analysis_that_running_on_server_raise_error(self): @@ -697,7 +772,7 @@ def test_send_analysis_that_running_on_server_raise_error(self): json={'result_url': 'a/sd/asd'}) analysis = FileAnalysis(file_hash='a' * 64) # Act + Assert - with self.assertRaises(errors.AnalysisIsAlreadyRunning): + with self.assertRaises(errors.AnalysisIsAlreadyRunningError): analysis.send() def test_analysis_raise_value_error_when_no_file_option_given(self): @@ -840,7 +915,7 @@ def test_get_analysis_by_id_raises_when_analysis_is_not_finished(self): json={'status': AnalysisStatusCode.IN_PROGRESS.value}) # Act - with self.assertRaises(errors.AnalysisIsStillRunning): + with self.assertRaises(errors.AnalysisIsStillRunningError): FileAnalysis.from_analysis_id(analysis_id) def test_get_analysis_by_id_raises_when_analysis_is_queued(self): @@ -854,7 +929,7 @@ def test_get_analysis_by_id_raises_when_analysis_is_queued(self): json={'status': AnalysisStatusCode.QUEUED.value}) # Act - with self.assertRaises(errors.AnalysisIsStillRunning): + with self.assertRaises(errors.AnalysisIsStillRunningError): FileAnalysis.from_analysis_id(analysis_id) @@ -889,7 +964,7 @@ def test_get_analysis_by_id_raises_when_analysis_is_not_finished(self): json={'status': AnalysisStatusCode.IN_PROGRESS.value}) # Act - with self.assertRaises(errors.AnalysisIsStillRunning): + with self.assertRaises(errors.AnalysisIsStillRunningError): UrlAnalysis.from_analysis_id(analysis_id) def test_get_analysis_by_id_raises_when_analysis_failed(self): @@ -941,7 +1016,7 @@ def test_send_fail_when_on_premise(self): get_global_api().on_premise_version = OnPremiseVersion.V21_11 # Act - with self.assertRaises(errors.UnsupportedOnPremiseVersion): + with self.assertRaises(errors.UnsupportedOnPremiseVersionError): _ = UrlAnalysis(url='httpdddds://intezer.com') def test_send_waits_to_compilation_when_requested(self): diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py index c74edc5..a3cc3e7 100644 --- a/tests/unit/test_index.py +++ b/tests/unit/test_index.py @@ -46,7 +46,7 @@ def test_failed_index_raise_index_failed(self): index = Index(sha256='a', index_as=consts.IndexType.TRUSTED) # Act + Assert - with self.assertRaises(errors.IndexFailed): + with self.assertRaises(errors.IndexFailedError): index.send(wait=True) def test_malicious_index_by_sha256_status_change_to_created(self):