From e5041d84de43f04d03cae0d7670feac680da985e Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 22 Jul 2024 06:59:54 +0530 Subject: [PATCH 001/211] Added addition of multiple nuclei templates closes #461 --- web/reNgine/settings.py | 5 ++++ .../templates/scanEngine/settings/tool.html | 2 +- web/scanEngine/views.py | 29 ++++++++++--------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/web/reNgine/settings.py b/web/reNgine/settings.py index 0924a6391..f84b95401 100644 --- a/web/reNgine/settings.py +++ b/web/reNgine/settings.py @@ -306,3 +306,8 @@ } }, } + +''' +File upload settings +''' +DATA_UPLOAD_MAX_NUMBER_FIELDS = None \ No newline at end of file diff --git a/web/scanEngine/templates/scanEngine/settings/tool.html b/web/scanEngine/templates/scanEngine/settings/tool.html index 57dfc2fe4..0c50c530c 100644 --- a/web/scanEngine/templates/scanEngine/settings/tool.html +++ b/web/scanEngine/templates/scanEngine/settings/tool.html @@ -113,7 +113,7 @@
Currently available custom Nuclei templates
{% csrf_token %}
- +
diff --git a/web/scanEngine/views.py b/web/scanEngine/views.py index 21b9b4baf..bed9c8d63 100644 --- a/web/scanEngine/views.py +++ b/web/scanEngine/views.py @@ -197,8 +197,6 @@ def tool_specific_settings(request, slug): context = {} # check for incoming form requests if request.method == "POST": - - print(request.FILES) if 'gfFileUpload' in request.FILES: gf_file = request.FILES['gfFileUpload'] file_extension = gf_file.name.split('.')[len(gf_file.name.split('.'))-1] @@ -214,18 +212,23 @@ def tool_specific_settings(request, slug): messages.add_message(request, messages.INFO, f'Pattern {gf_file.name[:4]} successfully uploaded') return http.HttpResponseRedirect(reverse('tool_settings', kwargs={'slug': slug})) - elif 'nucleiFileUpload' in request.FILES: - nuclei_file = request.FILES['nucleiFileUpload'] - file_extension = nuclei_file.name.split('.')[len(nuclei_file.name.split('.'))-1] - if file_extension != 'yaml': + elif 'nucleiFileUpload[]' in request.FILES: + nuclei_files = request.FILES.getlist('nucleiFileUpload[]') + upload_count = 0 + for nuclei_file in nuclei_files: + original_filename = nuclei_file.name if isinstance(nuclei_file.name, str) else nuclei_file.name.decode('utf-8') + original_filename = re.sub(r'[\\/*?:"<>|]',"", original_filename) + file_extension = original_filename.split('.')[len(nuclei_file.name.split('.'))-1] + if file_extension in ['yaml', 'yml']: + base_filename = os.path.splitext(original_filename)[0] + file_path = '/root/nuclei-templates/' + base_filename + '.yaml' + file = open(file_path, "w") + file.write(nuclei_file.read().decode("utf-8")) + file.close() + upload_count += 1 + if upload_count == 0: messages.add_message(request, messages.ERROR, 'Invalid Nuclei Pattern, upload only *.yaml extension') - else: - filename = re.sub(r'[\\/*?:"<>|]',"", nuclei_file.name) - file_path = '/root/nuclei-templates/' + filename - file = open(file_path, "w") - file.write(nuclei_file.read().decode("utf-8")) - file.close() - messages.add_message(request, messages.INFO, f'Nuclei Pattern {nuclei_file.name[:-5]} successfully uploaded') + messages.add_message(request, messages.INFO, f'{upload_count} Nuclei Patterns successfully uploaded') return http.HttpResponseRedirect(reverse('tool_settings', kwargs={'slug': slug})) elif 'nuclei_config_text_area' in request.POST: From 9ed067a13de5fda0616af5658d89e510fc68784d Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 22 Jul 2024 07:08:42 +0530 Subject: [PATCH 002/211] Add feat multiple gf pattern upload --- .../templates/scanEngine/settings/tool.html | 2 +- web/scanEngine/views.py | 27 ++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/web/scanEngine/templates/scanEngine/settings/tool.html b/web/scanEngine/templates/scanEngine/settings/tool.html index 57dfc2fe4..6d66df65d 100644 --- a/web/scanEngine/templates/scanEngine/settings/tool.html +++ b/web/scanEngine/templates/scanEngine/settings/tool.html @@ -39,7 +39,7 @@

Currently available GF patterns

{% csrf_token %}
- +
diff --git a/web/scanEngine/views.py b/web/scanEngine/views.py index 21b9b4baf..1023d9b33 100644 --- a/web/scanEngine/views.py +++ b/web/scanEngine/views.py @@ -199,19 +199,22 @@ def tool_specific_settings(request, slug): if request.method == "POST": print(request.FILES) - if 'gfFileUpload' in request.FILES: - gf_file = request.FILES['gfFileUpload'] - file_extension = gf_file.name.split('.')[len(gf_file.name.split('.'))-1] - if file_extension != 'json': - messages.add_message(request, messages.ERROR, 'Invalid GF Pattern, upload only *.json extension') - else: + if 'gfFileUpload[]' in request.FILES: + gf_files = request.FILES.getlist('gfFileUpload[]') + upload_count = 0 + for gf_file in gf_files: + original_filename = gf_file.name if isinstance(gf_file.name, str) else gf_file.name.decode('utf-8') # remove special chars from filename, that could possibly do directory traversal or XSS - filename = re.sub(r'[\\/*?:"<>|]',"", gf_file.name) - file_path = '/root/.gf/' + filename - file = open(file_path, "w") - file.write(gf_file.read().decode("utf-8")) - file.close() - messages.add_message(request, messages.INFO, f'Pattern {gf_file.name[:4]} successfully uploaded') + original_filename = re.sub(r'[\\/*?:"<>|]',"", original_filename) + file_extension = original_filename.split('.')[len(gf_file.name.split('.'))-1] + if file_extension == 'json': + base_filename = os.path.splitext(original_filename)[0] + file_path = '/root/.gf/' + base_filename + '.json' + file = open(file_path, "w") + file.write(gf_file.read().decode("utf-8")) + file.close() + upload_count += 1 + messages.add_message(request, messages.INFO, f'{upload_count} GF files successfully uploaded') return http.HttpResponseRedirect(reverse('tool_settings', kwargs={'slug': slug})) elif 'nucleiFileUpload' in request.FILES: From bbb0ed609c6190a191e6f652708b20401660a70f Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 22 Jul 2024 07:09:54 +0530 Subject: [PATCH 003/211] update text --- web/scanEngine/templates/scanEngine/settings/tool.html | 1 + 1 file changed, 1 insertion(+) diff --git a/web/scanEngine/templates/scanEngine/settings/tool.html b/web/scanEngine/templates/scanEngine/settings/tool.html index 6d66df65d..5f884540c 100644 --- a/web/scanEngine/templates/scanEngine/settings/tool.html +++ b/web/scanEngine/templates/scanEngine/settings/tool.html @@ -39,6 +39,7 @@

Currently available GF patterns

{% csrf_token %}
+ (Multiple files can be uploaded.)
From 15b857a20837a39ee46fe9b75f904b4353d74539 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 22 Jul 2024 07:10:25 +0530 Subject: [PATCH 004/211] update label --- web/scanEngine/templates/scanEngine/settings/tool.html | 1 + 1 file changed, 1 insertion(+) diff --git a/web/scanEngine/templates/scanEngine/settings/tool.html b/web/scanEngine/templates/scanEngine/settings/tool.html index 0c50c530c..9500aa7d4 100644 --- a/web/scanEngine/templates/scanEngine/settings/tool.html +++ b/web/scanEngine/templates/scanEngine/settings/tool.html @@ -113,6 +113,7 @@
Currently available custom Nuclei templates
{% csrf_token %}
+ (Multiple files can be uploaded.)
From 9e80ccab5d70df1a6b75c2ae017cf9218f61f293 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 22 Jul 2024 08:34:01 +0530 Subject: [PATCH 005/211] added option to stop multiple scans --- web/api/views.py | 4 +- .../templates/startScan/history.html | 38 +++++++++++++- web/startScan/urls.py | 4 ++ web/startScan/views.py | 49 +++++++++++++++++-- 4 files changed, 87 insertions(+), 8 deletions(-) diff --git a/web/api/views.py b/web/api/views.py index b914ac4ad..f7ea1f664 100644 --- a/web/api/views.py +++ b/web/api/views.py @@ -789,7 +789,7 @@ def post(self, request): create_scan_activity( subscan.scan_history.id, f'Subscan {subscan_id} aborted', - SUCCESS_TASK) + ABORTED_TASK) response['status'] = True except Exception as e: logging.error(e) @@ -805,7 +805,7 @@ def post(self, request): create_scan_activity( scan.id, "Scan aborted", - SUCCESS_TASK) + ABORTED_TASK) response['status'] = True except Exception as e: logging.error(e) diff --git a/web/startScan/templates/startScan/history.html b/web/startScan/templates/startScan/history.html index 3221c800b..06830940f 100644 --- a/web/startScan/templates/startScan/history.html +++ b/web/startScan/templates/startScan/history.html @@ -58,9 +58,12 @@

Filters

Reset Filters + {% if user|can:'initiate_scans_subscans' %}
- Delete Multiple Scans + Delete Multiple Scans + Stop Multiple Scans
+ {% endif %} @@ -435,17 +438,21 @@ function toggleMultipleTargetButton() { if (checkedCount() > 0) { $("#delete_multiple_button").removeClass("disabled"); + $("#stop_multiple_button").removeClass("disabled"); } else { $("#delete_multiple_button").addClass("disabled"); + $("#stop_multiple_button").addClass("disabled"); } } function mainCheckBoxSelected(checkbox) { if (checkbox.checked) { $("#delete_multiple_button").removeClass("disabled"); + $("#stop_multiple_button").removeClass("disabled"); $(".targets_checkbox").prop('checked', true); } else { $("#delete_multiple_button").addClass("disabled"); + $("#stop_multiple_button").addClass("disabled"); $(".targets_checkbox").prop('checked', false); } } @@ -454,7 +461,7 @@ if (!checkedCount()) { swal({ title: '', - text: "Oops! No targets has been selected!", + text: "Oops! You haven't selected any scan to delete.", type: 'error', padding: '2em' }) @@ -477,6 +484,33 @@ } } + function stopMultipleScan() { + if (!checkedCount()) { + swal({ + title: '', + text: "Oops! You haven't selected any scans to stop.", + type: 'error', + padding: '2em' + }) + } else { + // atleast one target is selected + swal.queue([{ + title: 'Are you sure you want to stop ' + checkedCount() + ' Scans?', + text: "This action is irreversible.\nThis will stop all the selected scans if they are running.", + type: 'warning', + showCancelButton: true, + confirmButtonText: 'Delete', + padding: '2em', + showLoaderOnConfirm: true, + preConfirm: function() { + deleteForm = document.getElementById("scan_history_form"); + deleteForm.action = "../stop/multiple"; + deleteForm.submit(); + } + }]) + } + } + // select option listener for report_type_select var report_type = document.getElementById("report_type_select"); report_type.addEventListener("change", function() { diff --git a/web/startScan/urls.py b/web/startScan/urls.py index 31054f069..7ff6a57d0 100644 --- a/web/startScan/urls.py +++ b/web/startScan/urls.py @@ -106,4 +106,8 @@ '/delete/multiple', views.delete_scans, name='delete_multiple_scans'), + path( + '/stop/multiple', + views.stop_scans, + name='stop_multiple_scans'), ] diff --git a/web/startScan/views.py b/web/startScan/views.py index ac08da282..6d2a16ece 100644 --- a/web/startScan/views.py +++ b/web/startScan/views.py @@ -462,11 +462,13 @@ def delete_scan(request, id): def stop_scan(request, id): if request.method == "POST": scan = get_object_or_404(ScanHistory, id=id) - scan.scan_status = ABORTED_TASK - scan.save() try: for task_id in scan.celery_ids: app.control.revoke(task_id, terminate=True, signal='SIGKILL') + + # after celery task is stopped, update the scan status + scan.scan_status = ABORTED_TASK + scan.save() tasks = ( ScanActivity.objects .filter(scan_of=scan) @@ -474,10 +476,11 @@ def stop_scan(request, id): .order_by('-pk') ) for task in tasks: + app.control.revoke(task.celery_id, terminate=True, signal='SIGKILL') task.status = ABORTED_TASK task.time = timezone.now() task.save() - create_scan_activity(scan.id, "Scan aborted", SUCCESS_TASK) + create_scan_activity(scan.id, "Scan aborted", ABORTED_TASK) response = {'status': True} messages.add_message( request, @@ -496,6 +499,44 @@ def stop_scan(request, id): return scan_history(request) +@has_permission_decorator(PERM_INITATE_SCANS_SUBSCANS, redirect_url=FOUR_OH_FOUR_URL) +def stop_scans(request, slug): + if request.method == "POST": + for key, value in request.POST.items(): + if key == 'scan_history_table_length' or key == 'csrfmiddlewaretoken': + continue + scan = get_object_or_404(ScanHistory, id=value) + try: + for task_id in scan.celery_ids: + app.control.revoke(task_id, terminate=True, signal='SIGKILL') + tasks = ( + ScanActivity.objects + .filter(scan_of=scan) + .filter(status=RUNNING_TASK) + .order_by('-pk') + ) + for task in tasks: + app.control.revoke(task.celery_id, terminate=True, signal='SIGKILL') + task.status = ABORTED_TASK + task.time = timezone.now() + task.save() + create_scan_activity(scan.id, "Scan aborted", ABORTED_TASK) + messages.add_message( + request, + messages.INFO, + 'Multiple scans successfully stopped!' + ) + except Exception as e: + logger.error(e) + messages.add_message( + request, + messages.ERROR, + f'Scans failed to stop ! Error: {str(e)}' + ) + return HttpResponseRedirect(reverse('scan_history', kwargs={'slug': slug})) + + + @has_permission_decorator(PERM_INITATE_SCANS_SUBSCANS, redirect_url=FOUR_OH_FOUR_URL) def schedule_scan(request, host_id, slug): domain = Domain.objects.get(id=host_id) @@ -825,7 +866,7 @@ def delete_scans(request, slug): messages.add_message( request, messages.INFO, - 'All Scans deleted!') + 'Multiple scans successfully deleted!') return HttpResponseRedirect(reverse('scan_history', kwargs={'slug': slug})) From 364e1594bdea791c84b3535df1f6bb3d203d881e Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 22 Jul 2024 09:17:07 +0530 Subject: [PATCH 006/211] update logger --- web/api/views.py | 6 ++++-- web/reNgine/settings.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/web/api/views.py b/web/api/views.py index f7ea1f664..7a9b8885e 100644 --- a/web/api/views.py +++ b/web/api/views.py @@ -792,12 +792,14 @@ def post(self, request): ABORTED_TASK) response['status'] = True except Exception as e: - logging.error(e) + logger.error(e) response = {'status': False, 'message': str(e)} elif scan_id: try: + logger.info(f'Aborting scan History') scan = get_object_or_404(ScanHistory, id=scan_id) task_ids = scan.celery_ids + logger.info(f"Setting scan {scan} status to ABORTED_TASK") scan.scan_status = ABORTED_TASK scan.stop_scan_date = timezone.now() scan.aborted_by = request.user @@ -808,7 +810,7 @@ def post(self, request): ABORTED_TASK) response['status'] = True except Exception as e: - logging.error(e) + logger.error(e) response = {'status': False, 'message': str(e)} logger.warning(f'Revoking tasks {task_ids}') diff --git a/web/reNgine/settings.py b/web/reNgine/settings.py index 0924a6391..c6b77e9ab 100644 --- a/web/reNgine/settings.py +++ b/web/reNgine/settings.py @@ -303,6 +303,11 @@ 'handlers': ['task'], 'level': 'DEBUG' if DEBUG else 'INFO', 'propagate': False + }, + 'api.views': { + 'handlers': ['console'], + 'level': 'DEBUG' if DEBUG else 'INFO', + 'propagate': False } }, } From 7c4239d8a1ae44b46a3c9a5bd1ae22355c9adaf9 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 22 Jul 2024 09:45:24 +0530 Subject: [PATCH 007/211] Modified StopScan ApiView to accept multiple scan ids --- web/api/views.py | 112 +++++++++++------- .../templates/startScan/history.html | 64 ++++++++-- 2 files changed, 124 insertions(+), 52 deletions(-) diff --git a/web/api/views.py b/web/api/views.py index 7a9b8885e..b1b102c21 100644 --- a/web/api/views.py +++ b/web/api/views.py @@ -772,65 +772,89 @@ class StopScan(APIView): def post(self, request): req = self.request data = req.data - scan_id = data.get('scan_id') - subscan_id = data.get('subscan_id') - response = {} - task_ids = [] - scan = None - subscan = None - if subscan_id: - try: - subscan = get_object_or_404(SubScan, id=subscan_id) - scan = subscan.scan_history - task_ids = subscan.celery_ids - subscan.status = ABORTED_TASK - subscan.stop_scan_date = timezone.now() - subscan.save() - create_scan_activity( - subscan.scan_history.id, - f'Subscan {subscan_id} aborted', - ABORTED_TASK) - response['status'] = True - except Exception as e: - logger.error(e) - response = {'status': False, 'message': str(e)} - elif scan_id: + scan_ids = data.get('scan_ids', []) + subscan_ids = data.get('subscan_ids', []) + + scan_ids = [int(id) for id in scan_ids] + subscan_ids = [int(id) for id in subscan_ids] + + + def abort_scan(scan): + response = {} + logger.info(f'Aborting scan History') try: - logger.info(f'Aborting scan History') - scan = get_object_or_404(ScanHistory, id=scan_id) - task_ids = scan.celery_ids logger.info(f"Setting scan {scan} status to ABORTED_TASK") + task_ids = scan.celery_ids scan.scan_status = ABORTED_TASK scan.stop_scan_date = timezone.now() scan.aborted_by = request.user scan.save() + for task_id in task_ids: + app.control.revoke(task_id, terminate=True, signal='SIGKILL') + + tasks = ( + ScanActivity.objects + .filter(scan_of=scan) + .filter(status=RUNNING_TASK) + .order_by('-pk') + ) + for task in tasks: + task.status = ABORTED_TASK + task.time = timezone.now() + task.save() + create_scan_activity( scan.id, "Scan aborted", - ABORTED_TASK) + ABORTED_TASK + ) response['status'] = True except Exception as e: logger.error(e) response = {'status': False, 'message': str(e)} - logger.warning(f'Revoking tasks {task_ids}') - for task_id in task_ids: - app.control.revoke(task_id, terminate=True, signal='SIGKILL') + return response - # Abort running tasks - tasks = ( - ScanActivity.objects - .filter(scan_of=scan) - .filter(status=RUNNING_TASK) - .order_by('-pk') - ) - if tasks.exists(): - for task in tasks: - if subscan_id and task.id not in subscan.celery_ids: - continue - task.status = ABORTED_TASK - task.time = timezone.now() - task.save() + def abort_subscan(subscan): + response = {} + logger.info(f'Aborting subscan') + try: + logger.info(f"Setting scan {subscan} status to ABORTED_TASK") + task_ids = subscan.celery_ids + + for task_id in task_ids: + app.control.revoke(task_id, terminate=True, signal='SIGKILL') + + subscan.status = ABORTED_TASK + subscan.stop_scan_date = timezone.now() + subscan.save() + create_scan_activity( + subscan.scan_history.id, + f'Subscan aborted', + ABORTED_TASK + ) + response['status'] = True + except Exception as e: + logger.error(e) + response = {'status': False, 'message': str(e)} + + return response + + for scan_id in scan_ids: + try: + scan = ScanHistory.objects.get(id=scan_id) + response = abort_scan(scan) + except Exception as e: + logger.error(e) + response = {'status': False, 'message': str(e)} + + for subscan_id in subscan_ids: + try: + subscan = SubScan.objects.get(id=subscan_id) + response = abort_subscan(subscan) + except Exception as e: + logger.error(e) + response = {'status': False, 'message': str(e)} return Response(response) diff --git a/web/startScan/templates/startScan/history.html b/web/startScan/templates/startScan/history.html index 06830940f..ff9bfba1f 100644 --- a/web/startScan/templates/startScan/history.html +++ b/web/startScan/templates/startScan/history.html @@ -494,20 +494,68 @@ }) } else { // atleast one target is selected - swal.queue([{ + Swal.fire({ title: 'Are you sure you want to stop ' + checkedCount() + ' Scans?', text: "This action is irreversible.\nThis will stop all the selected scans if they are running.", - type: 'warning', + icon: 'warning', showCancelButton: true, - confirmButtonText: 'Delete', - padding: '2em', + confirmButtonColor: '#d33', + cancelButtonColor: '#3085d6', + confirmButtonText: 'Stop', + cancelButtonText: 'Cancel', showLoaderOnConfirm: true, preConfirm: function() { - deleteForm = document.getElementById("scan_history_form"); - deleteForm.action = "../stop/multiple"; - deleteForm.submit(); + var selected_scan_ids = []; + $('.targets_checkbox:checked').each(function() { + selected_scan_ids.push($(this).val()); + }); + data = { + 'scan_ids': selected_scan_ids + } + fetch('/api/action/stop/scan/', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-CSRFToken': getCookie('csrftoken') + }, + body: JSON.stringify(data) + }).then(function(response) { + if (response.ok) { + return response.json(); + } + throw new Error('Network response was not ok.'); + }).then(function(data) { + if (data['status']) { + Swal.fire({ + title: 'Success', + text: data['message'], + icon: 'success', + showConfirmButton: false, + timer: 1500 + }); + setTimeout(function() { + location.reload(); + }, 1500); + } else { + Swal.fire({ + title: 'Error', + text: data['message'], + icon: 'error', + showConfirmButton: false, + timer: 1500 + }); + } + }).catch(function(error) { + Swal.fire({ + title: 'Error', + text: 'An error occurred while stopping the scans', + icon: 'error', + showConfirmButton: false, + timer: 1500 + }); + }); } - }]) + }); } } From c2b1c10ae2c7c428880c5e0e9b1500e52166ab71 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 22 Jul 2024 09:49:00 +0530 Subject: [PATCH 008/211] modify abort scan --- web/api/views.py | 1 + web/static/custom/custom.js | 28 ++++++++++++++++------------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/web/api/views.py b/web/api/views.py index b1b102c21..5b0af63ec 100644 --- a/web/api/views.py +++ b/web/api/views.py @@ -778,6 +778,7 @@ def post(self, request): scan_ids = [int(id) for id in scan_ids] subscan_ids = [int(id) for id in subscan_ids] + response = {'status': False} def abort_scan(scan): response = {} diff --git a/web/static/custom/custom.js b/web/static/custom/custom.js index 1755b0508..a0ae9694f 100644 --- a/web/static/custom/custom.js +++ b/web/static/custom/custom.js @@ -830,10 +830,10 @@ function stop_scan(scan_id=null, subscan_id=null, reload_scan_bar=true, reload_l const stopAPI = "/api/action/stop/scan/"; if (scan_id) { - var data = {'scan_id': scan_id} + var data = {'scan_ids': [scan_id]} } else if (subscan_id) { - var data = {'subscan_id': subscan_id} + var data = {'subscan_ids': [subscan_id]} } swal.queue([{ title: 'Are you sure you want to stop this scan?', @@ -857,17 +857,21 @@ function stop_scan(scan_id=null, subscan_id=null, reload_scan_bar=true, reload_l }).then(function(data) { // TODO Look for better way if (data.status) { - Snackbar.show({ - text: 'Scan Successfully Aborted.', - pos: 'top-right', - duration: 1500 + Swal.fire({ + title: 'Success', + text: data['message'], + icon: 'success', + showConfirmButton: false, + timer: 1500 }); - if (reload_scan_bar) { - getScanStatusSidebar(); - } - if (reload_location) { - window.location.reload(); - } + setTimeout(function() { + if (reload_scan_bar) { + getScanStatusSidebar(); + } + if (reload_location) { + window.location.reload(); + } + }, 1500); } else { Snackbar.show({ text: 'Oops! Could not abort the scan. ' + data.message, From 75502308fb80d39dde41c1a5d7ae7a0d7aedea16 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 22 Jul 2024 09:57:28 +0530 Subject: [PATCH 009/211] added feature to stop multiple subscans --- .../templates/startScan/subscan_history.html | 84 ++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/web/startScan/templates/startScan/subscan_history.html b/web/startScan/templates/startScan/subscan_history.html index add9e21bd..4b4f1c82e 100644 --- a/web/startScan/templates/startScan/subscan_history.html +++ b/web/startScan/templates/startScan/subscan_history.html @@ -58,9 +58,10 @@

Filters

Reset Filters - {% if user|can:'modify_scan_results' %} + {% if user|can:'initiate_scans_subscans' %}
Delete Multiple SubScans + Stop Multiple SubScans
{% endif %} @@ -353,17 +354,21 @@

Filters

function toggleMultipleTargetButton() { if (checkedCount() > 0) { $("#delete_multiple_button").removeClass("disabled"); + $("#stop_multiple_button").removeClass("disabled"); } else { $("#delete_multiple_button").addClass("disabled"); + $("#stop_multiple_button").addClass("disabled"); } } function mainCheckBoxSelected(checkbox) { if (checkbox.checked) { $("#delete_multiple_button").removeClass("disabled"); + $("#stop_multiple_button").removeClass("disabled"); $(".targets_checkbox").prop('checked', true); } else { $("#delete_multiple_button").addClass("disabled"); + $("#stop_multiple_button").addClass("disabled"); $(".targets_checkbox").prop('checked', false); } } @@ -423,5 +428,82 @@

Filters

}]) } } + + + function stopMultipleSubScan() { + if (!checkedCount()) { + swal({ + title: '', + text: "Oops! You haven't selected any subscans to stop.", + type: 'error', + padding: '2em' + }) + } else { + // atleast one target is selected + Swal.fire({ + title: 'Are you sure you want to stop ' + checkedCount() + ' SubScans?', + text: "This action is irreversible.\nThis will stop all the selected subscans if they are running.", + icon: 'warning', + showCancelButton: true, + confirmButtonColor: '#d33', + cancelButtonColor: '#3085d6', + confirmButtonText: 'Stop', + cancelButtonText: 'Cancel', + showLoaderOnConfirm: true, + preConfirm: function() { + var selected_subscan_ids = []; + $('.targets_checkbox:checked').each(function() { + selected_subscan_ids.push($(this).val()); + }); + data = { + 'subscan_ids': selected_subscan_ids + } + fetch('/api/action/stop/scan/', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-CSRFToken': getCookie('csrftoken') + }, + body: JSON.stringify(data) + }).then(function(response) { + if (response.ok) { + return response.json(); + } + throw new Error('Network response was not ok.'); + }).then(function(data) { + if (data['status']) { + Swal.fire({ + title: 'Success', + text: data['message'], + icon: 'success', + showConfirmButton: false, + timer: 1500 + }); + setTimeout(function() { + location.reload(); + }, 1500); + } else { + Swal.fire({ + title: 'Error', + text: data['message'], + icon: 'error', + showConfirmButton: false, + timer: 1500 + }); + } + }).catch(function(error) { + Swal.fire({ + title: 'Error', + text: 'An error occurred while stopping the subscans', + icon: 'error', + showConfirmButton: false, + timer: 1500 + }); + }); + } + }); + } + } + {% endblock page_level_script %} From ded8c16785fe55156caf1c5f0eb4d126dbf8926d Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 22 Jul 2024 10:00:32 +0530 Subject: [PATCH 010/211] fix issue on aborting scans that are already success or aborted --- web/api/views.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/web/api/views.py b/web/api/views.py index 5b0af63ec..6e7f8e529 100644 --- a/web/api/views.py +++ b/web/api/views.py @@ -844,6 +844,9 @@ def abort_subscan(subscan): for scan_id in scan_ids: try: scan = ScanHistory.objects.get(id=scan_id) + # if scan is already successful or aborted then do nothing + if scan.scan_status == SUCCESS_TASK or scan.scan_status == ABORTED_TASK: + continue response = abort_scan(scan) except Exception as e: logger.error(e) @@ -852,6 +855,8 @@ def abort_subscan(subscan): for subscan_id in subscan_ids: try: subscan = SubScan.objects.get(id=subscan_id) + if subscan.scan_status == SUCCESS_TASK or subscan.scan_status == ABORTED_TASK: + continue response = abort_subscan(subscan) except Exception as e: logger.error(e) From 3ceeb1125ed7b4ce78d7cd9f7f8f73ab47890553 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 26 Jul 2024 10:35:53 +0530 Subject: [PATCH 011/211] Hide API keys in vault --- .../templates/scanEngine/settings/api.html | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/web/scanEngine/templates/scanEngine/settings/api.html b/web/scanEngine/templates/scanEngine/settings/api.html index 150e912a0..a98a9306b 100644 --- a/web/scanEngine/templates/scanEngine/settings/api.html +++ b/web/scanEngine/templates/scanEngine/settings/api.html @@ -29,21 +29,31 @@

OpenAI keys will be used to generate vulnerability description, remediation, impact and vulnerability report writing using GPT.

+
{% if openai_key %} - + {% else %} {% endif %} +
+ +
+
This is optional but recommended.

Netlas keys will be used to get whois information and other OSINT related data.

+
{% if netlas_key %} - + {% else %} {% endif %} +
+ +
+
This is optional
From 928af18ef6e7bca40c5f0b1429a48a9e46a5989f Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 26 Jul 2024 11:25:20 +0530 Subject: [PATCH 012/211] hide hackerone api key --- web/scanEngine/admin.py | 1 + web/scanEngine/forms.py | 6 ++++-- web/scanEngine/templates/scanEngine/settings/hackerone.html | 5 +++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/web/scanEngine/admin.py b/web/scanEngine/admin.py index b2f2e0c10..edca16621 100644 --- a/web/scanEngine/admin.py +++ b/web/scanEngine/admin.py @@ -9,3 +9,4 @@ admin.site.register(Notification) admin.site.register(VulnerabilityReportSetting) admin.site.register(InstalledExternalTool) +admin.site.register(Hackerone) \ No newline at end of file diff --git a/web/scanEngine/forms.py b/web/scanEngine/forms.py index 4eddf0d92..3f6431b54 100644 --- a/web/scanEngine/forms.py +++ b/web/scanEngine/forms.py @@ -399,12 +399,14 @@ class Meta: api_key = forms.CharField( required=True, - widget=forms.TextInput( + widget=forms.PasswordInput( attrs={ "class": "form-control form-control-lg", "id": "api_key", "placeholder": "Hackerone API Token", - })) + }, + render_value=True + )) send_critical = forms.BooleanField( required=False, diff --git a/web/scanEngine/templates/scanEngine/settings/hackerone.html b/web/scanEngine/templates/scanEngine/settings/hackerone.html index 1cbd18142..bb1288042 100644 --- a/web/scanEngine/templates/scanEngine/settings/hackerone.html +++ b/web/scanEngine/templates/scanEngine/settings/hackerone.html @@ -47,7 +47,12 @@

Hackerone Automatic Vulnerability Report Settings

+
{{form.api_key}} +
+ +
+
From d11adc29db9463754e6b8a984a944fcaf46ed5eb Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 28 Jul 2024 07:03:00 +0530 Subject: [PATCH 013/211] fix slack url hide --- web/scanEngine/forms.py | 32 ++++++++++++------- .../scanEngine/settings/notification.html | 19 +++++++---- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/web/scanEngine/forms.py b/web/scanEngine/forms.py index 3f6431b54..dd8151e9e 100644 --- a/web/scanEngine/forms.py +++ b/web/scanEngine/forms.py @@ -176,12 +176,14 @@ class Meta: slack_hook_url = forms.CharField( required=False, - widget=forms.TextInput( + widget=forms.PasswordInput( attrs={ - "class": "form-control", + "class": "form-control h-100", "id": "slack_hook_url", "placeholder": "https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX", - })) + }, + render_value=True + )) send_to_lark = forms.BooleanField( required=False, @@ -193,12 +195,14 @@ class Meta: lark_hook_url = forms.CharField( required=False, - widget=forms.TextInput( + widget=forms.PasswordInput( attrs={ "class": "form-control", "id": "lark_hook_url", "placeholder": "https://open.larksuite.com/open-apis/bot/v2/hook/XXXXXXXXXXXXXXXXXXXXXXXX", - })) + }, + render_value=True + )) send_to_discord = forms.BooleanField( required=False, @@ -210,12 +214,14 @@ class Meta: discord_hook_url = forms.CharField( required=False, - widget=forms.TextInput( + widget=forms.PasswordInput( attrs={ "class": "form-control", "id": "discord_hook_url", "placeholder": "https://discord.com/api/webhooks/000000000000000000/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", - })) + }, + render_value=True + )) send_to_telegram = forms.BooleanField( required=False, @@ -227,21 +233,25 @@ class Meta: telegram_bot_token = forms.CharField( required=False, - widget=forms.TextInput( + widget=forms.PasswordInput( attrs={ "class": "form-control", "id": "telegram_bot_token", "placeholder": "Bot Token", - })) + }, + render_value=True + )) telegram_bot_chat_id = forms.CharField( required=False, - widget=forms.TextInput( + widget=forms.PasswordInput( attrs={ "class": "form-control", "id": "telegram_bot_chat_id", "placeholder": "Bot Chat ID", - })) + }, + render_value=True + )) send_scan_status_notif = forms.BooleanField( required=False, diff --git a/web/scanEngine/templates/scanEngine/settings/notification.html b/web/scanEngine/templates/scanEngine/settings/notification.html index cffc23511..6f02519e4 100644 --- a/web/scanEngine/templates/scanEngine/settings/notification.html +++ b/web/scanEngine/templates/scanEngine/settings/notification.html @@ -31,19 +31,24 @@

Send Notifications to:

-
+
- +
- {{form.send_to_slack}} + {{ form.send_to_slack }}
- +
+
+
+ {{ form.slack_hook_url }} +
+ +
- {{form.slack_hook_url}}
- +
- +
From d13d694149e5f69ee0e6f02926fbc53ce8891b56 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 28 Jul 2024 07:05:30 +0530 Subject: [PATCH 014/211] added hide icon for lark --- web/scanEngine/forms.py | 4 ++-- .../scanEngine/settings/notification.html | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/web/scanEngine/forms.py b/web/scanEngine/forms.py index dd8151e9e..0f3e0d411 100644 --- a/web/scanEngine/forms.py +++ b/web/scanEngine/forms.py @@ -197,7 +197,7 @@ class Meta: required=False, widget=forms.PasswordInput( attrs={ - "class": "form-control", + "class": "form-control h-100", "id": "lark_hook_url", "placeholder": "https://open.larksuite.com/open-apis/bot/v2/hook/XXXXXXXXXXXXXXXXXXXXXXXX", }, @@ -216,7 +216,7 @@ class Meta: required=False, widget=forms.PasswordInput( attrs={ - "class": "form-control", + "class": "form-control h-100", "id": "discord_hook_url", "placeholder": "https://discord.com/api/webhooks/000000000000000000/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", }, diff --git a/web/scanEngine/templates/scanEngine/settings/notification.html b/web/scanEngine/templates/scanEngine/settings/notification.html index 6f02519e4..6be78ef68 100644 --- a/web/scanEngine/templates/scanEngine/settings/notification.html +++ b/web/scanEngine/templates/scanEngine/settings/notification.html @@ -59,19 +59,24 @@

Send Notifications to:

-
+
- +
- {{form.send_to_lark}} + {{ form.send_to_lark }}
- +
+
+
+ {{ form.lark_hook_url }} +
+ +
- {{form.lark_hook_url}}
- +
- +
From 11aad079f9856a51159b7abe4f1b849f69b687c1 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 28 Jul 2024 07:07:59 +0530 Subject: [PATCH 015/211] hide icon for discord --- .../scanEngine/settings/notification.html | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/web/scanEngine/templates/scanEngine/settings/notification.html b/web/scanEngine/templates/scanEngine/settings/notification.html index 6be78ef68..7410c9936 100644 --- a/web/scanEngine/templates/scanEngine/settings/notification.html +++ b/web/scanEngine/templates/scanEngine/settings/notification.html @@ -87,19 +87,24 @@

Send Notifications to:

-
+
- +
- {{form.send_to_discord}} + {{ form.send_to_discord }}
- +
+
+
+ {{ form.discord_hook_url }} +
+ +
- {{form.discord_hook_url}}
- +
- +
From cbc71167e8b31525199d8e11903a9bdb2a886751 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 28 Jul 2024 07:13:21 +0530 Subject: [PATCH 016/211] fix telegram input box hide icon --- web/scanEngine/forms.py | 4 +-- .../scanEngine/settings/notification.html | 34 +++++++++++++------ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/web/scanEngine/forms.py b/web/scanEngine/forms.py index 0f3e0d411..e55744621 100644 --- a/web/scanEngine/forms.py +++ b/web/scanEngine/forms.py @@ -235,7 +235,7 @@ class Meta: required=False, widget=forms.PasswordInput( attrs={ - "class": "form-control", + "class": "form-control h-100", "id": "telegram_bot_token", "placeholder": "Bot Token", }, @@ -246,7 +246,7 @@ class Meta: required=False, widget=forms.PasswordInput( attrs={ - "class": "form-control", + "class": "form-control h-100", "id": "telegram_bot_chat_id", "placeholder": "Bot Chat ID", }, diff --git a/web/scanEngine/templates/scanEngine/settings/notification.html b/web/scanEngine/templates/scanEngine/settings/notification.html index 7410c9936..d5b9aa4b5 100644 --- a/web/scanEngine/templates/scanEngine/settings/notification.html +++ b/web/scanEngine/templates/scanEngine/settings/notification.html @@ -57,7 +57,7 @@

Send Notifications to:

- +
@@ -85,7 +85,7 @@

Send Notifications to:

- +
@@ -113,24 +113,36 @@

Send Notifications to:

- +
-
+
- +
- {{form.send_to_telegram}} + {{ form.send_to_telegram }} +
+
+
+
+
+ {{ form.telegram_bot_token }} +
+
- +
+
+ {{ form.telegram_bot_chat_id }} +
+ +
+
- {{form.telegram_bot_token}} - {{form.telegram_bot_chat_id}}
- +
- +
From b64692030e2bc0e46df96432e9c0c693b6849909 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 28 Jul 2024 07:20:11 +0530 Subject: [PATCH 017/211] use new eye icon inside input box for api --- web/scanEngine/templates/scanEngine/settings/api.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/scanEngine/templates/scanEngine/settings/api.html b/web/scanEngine/templates/scanEngine/settings/api.html index a98a9306b..9f57f1a6d 100644 --- a/web/scanEngine/templates/scanEngine/settings/api.html +++ b/web/scanEngine/templates/scanEngine/settings/api.html @@ -29,13 +29,13 @@

OpenAI keys will be used to generate vulnerability description, remediation, impact and vulnerability report writing using GPT.

-
+
{% if openai_key %} {% else %} {% endif %} -
+
@@ -44,13 +44,13 @@

Netlas keys will be used to get whois information and other OSINT related data.

-
+
{% if netlas_key %} {% else %} {% endif %} -
+
From 2e04e2ec1c9147afb6ffa63ff51a722d7a104d8f Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 28 Jul 2024 07:23:15 +0530 Subject: [PATCH 018/211] hackerone hide icon fix --- .../templates/scanEngine/settings/hackerone.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/web/scanEngine/templates/scanEngine/settings/hackerone.html b/web/scanEngine/templates/scanEngine/settings/hackerone.html index bb1288042..eec049813 100644 --- a/web/scanEngine/templates/scanEngine/settings/hackerone.html +++ b/web/scanEngine/templates/scanEngine/settings/hackerone.html @@ -43,15 +43,15 @@

Hackerone Automatic Vulnerability Report Settings

- {{form.username}} + {{ form.username }}
-
- {{form.api_key}} -
- -
+
+ {{ form.api_key }} +
+ +
From 340dfedc8b66fbb1771e0032e9bae52d6d5ce1b9 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 31 Jul 2024 19:19:16 +0530 Subject: [PATCH 019/211] introduce github action for auto verion and changelog --- .github/workflows/auto-release.yml | 60 ++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/auto-release.yml diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml new file mode 100644 index 000000000..7c068aa2c --- /dev/null +++ b/.github/workflows/auto-release.yml @@ -0,0 +1,60 @@ +name: Update Version and Changelog and Readme + +on: + release: + types: [published] + +jobs: + update-version-and-changelog: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get latest release info + id: get_release + uses: actions/github-script@v6 + with: + script: | + const release = await github.rest.repos.getLatestRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + core.setOutput('tag_name', release.data.tag_name); + core.setOutput('body', release.data.body); + + - name: Update version file + run: echo ${{ steps.get_release.outputs.tag_name }} > web/.version + + - name: Update CHANGELOG.md + run: | + echo "# Changelog" > CHANGELOG.md.new + echo "" >> CHANGELOG.md.new + echo "## ${{ steps.get_release.outputs.tag_name }}" >> CHANGELOG.md.new + echo "" >> CHANGELOG.md.new + echo "${{ steps.get_release.outputs.body }}" >> CHANGELOG.md.new + echo "" >> CHANGELOG.md.new + if [ -f CHANGELOG.md ]; then + sed '1,2d' CHANGELOG.md >> CHANGELOG.md.new + fi + mv CHANGELOG.md.new CHANGELOG.md + + - name: Update README.md + run: | + sed -i 's|https://img.shields.io/badge/version-.*-informational|https://img.shields.io/badge/version-${{ steps.get_release.outputs.tag_name }}-informational|g' README.md + + - name: Commit and push changes + run: | + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + git add web/.version CHANGELOG.md README.md + if git diff --staged --quiet; then + echo "No changes to commit" + else + git commit -m "reNgine release: ${{ steps.get_release.outputs.tag_name }} :rocket:" + git push origin HEAD:${{ github.event.repository.default_branch }} + fi From e1e8b47f23e3e31077fe3f098ace1b2572e23b2b Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 31 Jul 2024 19:19:46 +0530 Subject: [PATCH 020/211] remove hardcoded version to use context processor --- docker-compose.dev.yml | 3 --- docker-compose.yml | 3 --- web/.version | 1 + web/api/views.py | 5 +---- web/art/reNgine.txt | 2 +- web/dashboard/templates/dashboard/index.html | 2 +- web/reNgine/context_processors.py | 12 ++++++++++-- web/reNgine/settings.py | 18 +++++++++++++++++- web/templates/base/_items/top_bar.html | 8 ++++---- web/templates/base/login.html | 2 +- 10 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 web/.version diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f80e3d91b..3359cfb8f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -94,9 +94,6 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_PORT=${POSTGRES_PORT} - POSTGRES_HOST=${POSTGRES_HOST} - # THIS IS A MUST FOR CHECKING UPDATE, EVERYTIME A COMMIT IS MERGED INTO - # MASTER, UPDATE THIS!!! MAJOR.MINOR.PATCH https://semver.org/ - - RENGINE_CURRENT_VERSION='2.1.2' volumes: - ./web:/usr/src/app - github_repos:/usr/src/github diff --git a/docker-compose.yml b/docker-compose.yml index e46db5430..6f40c5bd6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -96,9 +96,6 @@ services: - POSTGRES_PORT=${POSTGRES_PORT} - POSTGRES_HOST=${POSTGRES_HOST} - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD} - # THIS IS A MUST FOR CHECKING UPDATE, EVERYTIME A COMMIT IS MERGED INTO - # MASTER, UPDATE THIS!!! MAJOR.MINOR.PATCH https://semver.org/ - - RENGINE_CURRENT_VERSION='2.1.2' volumes: - ./web:/usr/src/app - github_repos:/usr/src/github diff --git a/web/.version b/web/.version new file mode 100644 index 000000000..cf2dc0bc4 --- /dev/null +++ b/web/.version @@ -0,0 +1 @@ +v2.2.0 \ No newline at end of file diff --git a/web/api/views.py b/web/api/views.py index e2c7805a0..1a75e37e4 100644 --- a/web/api/views.py +++ b/web/api/views.py @@ -890,10 +890,7 @@ def get(self, request): # get current version_number # remove quotes from current_version - current_version = ((os.environ['RENGINE_CURRENT_VERSION' - ])[1:] if os.environ['RENGINE_CURRENT_VERSION' - ][0] == 'v' - else os.environ['RENGINE_CURRENT_VERSION']).replace("'", "") + current_version = RENGINE_CURRENT_VERSION # for consistency remove v from both if exists latest_version = re.search(r'v(\d+\.)?(\d+\.)?(\*|\d+)', diff --git a/web/art/reNgine.txt b/web/art/reNgine.txt index cf0082bd3..a94a0ea1d 100644 --- a/web/art/reNgine.txt +++ b/web/art/reNgine.txt @@ -3,6 +3,6 @@ _ __ ___| \| | __ _ _ _ __ ___ | '__/ _ \ . ` |/ _` | | '_ \ / _ \ | | | __/ |\ | (_| | | | | | __/ - |_| \___|_| \_|\__, |_|_| |_|\___| v2.1.1 + |_| \___|_| \_|\__, |_|_| |_|\___| __/ | |___/ diff --git a/web/dashboard/templates/dashboard/index.html b/web/dashboard/templates/dashboard/index.html index 99276d2d2..394d1c560 100644 --- a/web/dashboard/templates/dashboard/index.html +++ b/web/dashboard/templates/dashboard/index.html @@ -17,7 +17,7 @@ {% endblock custom_js_css_link %} {% block breadcrumb_title %} -reNgine 2.1.2 +reNgine {{ RENGINE_CURRENT_VERSION }} {% endblock breadcrumb_title %} {% block main_content %} diff --git a/web/reNgine/context_processors.py b/web/reNgine/context_processors.py index 8fefeae09..cbf89af9b 100644 --- a/web/reNgine/context_processors.py +++ b/web/reNgine/context_processors.py @@ -1,6 +1,9 @@ -from dashboard.models import * import requests +from dashboard.models import * +from django.conf import settings + + def projects(request): projects = Project.objects.all() try: @@ -17,4 +20,9 @@ def misc(request): externalIp = requests.get('https://checkip.amazonaws.com').text.strip() return { 'external_ip': externalIp - } \ No newline at end of file + } + +def version_context(request): + return { + 'RENGINE_CURRENT_VERSION': settings.RENGINE_CURRENT_VERSION + } diff --git a/web/reNgine/settings.py b/web/reNgine/settings.py index 0924a6391..510c2516d 100644 --- a/web/reNgine/settings.py +++ b/web/reNgine/settings.py @@ -43,6 +43,21 @@ ALLOWED_HOSTS = ['*'] SECRET_KEY = first_run(SECRET_FILE, BASE_DIR) +# Rengine version +# reads current version from a file called .version +VERSION_FILE = os.path.join(BASE_DIR, '.version') +if os.path.exists(VERSION_FILE): + with open(VERSION_FILE, 'r') as f: + _version = f.read().strip() +else: + _version = 'unknown' + +# removes v from _version if exists +if _version.startswith('v'): + _version = _version[1:] + +RENGINE_CURRENT_VERSION = _version + # Databases DATABASES = { 'default': { @@ -103,7 +118,8 @@ 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', 'reNgine.context_processors.projects', - 'reNgine.context_processors.misc' + 'reNgine.context_processors.misc', + 'reNgine.context_processors.version_context' ], }, }] diff --git a/web/templates/base/_items/top_bar.html b/web/templates/base/_items/top_bar.html index 9819958ae..8b3d3c074 100644 --- a/web/templates/base/_items/top_bar.html +++ b/web/templates/base/_items/top_bar.html @@ -170,18 +170,18 @@
Welcome {{user.get_username}}!
diff --git a/web/templates/base/login.html b/web/templates/base/login.html index 1ce18a7ba..326f7d56a 100644 --- a/web/templates/base/login.html +++ b/web/templates/base/login.html @@ -58,7 +58,7 @@

Login to reNgine

-

Current release: v2.1.2

+

Current release: v{{ RENGINE_CURRENT_VERSION }}

@@ -353,17 +354,21 @@

Filters

function toggleMultipleTargetButton() { if (checkedCount() > 0) { $("#delete_multiple_button").removeClass("disabled"); + $("#stop_multiple_button").removeClass("disabled"); } else { $("#delete_multiple_button").addClass("disabled"); + $("#stop_multiple_button").addClass("disabled"); } } function mainCheckBoxSelected(checkbox) { if (checkbox.checked) { $("#delete_multiple_button").removeClass("disabled"); + $("#stop_multiple_button").removeClass("disabled"); $(".targets_checkbox").prop('checked', true); } else { $("#delete_multiple_button").addClass("disabled"); + $("#stop_multiple_button").addClass("disabled"); $(".targets_checkbox").prop('checked', false); } } @@ -423,5 +428,82 @@

Filters

}]) } } + + + function stopMultipleSubScan() { + if (!checkedCount()) { + swal({ + title: '', + text: "Oops! You haven't selected any subscans to stop.", + type: 'error', + padding: '2em' + }) + } else { + // atleast one target is selected + Swal.fire({ + title: 'Are you sure you want to stop ' + checkedCount() + ' SubScans?', + text: "This action is irreversible.\nThis will stop all the selected subscans if they are running.", + icon: 'warning', + showCancelButton: true, + confirmButtonColor: '#d33', + cancelButtonColor: '#3085d6', + confirmButtonText: 'Stop', + cancelButtonText: 'Cancel', + showLoaderOnConfirm: true, + preConfirm: function() { + var selected_subscan_ids = []; + $('.targets_checkbox:checked').each(function() { + selected_subscan_ids.push($(this).val()); + }); + data = { + 'subscan_ids': selected_subscan_ids + } + fetch('/api/action/stop/scan/', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-CSRFToken': getCookie('csrftoken') + }, + body: JSON.stringify(data) + }).then(function(response) { + if (response.ok) { + return response.json(); + } + throw new Error('Network response was not ok.'); + }).then(function(data) { + if (data['status']) { + Swal.fire({ + title: 'Success', + text: data['message'], + icon: 'success', + showConfirmButton: false, + timer: 1500 + }); + setTimeout(function() { + location.reload(); + }, 1500); + } else { + Swal.fire({ + title: 'Error', + text: data['message'], + icon: 'error', + showConfirmButton: false, + timer: 1500 + }); + } + }).catch(function(error) { + Swal.fire({ + title: 'Error', + text: 'An error occurred while stopping the subscans', + icon: 'error', + showConfirmButton: false, + timer: 1500 + }); + }); + } + }); + } + } + {% endblock page_level_script %} From f9c35ddd55b51235cf4189d52c61ada657c55626 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 22 Jul 2024 10:00:32 +0530 Subject: [PATCH 050/211] fix issue on aborting scans that are already success or aborted --- web/api/views.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/web/api/views.py b/web/api/views.py index 126434536..84bd49ec3 100644 --- a/web/api/views.py +++ b/web/api/views.py @@ -846,6 +846,9 @@ def abort_subscan(subscan): for scan_id in scan_ids: try: scan = ScanHistory.objects.get(id=scan_id) + # if scan is already successful or aborted then do nothing + if scan.scan_status == SUCCESS_TASK or scan.scan_status == ABORTED_TASK: + continue response = abort_scan(scan) except Exception as e: logger.error(e) @@ -854,6 +857,8 @@ def abort_subscan(subscan): for subscan_id in subscan_ids: try: subscan = SubScan.objects.get(id=subscan_id) + if subscan.scan_status == SUCCESS_TASK or subscan.scan_status == ABORTED_TASK: + continue response = abort_subscan(subscan) except Exception as e: logger.error(e) From 80d577cc23c1a2309d4340b0e1628ba5635b7957 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 26 Jul 2024 10:35:53 +0530 Subject: [PATCH 051/211] Hide API keys in vault --- .../templates/scanEngine/settings/api.html | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/web/scanEngine/templates/scanEngine/settings/api.html b/web/scanEngine/templates/scanEngine/settings/api.html index 150e912a0..a98a9306b 100644 --- a/web/scanEngine/templates/scanEngine/settings/api.html +++ b/web/scanEngine/templates/scanEngine/settings/api.html @@ -29,21 +29,31 @@

OpenAI keys will be used to generate vulnerability description, remediation, impact and vulnerability report writing using GPT.

+
{% if openai_key %} - + {% else %} {% endif %} +
+ +
+
This is optional but recommended.

Netlas keys will be used to get whois information and other OSINT related data.

+
{% if netlas_key %} - + {% else %} {% endif %} +
+ +
+
This is optional
From 1fc3fad5afe88680a48e0904c02ad9ee6c0732ba Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 26 Jul 2024 11:25:20 +0530 Subject: [PATCH 052/211] hide hackerone api key --- web/scanEngine/admin.py | 1 + web/scanEngine/forms.py | 6 ++++-- web/scanEngine/templates/scanEngine/settings/hackerone.html | 5 +++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/web/scanEngine/admin.py b/web/scanEngine/admin.py index b2f2e0c10..edca16621 100644 --- a/web/scanEngine/admin.py +++ b/web/scanEngine/admin.py @@ -9,3 +9,4 @@ admin.site.register(Notification) admin.site.register(VulnerabilityReportSetting) admin.site.register(InstalledExternalTool) +admin.site.register(Hackerone) \ No newline at end of file diff --git a/web/scanEngine/forms.py b/web/scanEngine/forms.py index 4eddf0d92..3f6431b54 100644 --- a/web/scanEngine/forms.py +++ b/web/scanEngine/forms.py @@ -399,12 +399,14 @@ class Meta: api_key = forms.CharField( required=True, - widget=forms.TextInput( + widget=forms.PasswordInput( attrs={ "class": "form-control form-control-lg", "id": "api_key", "placeholder": "Hackerone API Token", - })) + }, + render_value=True + )) send_critical = forms.BooleanField( required=False, diff --git a/web/scanEngine/templates/scanEngine/settings/hackerone.html b/web/scanEngine/templates/scanEngine/settings/hackerone.html index 1cbd18142..bb1288042 100644 --- a/web/scanEngine/templates/scanEngine/settings/hackerone.html +++ b/web/scanEngine/templates/scanEngine/settings/hackerone.html @@ -47,7 +47,12 @@

Hackerone Automatic Vulnerability Report Settings

+
{{form.api_key}} +
+ +
+
From 3be72aca043c183cd9770fd6d19033af83d46859 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 28 Jul 2024 07:03:00 +0530 Subject: [PATCH 053/211] fix slack url hide --- web/scanEngine/forms.py | 32 ++++++++++++------- .../scanEngine/settings/notification.html | 19 +++++++---- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/web/scanEngine/forms.py b/web/scanEngine/forms.py index 3f6431b54..dd8151e9e 100644 --- a/web/scanEngine/forms.py +++ b/web/scanEngine/forms.py @@ -176,12 +176,14 @@ class Meta: slack_hook_url = forms.CharField( required=False, - widget=forms.TextInput( + widget=forms.PasswordInput( attrs={ - "class": "form-control", + "class": "form-control h-100", "id": "slack_hook_url", "placeholder": "https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX", - })) + }, + render_value=True + )) send_to_lark = forms.BooleanField( required=False, @@ -193,12 +195,14 @@ class Meta: lark_hook_url = forms.CharField( required=False, - widget=forms.TextInput( + widget=forms.PasswordInput( attrs={ "class": "form-control", "id": "lark_hook_url", "placeholder": "https://open.larksuite.com/open-apis/bot/v2/hook/XXXXXXXXXXXXXXXXXXXXXXXX", - })) + }, + render_value=True + )) send_to_discord = forms.BooleanField( required=False, @@ -210,12 +214,14 @@ class Meta: discord_hook_url = forms.CharField( required=False, - widget=forms.TextInput( + widget=forms.PasswordInput( attrs={ "class": "form-control", "id": "discord_hook_url", "placeholder": "https://discord.com/api/webhooks/000000000000000000/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", - })) + }, + render_value=True + )) send_to_telegram = forms.BooleanField( required=False, @@ -227,21 +233,25 @@ class Meta: telegram_bot_token = forms.CharField( required=False, - widget=forms.TextInput( + widget=forms.PasswordInput( attrs={ "class": "form-control", "id": "telegram_bot_token", "placeholder": "Bot Token", - })) + }, + render_value=True + )) telegram_bot_chat_id = forms.CharField( required=False, - widget=forms.TextInput( + widget=forms.PasswordInput( attrs={ "class": "form-control", "id": "telegram_bot_chat_id", "placeholder": "Bot Chat ID", - })) + }, + render_value=True + )) send_scan_status_notif = forms.BooleanField( required=False, diff --git a/web/scanEngine/templates/scanEngine/settings/notification.html b/web/scanEngine/templates/scanEngine/settings/notification.html index cffc23511..6f02519e4 100644 --- a/web/scanEngine/templates/scanEngine/settings/notification.html +++ b/web/scanEngine/templates/scanEngine/settings/notification.html @@ -31,19 +31,24 @@

Send Notifications to:

-
+
- +
- {{form.send_to_slack}} + {{ form.send_to_slack }}
- +
+
+
+ {{ form.slack_hook_url }} +
+ +
- {{form.slack_hook_url}}
- +
- +
From 7d8e3edb9465bbaf3a24b67a80519860caeb7c65 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 28 Jul 2024 07:05:30 +0530 Subject: [PATCH 054/211] added hide icon for lark --- web/scanEngine/forms.py | 4 ++-- .../scanEngine/settings/notification.html | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/web/scanEngine/forms.py b/web/scanEngine/forms.py index dd8151e9e..0f3e0d411 100644 --- a/web/scanEngine/forms.py +++ b/web/scanEngine/forms.py @@ -197,7 +197,7 @@ class Meta: required=False, widget=forms.PasswordInput( attrs={ - "class": "form-control", + "class": "form-control h-100", "id": "lark_hook_url", "placeholder": "https://open.larksuite.com/open-apis/bot/v2/hook/XXXXXXXXXXXXXXXXXXXXXXXX", }, @@ -216,7 +216,7 @@ class Meta: required=False, widget=forms.PasswordInput( attrs={ - "class": "form-control", + "class": "form-control h-100", "id": "discord_hook_url", "placeholder": "https://discord.com/api/webhooks/000000000000000000/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", }, diff --git a/web/scanEngine/templates/scanEngine/settings/notification.html b/web/scanEngine/templates/scanEngine/settings/notification.html index 6f02519e4..6be78ef68 100644 --- a/web/scanEngine/templates/scanEngine/settings/notification.html +++ b/web/scanEngine/templates/scanEngine/settings/notification.html @@ -59,19 +59,24 @@

Send Notifications to:

-
+
- +
- {{form.send_to_lark}} + {{ form.send_to_lark }}
- +
+
+
+ {{ form.lark_hook_url }} +
+ +
- {{form.lark_hook_url}}
- +
- +
From cccb3a9e97caf8aa68c7c3387b57084dbe8f1177 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 28 Jul 2024 07:07:59 +0530 Subject: [PATCH 055/211] hide icon for discord --- .../scanEngine/settings/notification.html | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/web/scanEngine/templates/scanEngine/settings/notification.html b/web/scanEngine/templates/scanEngine/settings/notification.html index 6be78ef68..7410c9936 100644 --- a/web/scanEngine/templates/scanEngine/settings/notification.html +++ b/web/scanEngine/templates/scanEngine/settings/notification.html @@ -87,19 +87,24 @@

Send Notifications to:

-
+
- +
- {{form.send_to_discord}} + {{ form.send_to_discord }}
- +
+
+
+ {{ form.discord_hook_url }} +
+ +
- {{form.discord_hook_url}}
- +
- +
From 78c5ffcf2eb07820ef0b37611c86248c12308911 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 28 Jul 2024 07:13:21 +0530 Subject: [PATCH 056/211] fix telegram input box hide icon --- web/scanEngine/forms.py | 4 +-- .../scanEngine/settings/notification.html | 34 +++++++++++++------ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/web/scanEngine/forms.py b/web/scanEngine/forms.py index 0f3e0d411..e55744621 100644 --- a/web/scanEngine/forms.py +++ b/web/scanEngine/forms.py @@ -235,7 +235,7 @@ class Meta: required=False, widget=forms.PasswordInput( attrs={ - "class": "form-control", + "class": "form-control h-100", "id": "telegram_bot_token", "placeholder": "Bot Token", }, @@ -246,7 +246,7 @@ class Meta: required=False, widget=forms.PasswordInput( attrs={ - "class": "form-control", + "class": "form-control h-100", "id": "telegram_bot_chat_id", "placeholder": "Bot Chat ID", }, diff --git a/web/scanEngine/templates/scanEngine/settings/notification.html b/web/scanEngine/templates/scanEngine/settings/notification.html index 7410c9936..d5b9aa4b5 100644 --- a/web/scanEngine/templates/scanEngine/settings/notification.html +++ b/web/scanEngine/templates/scanEngine/settings/notification.html @@ -57,7 +57,7 @@

Send Notifications to:

- +
@@ -85,7 +85,7 @@

Send Notifications to:

- +
@@ -113,24 +113,36 @@

Send Notifications to:

- +
-
+
- +
- {{form.send_to_telegram}} + {{ form.send_to_telegram }} +
+
+
+
+
+ {{ form.telegram_bot_token }} +
+
- +
+
+ {{ form.telegram_bot_chat_id }} +
+ +
+
- {{form.telegram_bot_token}} - {{form.telegram_bot_chat_id}}
- +
- +
From 4ccb62b6254262a2702377f1d5d961e6ec8f7637 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 28 Jul 2024 07:20:11 +0530 Subject: [PATCH 057/211] use new eye icon inside input box for api --- web/scanEngine/templates/scanEngine/settings/api.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/scanEngine/templates/scanEngine/settings/api.html b/web/scanEngine/templates/scanEngine/settings/api.html index a98a9306b..9f57f1a6d 100644 --- a/web/scanEngine/templates/scanEngine/settings/api.html +++ b/web/scanEngine/templates/scanEngine/settings/api.html @@ -29,13 +29,13 @@

OpenAI keys will be used to generate vulnerability description, remediation, impact and vulnerability report writing using GPT.

-
+
{% if openai_key %} {% else %} {% endif %} -
+
@@ -44,13 +44,13 @@

Netlas keys will be used to get whois information and other OSINT related data.

-
+
{% if netlas_key %} {% else %} {% endif %} -
+
From b749c2a4b6a67e58621a4d580e0b34b7ad481db2 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 28 Jul 2024 07:23:15 +0530 Subject: [PATCH 058/211] hackerone hide icon fix --- .../templates/scanEngine/settings/hackerone.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/web/scanEngine/templates/scanEngine/settings/hackerone.html b/web/scanEngine/templates/scanEngine/settings/hackerone.html index bb1288042..eec049813 100644 --- a/web/scanEngine/templates/scanEngine/settings/hackerone.html +++ b/web/scanEngine/templates/scanEngine/settings/hackerone.html @@ -43,15 +43,15 @@

Hackerone Automatic Vulnerability Report Settings

- {{form.username}} + {{ form.username }}
-
- {{form.api_key}} -
- -
+
+ {{ form.api_key }} +
+ +
From 4297dcff12b28e322c5b2486e0b1291c7dbeada0 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 31 Jul 2024 19:51:11 +0530 Subject: [PATCH 059/211] fix mr conflict --- web/scanEngine/views.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/web/scanEngine/views.py b/web/scanEngine/views.py index 7ee695948..54659f9a3 100644 --- a/web/scanEngine/views.py +++ b/web/scanEngine/views.py @@ -214,16 +214,6 @@ def tool_specific_settings(request, slug): file.write(gf_file.read().decode("utf-8")) file.close() upload_count += 1 - messages.add_message(request, messages.INFO, f'{upload_count} GF files successfully uploaded') - original_filename = re.sub(r'[\\/*?:"<>|]',"", original_filename) - file_extension = original_filename.split('.')[len(gf_file.name.split('.'))-1] - if file_extension == 'json': - base_filename = os.path.splitext(original_filename)[0] - file_path = '/root/.gf/' + base_filename + '.json' - file = open(file_path, "w") - file.write(gf_file.read().decode("utf-8")) - file.close() - upload_count += 1 messages.add_message(request, messages.INFO, f'{upload_count} GF files successfully uploaded') return http.HttpResponseRedirect(reverse('tool_settings', kwargs={'slug': slug})) From 18a782a9d9ee3c7492aef15ae938bb61985a5b85 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 31 Jul 2024 19:51:51 +0530 Subject: [PATCH 060/211] remove print messages --- web/scanEngine/views.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/web/scanEngine/views.py b/web/scanEngine/views.py index 54659f9a3..08433ec44 100644 --- a/web/scanEngine/views.py +++ b/web/scanEngine/views.py @@ -174,9 +174,7 @@ def interesting_lookup(request, slug): form = InterestingLookupForm(request.POST, instance=lookup_keywords) else: form = InterestingLookupForm(request.POST or None) - print(form.errors) if form.is_valid(): - print(form.cleaned_data) form.save() messages.add_message( request, @@ -198,7 +196,6 @@ def tool_specific_settings(request, slug): # check for incoming form requests if request.method == "POST": - print(request.FILES) if 'gfFileUpload[]' in request.FILES: gf_files = request.FILES.getlist('gfFileUpload[]') upload_count = 0 @@ -536,7 +533,6 @@ def add_tool(request, slug): form = ExternalToolForm() if request.method == "POST": form = ExternalToolForm(request.POST) - print(form.errors) if form.is_valid(): # add tool install_command = form.data['install_command'] From 6733b62cf97dd524a4d92cfb65be0727335ae38c Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 31 Jul 2024 19:57:47 +0530 Subject: [PATCH 061/211] remove external ip from dashboard --- web/reNgine/context_processors.py | 8 +------- web/templates/base/_items/footer.html | 1 - web/templates/base/base.html | 2 +- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/web/reNgine/context_processors.py b/web/reNgine/context_processors.py index 8fefeae09..e91b726d1 100644 --- a/web/reNgine/context_processors.py +++ b/web/reNgine/context_processors.py @@ -1,5 +1,5 @@ from dashboard.models import * -import requests + def projects(request): projects = Project.objects.all() @@ -11,10 +11,4 @@ def projects(request): return { 'projects': projects, 'current_project': project - } - -def misc(request): - externalIp = requests.get('https://checkip.amazonaws.com').text.strip() - return { - 'external_ip': externalIp } \ No newline at end of file diff --git a/web/templates/base/_items/footer.html b/web/templates/base/_items/footer.html index 108ba8133..7c7789035 100644 --- a/web/templates/base/_items/footer.html +++ b/web/templates/base/_items/footer.html @@ -1,7 +1,6 @@
- External IP : {{external_ip}}
diff --git a/web/templates/base/base.html b/web/templates/base/base.html index a3d5ac984..b633e356f 100644 --- a/web/templates/base/base.html +++ b/web/templates/base/base.html @@ -92,7 +92,7 @@

{% block page_title %}{% endblock page_title %}

{% include 'base/_items/offcanvas.html' %}
{% include 'base/_items/right_bar.html' %} - {% include 'base/_items/footer.html' %} + {% comment %} {% include 'base/_items/footer.html' %} {% endcomment %}
From 21f4dfff8ff3cd2c62f2910392451431e7558b28 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 31 Jul 2024 19:59:31 +0530 Subject: [PATCH 062/211] Revert "remove external ip from dashboard" This reverts commit 6733b62cf97dd524a4d92cfb65be0727335ae38c. --- web/reNgine/context_processors.py | 8 +++++++- web/templates/base/_items/footer.html | 1 + web/templates/base/base.html | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/web/reNgine/context_processors.py b/web/reNgine/context_processors.py index e91b726d1..8fefeae09 100644 --- a/web/reNgine/context_processors.py +++ b/web/reNgine/context_processors.py @@ -1,5 +1,5 @@ from dashboard.models import * - +import requests def projects(request): projects = Project.objects.all() @@ -11,4 +11,10 @@ def projects(request): return { 'projects': projects, 'current_project': project + } + +def misc(request): + externalIp = requests.get('https://checkip.amazonaws.com').text.strip() + return { + 'external_ip': externalIp } \ No newline at end of file diff --git a/web/templates/base/_items/footer.html b/web/templates/base/_items/footer.html index 7c7789035..108ba8133 100644 --- a/web/templates/base/_items/footer.html +++ b/web/templates/base/_items/footer.html @@ -1,6 +1,7 @@
+ External IP : {{external_ip}}
diff --git a/web/templates/base/base.html b/web/templates/base/base.html index b633e356f..a3d5ac984 100644 --- a/web/templates/base/base.html +++ b/web/templates/base/base.html @@ -92,7 +92,7 @@

{% block page_title %}{% endblock page_title %}

{% include 'base/_items/offcanvas.html' %}
{% include 'base/_items/right_bar.html' %} - {% comment %} {% include 'base/_items/footer.html' %} {% endcomment %} + {% include 'base/_items/footer.html' %}
From 12627198606da6f5ac1911d5608ced79d8917a77 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 31 Jul 2024 20:03:40 +0530 Subject: [PATCH 063/211] remove external ip --- web/reNgine/context_processors.py | 8 -------- web/reNgine/settings.py | 1 - web/templates/base/_items/footer.html | 1 - web/templates/base/base.html | 2 +- 4 files changed, 1 insertion(+), 11 deletions(-) diff --git a/web/reNgine/context_processors.py b/web/reNgine/context_processors.py index cbf89af9b..356289a97 100644 --- a/web/reNgine/context_processors.py +++ b/web/reNgine/context_processors.py @@ -1,5 +1,3 @@ -import requests - from dashboard.models import * from django.conf import settings @@ -16,12 +14,6 @@ def projects(request): 'current_project': project } -def misc(request): - externalIp = requests.get('https://checkip.amazonaws.com').text.strip() - return { - 'external_ip': externalIp - } - def version_context(request): return { 'RENGINE_CURRENT_VERSION': settings.RENGINE_CURRENT_VERSION diff --git a/web/reNgine/settings.py b/web/reNgine/settings.py index e98ff21a6..841c1172c 100644 --- a/web/reNgine/settings.py +++ b/web/reNgine/settings.py @@ -118,7 +118,6 @@ 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', 'reNgine.context_processors.projects', - 'reNgine.context_processors.misc', 'reNgine.context_processors.version_context' ], }, diff --git a/web/templates/base/_items/footer.html b/web/templates/base/_items/footer.html index 108ba8133..7c7789035 100644 --- a/web/templates/base/_items/footer.html +++ b/web/templates/base/_items/footer.html @@ -1,7 +1,6 @@
- External IP : {{external_ip}}
diff --git a/web/templates/base/base.html b/web/templates/base/base.html index a3d5ac984..b633e356f 100644 --- a/web/templates/base/base.html +++ b/web/templates/base/base.html @@ -92,7 +92,7 @@

{% block page_title %}{% endblock page_title %}

{% include 'base/_items/offcanvas.html' %}
{% include 'base/_items/right_bar.html' %} - {% include 'base/_items/footer.html' %} + {% comment %} {% include 'base/_items/footer.html' %} {% endcomment %}
From fd68ce1f9e448ee0323b253e5a3af1d9322c3d3f Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Thu, 1 Aug 2024 09:19:52 +0530 Subject: [PATCH 064/211] Added excluded path in ui, refactor url filter to starting point url --- web/reNgine/definitions.py | 16 + .../templates/startScan/start_scan_ui.html | 105 +++-- web/startScan/views.py | 14 +- .../selectize/css/selectize.bootstrap3.css | 417 ++++++++++++++++++ .../plugins/selectize/css/selectize.css | 333 ++++++++++++++ .../selectize/js/standalone/selectize.min.js | 4 + 6 files changed, 841 insertions(+), 48 deletions(-) create mode 100644 web/static/plugins/selectize/css/selectize.bootstrap3.css create mode 100644 web/static/plugins/selectize/css/selectize.css create mode 100644 web/static/plugins/selectize/js/standalone/selectize.min.js diff --git a/web/reNgine/definitions.py b/web/reNgine/definitions.py index e5b80d577..3a9c9ad7c 100644 --- a/web/reNgine/definitions.py +++ b/web/reNgine/definitions.py @@ -423,6 +423,22 @@ '.pdf', ] +# Default Excluded Paths during Initate Scan +# Mostly static files and directories +DEFAULT_EXCLUDED_PATHS = [ + # Static assets (using regex patterns) + '/static/.*', + '/assets/.*', + '/css/.*', + '/js/.*', + '/images/.*', + '/img/.*', + '/fonts/.*', + + # File types (using regex patterns) + '.*\.ico', +] + # Roles and Permissions PERM_MODIFY_SYSTEM_CONFIGURATIONS = 'modify_system_configurations' PERM_MODIFY_SCAN_CONFIGURATIONS = 'modify_scan_configurations' diff --git a/web/startScan/templates/startScan/start_scan_ui.html b/web/startScan/templates/startScan/start_scan_ui.html index 8c5de75a6..d9c20aeaa 100644 --- a/web/startScan/templates/startScan/start_scan_ui.html +++ b/web/startScan/templates/startScan/start_scan_ui.html @@ -9,6 +9,7 @@ {% block custom_js_css_link %} + {% endblock custom_js_css_link %} {% block breadcrumb_title %} @@ -64,15 +65,33 @@

Out of Scope Subdomains(Optional)

-

Filters

-
+

URL Scope and Exclusions

+
-

Filters (Optional)

- You can filter on a specific path for {{domain.name}}. -
- - +

Starting Point URL (Optional)

+ + + Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains. +
+ If a path is provided (e.g., /home), the scan will focus only on that path and its subpaths, + skipping subdomain scanning. For example, entering '/home' for {{domain.name}} will scan + https://{{domain.name}}/home, but not other parts of {{domain.name}} or its subdomains. +
+
+

Excluded Paths (Optional)

+ + + Enter paths or regex patterns to exclude from the scan. Type a path or pattern and press Enter to add it. + Supports both exact path matching and regex patterns. Examples:
+ • /admin excludes paths starting with '/admin'
+ • /images/.*\.jpg excludes all .jpg files in the images directory
+ • /static/(?:css|js)/ excludes all contents of /static/css/ and /static/js/
+ Common exclusions:
+ • Static assets: /images/.*, /css/.*
+
+ Note: Use regex patterns carefully. While exclusions can speed up scans, be cautious not to exclude critical areas that may contain vulnerabilities. Test your patterns to ensure they match as intended. +
@@ -85,40 +104,13 @@

Filters (Optional)

{% block page_level_script %} + {% endblock page_level_script %} diff --git a/web/startScan/views.py b/web/startScan/views.py index 9e7403a10..edd242071 100644 --- a/web/startScan/views.py +++ b/web/startScan/views.py @@ -259,12 +259,7 @@ def start_scan_ui(request, slug, domain_id): subdomains_in = [s.rstrip() for s in subdomains_in if s] subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split() subdomains_out = [s.rstrip() for s in subdomains_out if s] - paths = request.POST['filterPath'].split() - filterPath = [s.rstrip() for s in paths if s] - if len(filterPath) > 0: - filterPath = filterPath[0] - else: - filterPath = '' + starting_point_url = request.POST['startingPointUrl'].split() # Get engine type engine_id = request.POST['scan_mode'] @@ -286,7 +281,7 @@ def start_scan_ui(request, slug, domain_id): 'results_dir': '/usr/src/scan_results', 'imported_subdomains': subdomains_in, 'out_of_scope_subdomains': subdomains_out, - 'url_filter': filterPath, + 'starting_point_url': starting_point_url, 'initiated_by_id': request.user.id } initiate_scan.apply_async(kwargs=kwargs) @@ -306,11 +301,14 @@ def start_scan_ui(request, slug, domain_id): .filter(default_engine=False) .count() ) + excluded_paths = ','.join(DEFAULT_EXCLUDED_PATHS) context = { 'scan_history_active': 'active', 'domain': domain, 'engines': engine, - 'custom_engine_count': custom_engine_count} + 'custom_engine_count': custom_engine_count, + 'excluded_paths': excluded_paths + } return render(request, 'startScan/start_scan_ui.html', context) diff --git a/web/static/plugins/selectize/css/selectize.bootstrap3.css b/web/static/plugins/selectize/css/selectize.bootstrap3.css new file mode 100644 index 000000000..7c50f611e --- /dev/null +++ b/web/static/plugins/selectize/css/selectize.bootstrap3.css @@ -0,0 +1,417 @@ +/** + * selectize.bootstrap3.css (v0.12.6) - Bootstrap 3 Theme + * Copyright (c) 2013–2015 Brian Reavis & contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF + * ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + * + * @author Brian Reavis + */ +.selectize-control.plugin-drag_drop.multi > .selectize-input > div.ui-sortable-placeholder { + visibility: visible !important; + background: #f2f2f2 !important; + background: rgba(0, 0, 0, 0.06) !important; + border: 0 none !important; + -webkit-box-shadow: inset 0 0 12px 4px #fff; + box-shadow: inset 0 0 12px 4px #fff; +} +.selectize-control.plugin-drag_drop .ui-sortable-placeholder::after { + content: '!'; + visibility: hidden; +} +.selectize-control.plugin-drag_drop .ui-sortable-helper { + -webkit-box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2); + box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2); +} +.selectize-dropdown-header { + position: relative; + padding: 3px 12px; + border-bottom: 1px solid #d0d0d0; + background: #f8f8f8; + -webkit-border-radius: 4px 4px 0 0; + -moz-border-radius: 4px 4px 0 0; + border-radius: 4px 4px 0 0; +} +.selectize-dropdown-header-close { + position: absolute; + right: 12px; + top: 50%; + color: #333333; + opacity: 0.4; + margin-top: -12px; + line-height: 20px; + font-size: 20px !important; +} +.selectize-dropdown-header-close:hover { + color: #000000; +} +.selectize-dropdown.plugin-optgroup_columns .optgroup { + border-right: 1px solid #f2f2f2; + border-top: 0 none; + float: left; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; +} +.selectize-dropdown.plugin-optgroup_columns .optgroup:last-child { + border-right: 0 none; +} +.selectize-dropdown.plugin-optgroup_columns .optgroup:before { + display: none; +} +.selectize-dropdown.plugin-optgroup_columns .optgroup-header { + border-top: 0 none; +} +.selectize-control.plugin-remove_button [data-value] { + position: relative; + padding-right: 24px !important; +} +.selectize-control.plugin-remove_button [data-value] .remove { + z-index: 1; + /* fixes ie bug (see #392) */ + position: absolute; + top: 0; + right: 0; + bottom: 0; + width: 17px; + text-align: center; + font-weight: bold; + font-size: 12px; + color: inherit; + text-decoration: none; + vertical-align: middle; + display: inline-block; + padding: 1px 0 0 0; + border-left: 1px solid rgba(0, 0, 0, 0); + -webkit-border-radius: 0 2px 2px 0; + -moz-border-radius: 0 2px 2px 0; + border-radius: 0 2px 2px 0; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; +} +.selectize-control.plugin-remove_button [data-value] .remove:hover { + background: rgba(0, 0, 0, 0.05); +} +.selectize-control.plugin-remove_button [data-value].active .remove { + border-left-color: rgba(0, 0, 0, 0); +} +.selectize-control.plugin-remove_button .disabled [data-value] .remove:hover { + background: none; +} +.selectize-control.plugin-remove_button .disabled [data-value] .remove { + border-left-color: rgba(77, 77, 77, 0); +} +.selectize-control.plugin-remove_button .remove-single { + position: absolute; + right: 0; + top: 0; + font-size: 23px; +} +.selectize-control { + position: relative; +} +.selectize-dropdown, +.selectize-input, +.selectize-input input { + color: #333333; + font-family: inherit; + font-size: inherit; + line-height: 20px; + -webkit-font-smoothing: inherit; +} +.selectize-input, +.selectize-control.single .selectize-input.input-active { + background: #fff; + cursor: text; + display: inline-block; +} +.selectize-input { + border: 1px solid #ccc; + padding: 6px 12px; + display: inline-block; + width: 100%; + overflow: hidden; + position: relative; + z-index: 1; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + -webkit-box-shadow: none; + box-shadow: none; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; +} +.selectize-control.multi .selectize-input.has-items { + padding: 5px 12px 2px; +} +.selectize-input.full { + background-color: #fff; +} +.selectize-input.disabled, +.selectize-input.disabled * { + cursor: default !important; +} +.selectize-input.focus { + -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.15); + box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.15); +} +.selectize-input.dropdown-active { + -webkit-border-radius: 4px 4px 0 0; + -moz-border-radius: 4px 4px 0 0; + border-radius: 4px 4px 0 0; +} +.selectize-input > * { + vertical-align: baseline; + display: -moz-inline-stack; + display: inline-block; + zoom: 1; + *display: inline; +} +.selectize-control.multi .selectize-input > div { + cursor: pointer; + margin: 0 3px 3px 0; + padding: 1px 3px; + background: #efefef; + color: #333333; + border: 0 solid rgba(0, 0, 0, 0); +} +.selectize-control.multi .selectize-input > div.active { + background: #428bca; + color: #fff; + border: 0 solid rgba(0, 0, 0, 0); +} +.selectize-control.multi .selectize-input.disabled > div, +.selectize-control.multi .selectize-input.disabled > div.active { + color: #808080; + background: #ffffff; + border: 0 solid rgba(77, 77, 77, 0); +} +.selectize-input > input { + display: inline-block !important; + padding: 0 !important; + min-height: 0 !important; + max-height: none !important; + max-width: 100% !important; + margin: 0 !important; + text-indent: 0 !important; + border: 0 none !important; + background: none !important; + line-height: inherit !important; + -webkit-user-select: auto !important; + -webkit-box-shadow: none !important; + box-shadow: none !important; +} +.selectize-input > input::-ms-clear { + display: none; +} +.selectize-input > input:focus { + outline: none !important; +} +.selectize-input::after { + content: ' '; + display: block; + clear: left; +} +.selectize-input.dropdown-active::before { + content: ' '; + display: block; + position: absolute; + background: #ffffff; + height: 1px; + bottom: 0; + left: 0; + right: 0; +} +.selectize-dropdown { + position: absolute; + z-index: 10; + border: 1px solid #d0d0d0; + background: #fff; + margin: -1px 0 0 0; + border-top: 0 none; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + -webkit-border-radius: 0 0 4px 4px; + -moz-border-radius: 0 0 4px 4px; + border-radius: 0 0 4px 4px; +} +.selectize-dropdown [data-selectable] { + cursor: pointer; + overflow: hidden; +} +.selectize-dropdown [data-selectable] .highlight { + background: rgba(255, 237, 40, 0.4); + -webkit-border-radius: 1px; + -moz-border-radius: 1px; + border-radius: 1px; +} +.selectize-dropdown .option, +.selectize-dropdown .optgroup-header { + padding: 3px 12px; +} +.selectize-dropdown .option, +.selectize-dropdown [data-disabled], +.selectize-dropdown [data-disabled] [data-selectable].option { + cursor: inherit; + opacity: 0.5; +} +.selectize-dropdown [data-selectable].option { + opacity: 1; +} +.selectize-dropdown .optgroup:first-child .optgroup-header { + border-top: 0 none; +} +.selectize-dropdown .optgroup-header { + color: #777777; + background: #fff; + cursor: default; +} +.selectize-dropdown .active { + background-color: #f5f5f5; + color: #262626; +} +.selectize-dropdown .active.create { + color: #262626; +} +.selectize-dropdown .create { + color: rgba(51, 51, 51, 0.5); +} +.selectize-dropdown-content { + overflow-y: auto; + overflow-x: hidden; + max-height: 200px; + -webkit-overflow-scrolling: touch; +} +.selectize-control.single .selectize-input, +.selectize-control.single .selectize-input input { + cursor: pointer; +} +.selectize-control.single .selectize-input.input-active, +.selectize-control.single .selectize-input.input-active input { + cursor: text; +} +.selectize-control.single .selectize-input:after { + content: ' '; + display: block; + position: absolute; + top: 50%; + right: 17px; + margin-top: -3px; + width: 0; + height: 0; + border-style: solid; + border-width: 5px 5px 0 5px; + border-color: #333333 transparent transparent transparent; +} +.selectize-control.single .selectize-input.dropdown-active:after { + margin-top: -4px; + border-width: 0 5px 5px 5px; + border-color: transparent transparent #333333 transparent; +} +.selectize-control.rtl.single .selectize-input:after { + left: 17px; + right: auto; +} +.selectize-control.rtl .selectize-input > input { + margin: 0 4px 0 -2px !important; +} +.selectize-control .selectize-input.disabled { + opacity: 0.5; + background-color: #fff; +} +.selectize-dropdown, +.selectize-dropdown.form-control { + height: auto; + padding: 0; + margin: 2px 0 0 0; + z-index: 1000; + background: #fff; + border: 1px solid #ccc; + border: 1px solid rgba(0, 0, 0, 0.15); + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; + -webkit-box-shadow: 0 6px 12px rgba(0, 0, 0, 0.175); + box-shadow: 0 6px 12px rgba(0, 0, 0, 0.175); +} +.selectize-dropdown .optgroup-header { + font-size: 12px; + line-height: 1.42857143; +} +.selectize-dropdown .optgroup:first-child:before { + display: none; +} +.selectize-dropdown .optgroup:before { + content: ' '; + display: block; + height: 1px; + margin: 9px 0; + overflow: hidden; + background-color: #e5e5e5; + margin-left: -12px; + margin-right: -12px; +} +.selectize-dropdown-content { + padding: 5px 0; +} +.selectize-dropdown-header { + padding: 6px 12px; +} +.selectize-input { + min-height: 34px; +} +.selectize-input.dropdown-active { + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; +} +.selectize-input.dropdown-active::before { + display: none; +} +.selectize-input.focus { + border-color: #66afe9; + outline: 0; + -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(102, 175, 233, 0.6); + box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(102, 175, 233, 0.6); +} +.has-error .selectize-input { + border-color: #a94442; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); +} +.has-error .selectize-input:focus { + border-color: #843534; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #ce8483; + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #ce8483; +} +.selectize-control.multi .selectize-input.has-items { + padding-left: 9px; + padding-right: 9px; +} +.selectize-control.multi .selectize-input > div { + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; +} +.form-control.selectize-control { + padding: 0; + height: auto; + border: none; + background: none; + -webkit-box-shadow: none; + box-shadow: none; + -webkit-border-radius: 0; + -moz-border-radius: 0; + border-radius: 0; +} diff --git a/web/static/plugins/selectize/css/selectize.css b/web/static/plugins/selectize/css/selectize.css new file mode 100644 index 000000000..3ec90ad0e --- /dev/null +++ b/web/static/plugins/selectize/css/selectize.css @@ -0,0 +1,333 @@ +/** + * selectize.css (v0.12.6) + * Copyright (c) 2013–2015 Brian Reavis & contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF + * ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + * + * @author Brian Reavis + */ + +.selectize-control.plugin-drag_drop.multi > .selectize-input > div.ui-sortable-placeholder { + visibility: visible !important; + background: #f2f2f2 !important; + background: rgba(0, 0, 0, 0.06) !important; + border: 0 none !important; + -webkit-box-shadow: inset 0 0 12px 4px #fff; + box-shadow: inset 0 0 12px 4px #fff; +} +.selectize-control.plugin-drag_drop .ui-sortable-placeholder::after { + content: '!'; + visibility: hidden; +} +.selectize-control.plugin-drag_drop .ui-sortable-helper { + -webkit-box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2); + box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2); +} +.selectize-dropdown-header { + position: relative; + padding: 5px 8px; + border-bottom: 1px solid #d0d0d0; + background: #f8f8f8; + -webkit-border-radius: 3px 3px 0 0; + -moz-border-radius: 3px 3px 0 0; + border-radius: 3px 3px 0 0; +} +.selectize-dropdown-header-close { + position: absolute; + right: 8px; + top: 50%; + color: #303030; + opacity: 0.4; + margin-top: -12px; + line-height: 20px; + font-size: 20px !important; +} +.selectize-dropdown-header-close:hover { + color: #000000; +} +.selectize-dropdown.plugin-optgroup_columns .optgroup { + border-right: 1px solid #f2f2f2; + border-top: 0 none; + float: left; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; +} +.selectize-dropdown.plugin-optgroup_columns .optgroup:last-child { + border-right: 0 none; +} +.selectize-dropdown.plugin-optgroup_columns .optgroup:before { + display: none; +} +.selectize-dropdown.plugin-optgroup_columns .optgroup-header { + border-top: 0 none; +} +.selectize-control.plugin-remove_button [data-value] { + position: relative; + padding-right: 24px !important; +} +.selectize-control.plugin-remove_button [data-value] .remove { + z-index: 1; + /* fixes ie bug (see #392) */ + position: absolute; + top: 0; + right: 0; + bottom: 0; + width: 17px; + text-align: center; + font-weight: bold; + font-size: 12px; + color: inherit; + text-decoration: none; + vertical-align: middle; + display: inline-block; + padding: 2px 0 0 0; + border-left: 1px solid #d0d0d0; + -webkit-border-radius: 0 2px 2px 0; + -moz-border-radius: 0 2px 2px 0; + border-radius: 0 2px 2px 0; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; +} +.selectize-control.plugin-remove_button [data-value] .remove:hover { + background: rgba(0, 0, 0, 0.05); +} +.selectize-control.plugin-remove_button [data-value].active .remove { + border-left-color: #cacaca; +} +.selectize-control.plugin-remove_button .disabled [data-value] .remove:hover { + background: none; +} +.selectize-control.plugin-remove_button .disabled [data-value] .remove { + border-left-color: #ffffff; +} +.selectize-control.plugin-remove_button .remove-single { + position: absolute; + right: 0; + top: 0; + font-size: 23px; +} +.selectize-control { + position: relative; +} +.selectize-dropdown, +.selectize-input, +.selectize-input input { + color: #303030; + font-family: inherit; + font-size: 13px; + line-height: 18px; + -webkit-font-smoothing: inherit; +} +.selectize-input, +.selectize-control.single .selectize-input.input-active { + background: #fff; + cursor: text; + display: inline-block; +} +.selectize-input { + border: 1px solid #d0d0d0; + padding: 8px 8px; + display: inline-block; + width: 100%; + overflow: hidden; + position: relative; + z-index: 1; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.1); + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.1); + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; +} +.selectize-control.multi .selectize-input.has-items { + padding: 6px 8px 3px; +} +.selectize-input.full { + background-color: #fff; +} +.selectize-input.disabled, +.selectize-input.disabled * { + cursor: default !important; +} +.selectize-input.focus { + -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.15); + box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.15); +} +.selectize-input.dropdown-active { + -webkit-border-radius: 3px 3px 0 0; + -moz-border-radius: 3px 3px 0 0; + border-radius: 3px 3px 0 0; +} +.selectize-input > * { + vertical-align: baseline; + display: -moz-inline-stack; + display: inline-block; + zoom: 1; + *display: inline; +} +.selectize-control.multi .selectize-input > div { + cursor: pointer; + margin: 0 3px 3px 0; + padding: 2px 6px; + background: #f2f2f2; + color: #303030; + border: 0 solid #d0d0d0; +} +.selectize-control.multi .selectize-input > div.active { + background: #e8e8e8; + color: #303030; + border: 0 solid #cacaca; +} +.selectize-control.multi .selectize-input.disabled > div, +.selectize-control.multi .selectize-input.disabled > div.active { + color: #7d7d7d; + background: #ffffff; + border: 0 solid #ffffff; +} +.selectize-input > input { + display: inline-block !important; + padding: 0 !important; + min-height: 0 !important; + max-height: none !important; + max-width: 100% !important; + margin: 0 2px 0 0 !important; + text-indent: 0 !important; + border: 0 none !important; + background: none !important; + line-height: inherit !important; + -webkit-user-select: auto !important; + -webkit-box-shadow: none !important; + box-shadow: none !important; +} +.selectize-input > input::-ms-clear { + display: none; +} +.selectize-input > input:focus { + outline: none !important; +} +.selectize-input::after { + content: ' '; + display: block; + clear: left; +} +.selectize-input.dropdown-active::before { + content: ' '; + display: block; + position: absolute; + background: #f0f0f0; + height: 1px; + bottom: 0; + left: 0; + right: 0; +} +.selectize-dropdown { + position: absolute; + z-index: 10; + border: 1px solid #d0d0d0; + background: #fff; + margin: -1px 0 0 0; + border-top: 0 none; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + -webkit-border-radius: 0 0 3px 3px; + -moz-border-radius: 0 0 3px 3px; + border-radius: 0 0 3px 3px; +} +.selectize-dropdown [data-selectable] { + cursor: pointer; + overflow: hidden; +} +.selectize-dropdown [data-selectable] .highlight { + background: rgba(125, 168, 208, 0.2); + -webkit-border-radius: 1px; + -moz-border-radius: 1px; + border-radius: 1px; +} +.selectize-dropdown .option, +.selectize-dropdown .optgroup-header { + padding: 5px 8px; +} +.selectize-dropdown .option, +.selectize-dropdown [data-disabled], +.selectize-dropdown [data-disabled] [data-selectable].option { + cursor: inherit; + opacity: 0.5; +} +.selectize-dropdown [data-selectable].option { + opacity: 1; +} +.selectize-dropdown .optgroup:first-child .optgroup-header { + border-top: 0 none; +} +.selectize-dropdown .optgroup-header { + color: #303030; + background: #fff; + cursor: default; +} +.selectize-dropdown .active { + background-color: #f5fafd; + color: #495c68; +} +.selectize-dropdown .active.create { + color: #495c68; +} +.selectize-dropdown .create { + color: rgba(48, 48, 48, 0.5); +} +.selectize-dropdown-content { + overflow-y: auto; + overflow-x: hidden; + max-height: 200px; + -webkit-overflow-scrolling: touch; +} +.selectize-control.single .selectize-input, +.selectize-control.single .selectize-input input { + cursor: pointer; +} +.selectize-control.single .selectize-input.input-active, +.selectize-control.single .selectize-input.input-active input { + cursor: text; +} +.selectize-control.single .selectize-input:after { + content: ' '; + display: block; + position: absolute; + top: 50%; + right: 15px; + margin-top: -3px; + width: 0; + height: 0; + border-style: solid; + border-width: 5px 5px 0 5px; + border-color: #808080 transparent transparent transparent; +} +.selectize-control.single .selectize-input.dropdown-active:after { + margin-top: -4px; + border-width: 0 5px 5px 5px; + border-color: transparent transparent #808080 transparent; +} +.selectize-control.rtl.single .selectize-input:after { + left: 15px; + right: auto; +} +.selectize-control.rtl .selectize-input > input { + margin: 0 4px 0 -2px !important; +} +.selectize-control .selectize-input.disabled { + opacity: 0.5; + background-color: #fafafa; +} diff --git a/web/static/plugins/selectize/js/standalone/selectize.min.js b/web/static/plugins/selectize/js/standalone/selectize.min.js new file mode 100644 index 000000000..43046c90d --- /dev/null +++ b/web/static/plugins/selectize/js/standalone/selectize.min.js @@ -0,0 +1,4 @@ +/*! selectize.js - v0.12.6 | https://github.com/selectize/selectize.js | Apache License (v2) */ + +!function(a,b){"function"==typeof define&&define.amd?define("sifter",b):"object"==typeof exports?module.exports=b():a.Sifter=b()}(this,function(){var a=function(a,b){this.items=a,this.settings=b||{diacritics:!0}};a.prototype.tokenize=function(a){if(!(a=e(String(a||"").toLowerCase()))||!a.length)return[];var b,c,d,g,i=[],j=a.split(/ +/);for(b=0,c=j.length;b0)&&d.items.push({score:c,id:e})}):g.iterator(g.items,function(a,b){d.items.push({score:1,id:b})}),e=g.getSortFunction(d,b),e&&d.items.sort(e),d.total=d.items.length,"number"==typeof b.limit&&(d.items=d.items.slice(0,b.limit)),d};var b=function(a,b){return"number"==typeof a&&"number"==typeof b?a>b?1:ab?1:b>a?-1:0)},c=function(a,b){var c,d,e,f;for(c=1,d=arguments.length;c=0&&a.data.length>0){var f=a.data.match(c),g=document.createElement("span");g.className="highlight";var h=a.splitText(e),i=(h.splitText(f[0].length),h.cloneNode(!0));g.appendChild(i),h.parentNode.replaceChild(g,h),b=1}}else if(1===a.nodeType&&a.childNodes&&!/(script|style)/i.test(a.tagName)&&("highlight"!==a.className||"SPAN"!==a.tagName))for(var j=0;j/g,">").replace(/"/g,""")},m={};m.before=function(a,b,c){var d=a[b];a[b]=function(){return c.apply(a,arguments),d.apply(a,arguments)}},m.after=function(a,b,c){var d=a[b];a[b]=function(){var b=d.apply(a,arguments);return c.apply(a,arguments),b}};var n=function(a){var b=!1;return function(){b||(b=!0,a.apply(this,arguments))}},o=function(a,b){var c;return function(){var d=this,e=arguments;window.clearTimeout(c),c=window.setTimeout(function(){a.apply(d,e)},b)}},p=function(a,b,c){var d,e=a.trigger,f={};a.trigger=function(){var c=arguments[0];if(-1===b.indexOf(c))return e.apply(a,arguments);f[c]=arguments},c.apply(a,[]),a.trigger=e;for(d in f)f.hasOwnProperty(d)&&e.apply(a,f[d])},q=function(a,b,c,d){a.on(b,c,function(b){for(var c=b.target;c&&c.parentNode!==a[0];)c=c.parentNode;return b.currentTarget=c,d.apply(this,[b])})},r=function(a){var b={};if("selectionStart"in a)b.start=a.selectionStart,b.length=a.selectionEnd-b.start;else if(document.selection){a.focus();var c=document.selection.createRange(),d=document.selection.createRange().text.length;c.moveStart("character",-a.value.length),b.start=c.text.length-d,b.length=d}return b},s=function(a,b,c){var d,e,f={};if(c)for(d=0,e=c.length;d").css({position:"absolute",top:-99999,left:-99999,width:"auto",padding:0,whiteSpace:"pre"}).appendTo("body")),w.$testInput.text(b),s(c,w.$testInput,["letterSpacing","fontSize","fontFamily","fontWeight","textTransform"]),w.$testInput.width()):0},u=function(a){var b=null,c=function(c,d){var e,f,g,h,i,j,k,l;c=c||window.event||{},d=d||{},c.metaKey||c.altKey||(d.force||!1!==a.data("grow"))&&(e=a.val(),c.type&&"keydown"===c.type.toLowerCase()&&(f=c.keyCode,g=f>=48&&f<=57||f>=65&&f<=90||f>=96&&f<=111||f>=186&&f<=222||32===f,46===f||8===f?(l=r(a[0]),l.length?e=e.substring(0,l.start)+e.substring(l.start+l.length):8===f&&l.start?e=e.substring(0,l.start-1)+e.substring(l.start+1):46===f&&void 0!==l.start&&(e=e.substring(0,l.start)+e.substring(l.start+1))):g&&(j=c.shiftKey,k=String.fromCharCode(c.keyCode),k=j?k.toUpperCase():k.toLowerCase(),e+=k)),h=a.attr("placeholder"),!e&&h&&(e=h),(i=t(e,a)+4)!==b&&(b=i,a.width(i),a.triggerHandler("resize")))};a.on("keydown keyup update blur",c),c()},v=function(a){var b=document.createElement("div");return b.appendChild(a.cloneNode(!0)),b.innerHTML},w=function(c,d){var e,f,g,h,i=this;h=c[0],h.selectize=i;var j=window.getComputedStyle&&window.getComputedStyle(h,null);if(g=j?j.getPropertyValue("direction"):h.currentStyle&&h.currentStyle.direction,g=g||c.parents("[dir]:first").attr("dir")||"",a.extend(i,{order:0,settings:d,$input:c,tabIndex:c.attr("tabindex")||"",tagType:"select"===h.tagName.toLowerCase()?1:2,rtl:/rtl/i.test(g),eventNS:".selectize"+ ++w.count,highlightedValue:null,isBlurring:!1,isOpen:!1,isDisabled:!1,isRequired:c.is("[required]"),isInvalid:!1,isLocked:!1,isFocused:!1,isInputHidden:!1,isSetup:!1,isShiftDown:!1,isCmdDown:!1,isCtrlDown:!1,ignoreFocus:!1,ignoreBlur:!1,ignoreHover:!1,hasOptions:!1,currentResults:null,lastValue:"",caretPos:0,loading:0,loadedSearches:{},$activeOption:null,$activeItems:[],optgroups:{},options:{},userOptions:{},items:[],renderCache:{},onSearchChange:null===d.loadThrottle?i.onSearchChange:o(i.onSearchChange,d.loadThrottle)}),i.sifter=new b(this.options,{diacritics:d.diacritics}),i.settings.options){for(e=0,f=i.settings.options.length;e").addClass(r.wrapperClass).addClass(m).addClass(l),c=a("
").addClass(r.inputClass).addClass("items").appendTo(b),d=a('').appendTo(c).attr("tabindex",w.is(":disabled")?"-1":p.tabIndex),k=a(r.dropdownParent||b),e=a("
").addClass(r.dropdownClass).addClass(l).hide().appendTo(k),j=a("
").addClass(r.dropdownContentClass).appendTo(e),(o=w.attr("id"))&&(d.attr("id",o+"-selectized"),a("label[for='"+o+"']").attr("for",o+"-selectized")),p.settings.copyClassesToDropdown&&e.addClass(m),b.css({width:w[0].style.width}),p.plugins.names.length&&(n="plugin-"+p.plugins.names.join(" plugin-"),b.addClass(n),e.addClass(n)),(null===r.maxItems||r.maxItems>1)&&1===p.tagType&&w.attr("multiple","multiple"),p.settings.placeholder&&d.attr("placeholder",r.placeholder),!p.settings.splitOn&&p.settings.delimiter){var x=p.settings.delimiter.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&");p.settings.splitOn=new RegExp("\\s*"+x+"+\\s*")}w.attr("autocorrect")&&d.attr("autocorrect",w.attr("autocorrect")),w.attr("autocapitalize")&&d.attr("autocapitalize",w.attr("autocapitalize")),d[0].type=w[0].type,p.$wrapper=b,p.$control=c,p.$control_input=d,p.$dropdown=e,p.$dropdown_content=j,e.on("mouseenter mousedown click","[data-disabled]>[data-selectable]",function(a){a.stopImmediatePropagation()}),e.on("mouseenter","[data-selectable]",function(){return p.onOptionHover.apply(p,arguments)}),e.on("mousedown click","[data-selectable]",function(){return p.onOptionSelect.apply(p,arguments)}),q(c,"mousedown","*:not(input)",function(){return p.onItemSelect.apply(p,arguments)}),u(d),c.on({mousedown:function(){return p.onMouseDown.apply(p,arguments)},click:function(){return p.onClick.apply(p,arguments)}}),d.on({mousedown:function(a){a.stopPropagation()},keydown:function(){return p.onKeyDown.apply(p,arguments)},keyup:function(){return p.onKeyUp.apply(p,arguments)},keypress:function(){return p.onKeyPress.apply(p,arguments)},resize:function(){p.positionDropdown.apply(p,[])},blur:function(){return p.onBlur.apply(p,arguments)},focus:function(){return p.ignoreBlur=!1,p.onFocus.apply(p,arguments)},paste:function(){return p.onPaste.apply(p,arguments)}}),v.on("keydown"+s,function(a){p.isCmdDown=a[f?"metaKey":"ctrlKey"],p.isCtrlDown=a[f?"altKey":"ctrlKey"],p.isShiftDown=a.shiftKey}),v.on("keyup"+s,function(a){a.keyCode===h&&(p.isCtrlDown=!1),16===a.keyCode&&(p.isShiftDown=!1),a.keyCode===g&&(p.isCmdDown=!1)}),v.on("mousedown"+s,function(a){if(p.isFocused){if(a.target===p.$dropdown[0]||a.target.parentNode===p.$dropdown[0])return!1;p.$control.has(a.target).length||a.target===p.$control[0]||p.blur(a.target)}}),t.on(["scroll"+s,"resize"+s].join(" "),function(){p.isOpen&&p.positionDropdown.apply(p,arguments)}),t.on("mousemove"+s,function(){p.ignoreHover=!1}),this.revertSettings={$children:w.children().detach(),tabindex:w.attr("tabindex")},w.attr("tabindex",-1).hide().after(p.$wrapper),a.isArray(r.items)&&(p.setValue(r.items),delete r.items),i&&w.on("invalid"+s,function(a){a.preventDefault(),p.isInvalid=!0,p.refreshState()}),p.updateOriginalInput(),p.refreshItems(),p.refreshState(),p.updatePlaceholder(),p.isSetup=!0,w.is(":disabled")&&p.disable(),p.on("change",this.onChange),w.data("selectize",p),w.addClass("selectized"),p.trigger("initialize"),!0===r.preload&&p.onSearchChange("")},setupTemplates:function(){var b=this,c=b.settings.labelField,d=b.settings.optgroupLabelField,e={optgroup:function(a){return'
'+a.html+"
"},optgroup_header:function(a,b){return'
'+b(a[d])+"
"},option:function(a,b){return'
'+b(a[c])+"
"},item:function(a,b){return'
'+b(a[c])+"
"},option_create:function(a,b){return'
Add '+b(a.input)+"…
"}};b.settings.render=a.extend({},e,b.settings.render)},setupCallbacks:function(){var a,b,c={initialize:"onInitialize",change:"onChange",item_add:"onItemAdd",item_remove:"onItemRemove",clear:"onClear",option_add:"onOptionAdd",option_remove:"onOptionRemove",option_clear:"onOptionClear",optgroup_add:"onOptionGroupAdd",optgroup_remove:"onOptionGroupRemove",optgroup_clear:"onOptionGroupClear",dropdown_open:"onDropdownOpen",dropdown_close:"onDropdownClose",type:"onType",load:"onLoad",focus:"onFocus",blur:"onBlur"};for(a in c)c.hasOwnProperty(a)&&(b=this.settings[c[a]])&&this.on(a,b)},onClick:function(a){var b=this;b.isFocused&&b.isOpen||(b.focus(),a.preventDefault())},onMouseDown:function(b){var c=this,d=b.isDefaultPrevented();a(b.target);if(c.isFocused){if(b.target!==c.$control_input[0])return"single"===c.settings.mode?c.isOpen?c.close():c.open():d||c.setActiveItem(null),!1}else d||window.setTimeout(function(){c.focus()},0)},onChange:function(){this.$input.trigger("change")},onPaste:function(b){var c=this;if(c.isFull()||c.isInputHidden||c.isLocked)return void b.preventDefault();c.settings.splitOn&&setTimeout(function(){var b=c.$control_input.val();if(b.match(c.settings.splitOn))for(var d=a.trim(b).split(c.settings.splitOn),e=0,f=d.length;eh&&(j=g,g=h,h=j),e=g;e<=h;e++)i=l.$control[0].childNodes[e],-1===l.$activeItems.indexOf(i)&&(a(i).addClass("active"),l.$activeItems.push(i));c.preventDefault()}else"mousedown"===d&&l.isCtrlDown||"keydown"===d&&this.isShiftDown?b.hasClass("active")?(f=l.$activeItems.indexOf(b[0]),l.$activeItems.splice(f,1),b.removeClass("active")):l.$activeItems.push(b.addClass("active")[0]):(a(l.$activeItems).removeClass("active"),l.$activeItems=[b.addClass("active")[0]]);l.hideInput(),this.isFocused||l.focus()}},setActiveOption:function(b,c,d){var e,f,g,h,i,k=this;k.$activeOption&&k.$activeOption.removeClass("active"),k.$activeOption=null,b=a(b),b.length&&(k.$activeOption=b.addClass("active"),!c&&j(c)||(e=k.$dropdown_content.height(),f=k.$activeOption.outerHeight(!0),c=k.$dropdown_content.scrollTop()||0,g=k.$activeOption.offset().top-k.$dropdown_content.offset().top+c,h=g,i=g-e+f,g+f>e+c?k.$dropdown_content.stop().animate({scrollTop:i},d?k.settings.scrollDuration:0):g=0;c--)-1!==f.items.indexOf(k(d.items[c].id))&&d.items.splice(c,1);return d},refreshOptions:function(b){var c,e,f,g,h,i,j,l,m,n,o,p,q,r,s,t;void 0===b&&(b=!0);var u=this,w=a.trim(u.$control_input.val()),x=u.search(w),y=u.$dropdown_content,z=u.$activeOption&&k(u.$activeOption.attr("data-value"));for(g=x.items.length,"number"==typeof u.settings.maxOptions&&(g=Math.min(g,u.settings.maxOptions)),h={},i=[],c=0;c0||q,u.hasOptions?(x.items.length>0?(s=z&&u.getOption(z),s&&s.length?r=s:"single"===u.settings.mode&&u.items.length&&(r=u.getOption(u.items[0])),r&&r.length||(r=t&&!u.settings.addPrecedence?u.getAdjacentOption(t,1):y.find("[data-selectable]:first"))):r=t,u.setActiveOption(r),b&&!u.isOpen&&u.open()):(u.setActiveOption(null),b&&u.isOpen&&u.close())},addOption:function(b){var c,d,e,f=this;if(a.isArray(b))for(c=0,d=b.length;c=0&&e0),b.$control_input.data("grow",!c&&!d)},isFull:function(){ +return null!==this.settings.maxItems&&this.items.length>=this.settings.maxItems},updateOriginalInput:function(a){var b,c,d,e,f=this;if(a=a||{},1===f.tagType){for(d=[],b=0,c=f.items.length;b'+l(e)+"");d.length||this.$input.attr("multiple")||d.push(''),f.$input.html(d.join(""))}else f.$input.val(f.getValue()),f.$input.attr("value",f.$input.val());f.isSetup&&(a.silent||f.trigger("change",f.$input.val()))},updatePlaceholder:function(){if(this.settings.placeholder){var a=this.$control_input;this.items.length?a.removeAttr("placeholder"):a.attr("placeholder",this.settings.placeholder),a.triggerHandler("update",{force:!0})}},open:function(){var a=this;a.isLocked||a.isOpen||"multi"===a.settings.mode&&a.isFull()||(a.focus(),a.isOpen=!0,a.refreshState(),a.$dropdown.css({visibility:"hidden",display:"block"}),a.positionDropdown(),a.$dropdown.css({visibility:"visible"}),a.trigger("dropdown_open",a.$dropdown))},close:function(){var a=this,b=a.isOpen;"single"===a.settings.mode&&a.items.length&&(a.hideInput(),a.isBlurring||a.$control_input.blur()),a.isOpen=!1,a.$dropdown.hide(),a.setActiveOption(null),a.refreshState(),b&&a.trigger("dropdown_close",a.$dropdown)},positionDropdown:function(){var a=this.$control,b="body"===this.settings.dropdownParent?a.offset():a.position();b.top+=a.outerHeight(!0),this.$dropdown.css({width:a[0].getBoundingClientRect().width,top:b.top,left:b.left})},clear:function(a){var b=this;b.items.length&&(b.$control.children(":not(input)").remove(),b.items=[],b.lastQuery=null,b.setCaret(0),b.setActiveItem(null),b.updatePlaceholder(),b.updateOriginalInput({silent:a}),b.refreshState(),b.showInput(),b.trigger("clear"))},insertAtCaret:function(a){var b=Math.min(this.caretPos,this.items.length),c=a[0],d=this.buffer||this.$control[0];0===b?d.insertBefore(c,d.firstChild):d.insertBefore(c,d.childNodes[b]),this.setCaret(b+1)},deleteSelection:function(b){var c,d,e,f,g,h,i,j,k,l=this;if(e=b&&8===b.keyCode?-1:1,f=r(l.$control_input[0]),l.$activeOption&&!l.settings.hideSelected&&(i=l.getAdjacentOption(l.$activeOption,-1).attr("data-value")),g=[],l.$activeItems.length){for(k=l.$control.children(".active:"+(e>0?"last":"first")),h=l.$control.children(":not(input)").index(k),e>0&&h++,c=0,d=l.$activeItems.length;c0&&f.start===l.$control_input.val().length&&g.push(l.items[l.caretPos]));if(!g.length||"function"==typeof l.settings.onDelete&&!1===l.settings.onDelete.apply(l,[g]))return!1;for(void 0!==h&&l.setCaret(h);g.length;)l.removeItem(g.pop());return l.showInput(),l.positionDropdown(),l.refreshOptions(!0),i&&(j=l.getOption(i),j.length&&l.setActiveOption(j)),!0},advanceSelection:function(a,b){var c,d,e,f,g,h=this;0!==a&&(h.rtl&&(a*=-1),c=a>0?"last":"first",d=r(h.$control_input[0]),h.isFocused&&!h.isInputHidden?(f=h.$control_input.val().length,(a<0?0===d.start&&0===d.length:d.start===f)&&!f&&h.advanceCaret(a,b)):(g=h.$control.children(".active:"+c),g.length&&(e=h.$control.children(":not(input)").index(g),h.setActiveItem(null),h.setCaret(a>0?e+1:e))))},advanceCaret:function(a,b){var c,d,e=this;0!==a&&(c=a>0?"next":"prev",e.isShiftDown?(d=e.$control_input[c](),d.length&&(e.hideInput(),e.setActiveItem(d),b&&b.preventDefault())):e.setCaret(e.caretPos+a))},setCaret:function(b){var c=this;if(b="single"===c.settings.mode?c.items.length:Math.max(0,Math.min(c.items.length,b)),!c.isPending){var d,e,f,g;for(f=c.$control.children(":not(input)"),d=0,e=f.length;d
'+a.title+'×
'}},b),c.setup=function(){var d=c.setup;return function(){d.apply(c,arguments),c.$dropdown_header=a(b.html(b)),c.$dropdown.prepend(c.$dropdown_header)}}()}),w.define("optgroup_columns",function(b){var c=this;b=a.extend({equalizeWidth:!0,equalizeHeight:!0},b),this.getAdjacentOption=function(b,c){var d=b.closest("[data-group]").find("[data-selectable]"),e=d.index(b)+c;return e>=0&&e
',a=a.firstChild,c.body.appendChild(a),b=d.width=a.offsetWidth-a.clientWidth,c.body.removeChild(a)),b},e=function(){var e,f,g,h,i,j,k;if(k=a("[data-group]",c.$dropdown_content),(f=k.length)&&c.$dropdown_content.width()){if(b.equalizeHeight){for(g=0,e=0;e1&&(i=j-h*(f-1),k.eq(f-1).css({width:i})))}};(b.equalizeHeight||b.equalizeWidth)&&(m.after(this,"positionDropdown",e),m.after(this,"refreshOptions",e))}),w.define("remove_button",function(b){b=a.extend({label:"×",title:"Remove",className:"remove",append:!0},b);if("single"===this.settings.mode)return void function(b,c){c.className="remove-single";var d=b,e=''+c.label+"",f=function(b,c){return a("").append(b).append(c)};b.setup=function(){var g=d.setup;return function(){if(c.append){var h=a(d.$input.context).attr("id"),i=(a("#"+h),d.settings.render.item);d.settings.render.item=function(a){return f(i.apply(b,arguments),e)}}g.apply(b,arguments),b.$control.on("click","."+c.className,function(a){a.preventDefault(),d.isLocked||d.clear()})}}()}(this,b);!function(b,c){var d=b,e=''+c.label+"",f=function(a,b){var c=a.search(/(<\/[^>]+>\s*)$/);return a.substring(0,c)+b+a.substring(c)};b.setup=function(){var g=d.setup;return function(){if(c.append){var h=d.settings.render.item;d.settings.render.item=function(a){return f(h.apply(b,arguments),e)}}g.apply(b,arguments),b.$control.on("click","."+c.className,function(b){if(b.preventDefault(),!d.isLocked){var c=a(b.currentTarget).parent();d.setActiveItem(c),d.deleteSelection()&&d.setCaret(d.items.length)}})}}()}(this,b)}),w.define("restore_on_backspace",function(a){var b=this;a.text=a.text||function(a){return a[this.settings.labelField]},this.onKeyDown=function(){var c=b.onKeyDown;return function(b){var d,e;return 8===b.keyCode&&""===this.$control_input.val()&&!this.$activeItems.length&&(d=this.caretPos-1)>=0&&d Date: Fri, 2 Aug 2024 20:25:41 +0530 Subject: [PATCH 065/211] refactor url_filter to starting_point_url --- web/reNgine/tasks.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 7ab9ed963..863d8d86a 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -57,7 +57,9 @@ def initiate_scan( imported_subdomains=[], out_of_scope_subdomains=[], initiated_by_id=None, - url_filter=''): + starting_point_url='', + excluded_paths=[], + ): """Initiate a new scan. Args: @@ -68,8 +70,9 @@ def initiate_scan( results_dir (str): Results directory. imported_subdomains (list): Imported subdomains. out_of_scope_subdomains (list): Out-of-scope subdomains. - url_filter (str): URL path. Default: ''. + starting_point_url (str): URL path. Default: '' Defined where to start the scan. initiated_by (int): User ID initiating the scan. + excluded_paths (list): Excluded paths. Default: [], url paths to exclude from scan. """ logger.info('Initiating scan on celery') scan = None @@ -89,7 +92,7 @@ def initiate_scan( domain.save() # Get path filter - url_filter = url_filter.rstrip('/') + starting_point_url = starting_point_url.rstrip('/') # for live scan scan history id is passed as scan_history_id # and no need to create scan_history object @@ -124,7 +127,7 @@ def initiate_scan( 'engine_id': engine_id, 'domain_id': domain.id, 'results_dir': scan.results_dir, - 'url_filter': url_filter, + 'starting_point_url': starting_point_url, 'yaml_configuration': config, 'out_of_scope_subdomains': out_of_scope_subdomains } @@ -148,7 +151,7 @@ def initiate_scan( # If enable_http_crawl is set, create an initial root HTTP endpoint so that # HTTP crawling can start somewhere - http_url = f'{domain.name}{url_filter}' if url_filter else domain.name + http_url = f'{domain.name}{starting_point_url}' if starting_point_url else domain.name endpoint, _ = save_endpoint( http_url, ctx=ctx, @@ -224,7 +227,7 @@ def initiate_subscan( engine_id=None, scan_type=None, results_dir=RENGINE_RESULTS, - url_filter=''): + starting_point_url=''): """Initiate a new subscan. Args: @@ -233,7 +236,7 @@ def initiate_subscan( engine_id (int): Engine ID. scan_type (int): Scan type (periodic, live). results_dir (str): Results directory. - url_filter (str): URL path. Default: '' + starting_point_url (str): URL path. Default: '' """ # Get Subdomain, Domain and ScanHistory @@ -291,12 +294,12 @@ def initiate_subscan( 'subdomain_id': subdomain.id, 'yaml_configuration': config, 'results_dir': results_dir, - 'url_filter': url_filter + 'starting_point_url': starting_point_url } # Create initial endpoints in DB: find domain HTTP endpoint so that HTTP # crawling can start somewhere - base_url = f'{subdomain.name}{url_filter}' if url_filter else subdomain.name + base_url = f'{subdomain.name}{starting_point_url}' if starting_point_url else subdomain.name endpoint, _ = save_endpoint( base_url, crawl=enable_http_crawl, @@ -398,8 +401,8 @@ def subdomain_discovery( if not host: host = self.subdomain.name if self.subdomain else self.domain.name - if self.url_filter: - logger.warning(f'Ignoring subdomains scan as an URL path filter was passed ({self.url_filter}).') + if self.starting_point_url: + logger.warning(f'Ignoring subdomains scan as an URL path filter was passed ({self.starting_point_url}).') return # Config @@ -1921,7 +1924,7 @@ def fetch_url(self, urls=[], ctx={}, description=None): if base_url and urlpath: subdomain = urlparse(base_url) - url = f'{subdomain.scheme}://{subdomain.netloc}{self.url_filter}' + url = f'{subdomain.scheme}://{subdomain.netloc}{self.starting_point_url}' if not validators.url(url): logger.warning(f'Invalid URL "{url}". Skipping.') @@ -1930,8 +1933,8 @@ def fetch_url(self, urls=[], ctx={}, description=None): all_urls.append(url) # Filter out URLs if a path filter was passed - if self.url_filter: - all_urls = [url for url in all_urls if self.url_filter in url] + if self.starting_point_url: + all_urls = [url for url in all_urls if self.starting_point_url in url] # Write result to output path with open(self.output_path, 'w') as f: @@ -2827,8 +2830,8 @@ def http_crawl( input_path = f'{self.results_dir}/httpx_input.txt' history_file = f'{self.results_dir}/commands.txt' if urls: # direct passing URLs to check - if self.url_filter: - urls = [u for u in urls if self.url_filter in u] + if self.starting_point_url: + urls = [u for u in urls if self.starting_point_url in u] with open(input_path, 'w') as f: f.write('\n'.join(urls)) else: From 79a266e253344d8dc9d0a5d318391e27cf9b37fa Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 2 Aug 2024 20:35:55 +0530 Subject: [PATCH 066/211] refactor starting_point_url --- web/reNgine/celery_custom_task.py | 3 ++- web/startScan/views.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/web/reNgine/celery_custom_task.py b/web/reNgine/celery_custom_task.py index 37bbdbbbb..2c6b11f25 100644 --- a/web/reNgine/celery_custom_task.py +++ b/web/reNgine/celery_custom_task.py @@ -67,7 +67,8 @@ def __call__(self, *args, **kwargs): self.subscan_id = ctx.get('subscan_id') self.engine_id = ctx.get('engine_id') self.filename = ctx.get('filename') - self.url_filter = ctx.get('url_filter', '') + self.starting_point_url = ctx.get('starting_point_url', '') + self.excluded_paths = ctx.get('excluded_paths', []) self.results_dir = ctx.get('results_dir', RENGINE_RESULTS) self.yaml_configuration = ctx.get('yaml_configuration', {}) self.out_of_scope_subdomains = ctx.get('out_of_scope_subdomains', []) diff --git a/web/startScan/views.py b/web/startScan/views.py index edd242071..c00af36a7 100644 --- a/web/startScan/views.py +++ b/web/startScan/views.py @@ -259,7 +259,8 @@ def start_scan_ui(request, slug, domain_id): subdomains_in = [s.rstrip() for s in subdomains_in if s] subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split() subdomains_out = [s.rstrip() for s in subdomains_out if s] - starting_point_url = request.POST['startingPointUrl'].split() + starting_point_url = request.POST['startingPointUrl'].strip() + excluded_paths = request.POST['excludedPaths'] # Get engine type engine_id = request.POST['scan_mode'] @@ -282,6 +283,7 @@ def start_scan_ui(request, slug, domain_id): 'imported_subdomains': subdomains_in, 'out_of_scope_subdomains': subdomains_out, 'starting_point_url': starting_point_url, + 'excluded_paths': excluded_paths, 'initiated_by_id': request.user.id } initiate_scan.apply_async(kwargs=kwargs) From fa761a794b051525bc669a16134bb0495b7436e1 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 2 Aug 2024 22:08:29 +0530 Subject: [PATCH 067/211] filter exclude path :rocket: --- web/reNgine/common_func.py | 47 ++++++++++++++++++- web/reNgine/tasks.py | 19 +++++++- .../templates/startScan/start_scan_ui.html | 4 +- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/web/reNgine/common_func.py b/web/reNgine/common_func.py index c9c50fe5d..c13c09eb5 100644 --- a/web/reNgine/common_func.py +++ b/web/reNgine/common_func.py @@ -1158,4 +1158,49 @@ def update_or_create_port(port_number, service_name=None, description=None): ) created = True finally: - return port, created \ No newline at end of file + return port, created + + +def exclude_urls_by_patterns(exclude_paths, urls): + """ + Filter out URLs based on a list of exclusion patterns provided from user + + Args: + exclude_patterns (list of str): A list of patterns to exclude. + These can be plain path or regex. + urls (list of str): A list of URLs to filter from. + + Returns: + list of str: A new list containing URLs that don't match any exclusion pattern. + """ + if not exclude_paths: + # if no exclude paths are passed and is empty list return all urls as it is + return urls + + compiled_patterns = [] + for path in exclude_paths: + # treat each path as either regex or plain path + try: + raw_pattern = r"{}".format(path) + compiled_patterns.append(re.compile(raw_pattern)) + except re.error: + compiled_patterns.append(path) + + filtered_urls = [] + for url in urls: + exclude = False + for pattern in compiled_patterns: + if isinstance(pattern, re.Pattern): + if pattern.search(url): + exclude = True + break + else: + if pattern in url: #if the word matches anywhere in url exclude + exclude = True + break + + # if none conditions matches then add the url to filtered urls + if not exclude: + filtered_urls.append(url) + + return filtered_urls \ No newline at end of file diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 863d8d86a..a75032b2d 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -128,6 +128,7 @@ def initiate_scan( 'domain_id': domain.id, 'results_dir': scan.results_dir, 'starting_point_url': starting_point_url, + 'excluded_paths': excluded_paths, 'yaml_configuration': config, 'out_of_scope_subdomains': out_of_scope_subdomains } @@ -227,7 +228,9 @@ def initiate_subscan( engine_id=None, scan_type=None, results_dir=RENGINE_RESULTS, - starting_point_url=''): + starting_point_url='', + excluded_paths=[], + ): """Initiate a new subscan. Args: @@ -237,6 +240,7 @@ def initiate_subscan( scan_type (int): Scan type (periodic, live). results_dir (str): Results directory. starting_point_url (str): URL path. Default: '' + excluded_paths (list): Excluded paths. Default: [], url paths to exclude from scan. """ # Get Subdomain, Domain and ScanHistory @@ -294,7 +298,8 @@ def initiate_subscan( 'subdomain_id': subdomain.id, 'yaml_configuration': config, 'results_dir': results_dir, - 'starting_point_url': starting_point_url + 'starting_point_url': starting_point_url, + 'excluded_paths': excluded_paths, } # Create initial endpoints in DB: find domain HTTP endpoint so that HTTP @@ -1936,6 +1941,10 @@ def fetch_url(self, urls=[], ctx={}, description=None): if self.starting_point_url: all_urls = [url for url in all_urls if self.starting_point_url in url] + # if exclude_paths is found, then remove urls matching those paths + if self.excluded_paths: + all_urls = exclude_urls_by_patterns(self.excluded_paths, all_urls) + # Write result to output path with open(self.output_path, 'w') as f: f.write('\n'.join(all_urls)) @@ -2830,8 +2839,14 @@ def http_crawl( input_path = f'{self.results_dir}/httpx_input.txt' history_file = f'{self.results_dir}/commands.txt' if urls: # direct passing URLs to check + print(vars(self).items()) if self.starting_point_url: urls = [u for u in urls if self.starting_point_url in u] + + # exclude urls by pattern + if self.excluded_paths: + urls = exclude_urls_by_patterns(self.excluded_paths, urls) + with open(input_path, 'w') as f: f.write('\n'.join(urls)) else: diff --git a/web/startScan/templates/startScan/start_scan_ui.html b/web/startScan/templates/startScan/start_scan_ui.html index d9c20aeaa..181a40400 100644 --- a/web/startScan/templates/startScan/start_scan_ui.html +++ b/web/startScan/templates/startScan/start_scan_ui.html @@ -69,7 +69,7 @@

URL Scope and Exclusions

Starting Point URL (Optional)

- + Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains.
@@ -80,7 +80,7 @@

Starting Point URL (Optional)

Excluded Paths (Optional)

- + Enter paths or regex patterns to exclude from the scan. Type a path or pattern and press Enter to add it. Supports both exact path matching and regex patterns. Examples:
From 7abaae33165213e039ab1f7126d4b14375fd8f09 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 2 Aug 2024 23:33:57 +0530 Subject: [PATCH 068/211] fix typos for excluded paths --- web/reNgine/common_func.py | 1 + web/reNgine/tasks.py | 9 ++++----- web/startScan/views.py | 5 ++++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/web/reNgine/common_func.py b/web/reNgine/common_func.py index c13c09eb5..2f1cec942 100644 --- a/web/reNgine/common_func.py +++ b/web/reNgine/common_func.py @@ -1173,6 +1173,7 @@ def exclude_urls_by_patterns(exclude_paths, urls): Returns: list of str: A new list containing URLs that don't match any exclusion pattern. """ + logger.info('exclude_urls_by_patterns') if not exclude_paths: # if no exclude paths are passed and is empty list return all urls as it is return urls diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index a75032b2d..3f0291d02 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -2839,14 +2839,9 @@ def http_crawl( input_path = f'{self.results_dir}/httpx_input.txt' history_file = f'{self.results_dir}/commands.txt' if urls: # direct passing URLs to check - print(vars(self).items()) if self.starting_point_url: urls = [u for u in urls if self.starting_point_url in u] - # exclude urls by pattern - if self.excluded_paths: - urls = exclude_urls_by_patterns(self.excluded_paths, urls) - with open(input_path, 'w') as f: f.write('\n'.join(urls)) else: @@ -2857,6 +2852,10 @@ def http_crawl( ) # logger.debug(urls) + # exclude urls by pattern + if self.excluded_paths: + urls = exclude_urls_by_patterns(self.excluded_paths, urls) + # If no URLs found, skip it if not urls: return diff --git a/web/startScan/views.py b/web/startScan/views.py index c00af36a7..37d7d6858 100644 --- a/web/startScan/views.py +++ b/web/startScan/views.py @@ -260,7 +260,10 @@ def start_scan_ui(request, slug, domain_id): subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split() subdomains_out = [s.rstrip() for s in subdomains_out if s] starting_point_url = request.POST['startingPointUrl'].strip() - excluded_paths = request.POST['excludedPaths'] + excluded_paths = request.POST['excludedPaths'] # string separated by , + + # split excluded paths by , + excluded_paths = [path.strip() for path in excluded_paths.split(',')] # Get engine type engine_id = request.POST['scan_mode'] From 999194c87145c740d32b16693e4b7ad1539d5fe8 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 7 Aug 2024 22:25:57 +0530 Subject: [PATCH 069/211] feat: add import, out_of_scope subdomains, and starting point url, etc in multiple scans --- .../startScan/_items/start_scan_wizard.html | 88 +++++++++++ .../startScan/start_multiple_scan_ui.html | 41 ++---- .../templates/startScan/start_scan_ui.html | 138 +----------------- web/startScan/views.py | 34 +++-- web/static/custom/start-scan-wizard-init.js | 59 ++++++++ 5 files changed, 183 insertions(+), 177 deletions(-) create mode 100644 web/startScan/templates/startScan/_items/start_scan_wizard.html create mode 100644 web/static/custom/start-scan-wizard-init.js diff --git a/web/startScan/templates/startScan/_items/start_scan_wizard.html b/web/startScan/templates/startScan/_items/start_scan_wizard.html new file mode 100644 index 000000000..cecb29f75 --- /dev/null +++ b/web/startScan/templates/startScan/_items/start_scan_wizard.html @@ -0,0 +1,88 @@ +
+
+
+
+ {% if domain_list %} +

Initiating multiple scan for {{ domain_list|length }} targets:

+ {% for domain in domain_list %} + {{domain}} + {% endfor %} +
+
+ {% endif %} +
+ {% csrf_token %} +
+

Choose Scan Engine

+
+

Select Scan Engine

+ {% if custom_engines_count == 0 %} + + {% endif %} + {% include "startScan/_items/scanEngine_select.html" %} +
+

Import/Ignore Subdomains

+
+
+

Import Subdomains(Optional)

+ You can import subdomains for {{domain.name}} using your private recon tools. +
+ + + +
+
+

Out of Scope Subdomains(Optional)

+ You can import subdomains for {{domain.name}} using your private recon tools. +
+ + + +
+
+ +

URL Scope and Exclusions

+
+
+

Starting Point URL (Optional)

+ + + Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains. +
+ If a path is provided (e.g., /home), the scan will focus only on that path and its subpaths, + skipping subdomain scanning. For example, entering '/home' for {{domain.name}} will scan + https://{{domain.name}}/home, but not other parts of {{domain.name}} or its subdomains. + +
+
+

Excluded Paths (Optional)

+ + + Enter paths or regex patterns to exclude from the scan. Type a path or pattern and press Enter to add it. + Supports both exact path matching and regex patterns. Examples:
+ • /admin excludes paths starting with '/admin'
+ • /images/.*\.jpg excludes all .jpg files in the images directory
+ • /static/(?:css|js)/ excludes all contents of /static/css/ and /static/js/
+ Common exclusions:
+ • Static assets: /images/.*, /css/.*
+
+ Note: Use regex patterns carefully. While exclusions can speed up scans, be cautious not to exclude critical areas that may contain vulnerabilities. Test your patterns to ensure they match as intended. + +
+
+ {% if domain_ids %} + + {% endif %} + +
+
+
+
+ diff --git a/web/startScan/templates/startScan/start_multiple_scan_ui.html b/web/startScan/templates/startScan/start_multiple_scan_ui.html index f69e85d47..0230b8f38 100644 --- a/web/startScan/templates/startScan/start_multiple_scan_ui.html +++ b/web/startScan/templates/startScan/start_multiple_scan_ui.html @@ -2,11 +2,15 @@ {% load static %} {% block title %} -Start Scan +Start Scan for Multiple Targets {% endblock title %} + {% block custom_js_css_link %} + + + {% endblock custom_js_css_link %} {% block breadcrumb_title %} @@ -15,41 +19,16 @@ {% endblock breadcrumb_title %} {% block page_title %} -Initiate Scan for Multiple Targets + Initiate Scan for Multiple Targets {% endblock page_title %} {% block main_content %} -
-
-
-
-

Initiating multiple scan for {{ domain_list|length }} targets:

- {% for domain in domain_list %} - {{domain}} - {% endfor %} -
- {% csrf_token %} -

Select the scan type

-
- {% if custom_engine_count == 0 %} - - {% endif %} - - {% include "startScan/_items/scanEngine_select.html" %} -
- -
-
-
-
-
+ {% include "startScan/_items/start_scan_wizard.html" %} {% endblock main_content %} {% block page_level_script %} + + + {% endblock page_level_script %} diff --git a/web/startScan/templates/startScan/start_scan_ui.html b/web/startScan/templates/startScan/start_scan_ui.html index 181a40400..296a6c5f6 100644 --- a/web/startScan/templates/startScan/start_scan_ui.html +++ b/web/startScan/templates/startScan/start_scan_ui.html @@ -19,148 +19,16 @@ {% endblock breadcrumb_title %} {% block page_title %} -Initiating scan for {{domain.name}} + Initiating scan for {{domain.name}} {% endblock page_title %} {% block main_content %} -
-
-
-
-
- {% csrf_token %} -
-

Choose Scan Engine

-
-

Select Scan Engine

- {% if custom_engine_count == 0 %} - - {% endif %} - {% include "startScan/_items/scanEngine_select.html" %} -
-

Import/Ignore Subdomains

-
-
-

Import Subdomains(Optional)

- You can import subdomains for {{domain.name}} using your private recon tools. -
- - - -
-
-

Out of Scope Subdomains(Optional)

- You can import subdomains for {{domain.name}} using your private recon tools. -
- - - -
-
- -

URL Scope and Exclusions

-
-
-

Starting Point URL (Optional)

- - - Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains. -
- If a path is provided (e.g., /home), the scan will focus only on that path and its subpaths, - skipping subdomain scanning. For example, entering '/home' for {{domain.name}} will scan - https://{{domain.name}}/home, but not other parts of {{domain.name}} or its subdomains. - -
-
-

Excluded Paths (Optional)

- - - Enter paths or regex patterns to exclude from the scan. Type a path or pattern and press Enter to add it. - Supports both exact path matching and regex patterns. Examples:
- • /admin excludes paths starting with '/admin'
- • /images/.*\.jpg excludes all .jpg files in the images directory
- • /static/(?:css|js)/ excludes all contents of /static/css/ and /static/js/
- Common exclusions:
- • Static assets: /images/.*, /css/.*
-
- Note: Use regex patterns carefully. While exclusions can speed up scans, be cautious not to exclude critical areas that may contain vulnerabilities. Test your patterns to ensure they match as intended. - -
-
- -
-
-
-
+ {% include "startScan/_items/start_scan_wizard.html" %} {% endblock main_content %} {% block page_level_script %} - + {% endblock page_level_script %} diff --git a/web/startScan/views.py b/web/startScan/views.py index 37d7d6858..6cb33db49 100644 --- a/web/startScan/views.py +++ b/web/startScan/views.py @@ -300,8 +300,8 @@ def start_scan_ui(request, slug, domain_id): return HttpResponseRedirect(reverse('scan_history', kwargs={'slug': slug})) # GET request - engine = EngineType.objects.order_by('engine_name') - custom_engine_count = ( + engines = EngineType.objects.order_by('engine_name') + custom_engines_count = ( EngineType.objects .filter(default_engine=False) .count() @@ -310,8 +310,8 @@ def start_scan_ui(request, slug, domain_id): context = { 'scan_history_active': 'active', 'domain': domain, - 'engines': engine, - 'custom_engine_count': custom_engine_count, + 'engines': engines, + 'custom_engines_count': custom_engines_count, 'excluded_paths': excluded_paths } return render(request, 'startScan/start_scan_ui.html', context) @@ -325,11 +325,20 @@ def start_multiple_scan(request, slug): # if scan mode is available, then start the scan # get engine type engine_id = request.POST['scan_mode'] - list_of_domains = request.POST['list_of_domain_id'] + list_of_domain_ids = request.POST['domain_ids'] + subdomains_in = request.POST['importSubdomainTextArea'].split() + subdomains_in = [s.rstrip() for s in subdomains_in if s] + subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split() + subdomains_out = [s.rstrip() for s in subdomains_out if s] + starting_point_url = request.POST['startingPointUrl'].strip() + excluded_paths = request.POST['excludedPaths'] # string separated by , + + # split excluded paths by , + excluded_paths = [path.strip() for path in excluded_paths.split(',')] grouped_scans = [] - for domain_id in list_of_domains.split(","): + for domain_id in list_of_domain_ids.split(","): # Start the celery task scan_history_id = create_scan_object( host_id=domain_id, @@ -344,10 +353,11 @@ def start_multiple_scan(request, slug): 'engine_id': engine_id, 'scan_type': LIVE_SCAN, 'results_dir': '/usr/src/scan_results', - 'initiated_by_id': request.user.id - # TODO: Add this to multiple scan view - # 'imported_subdomains': subdomains_in, - # 'out_of_scope_subdomains': subdomains_out + 'initiated_by_id': request.user.id, + 'imported_subdomains': subdomains_in, + 'out_of_scope_subdomains': subdomains_out, + 'starting_point_url': starting_point_url, + 'excluded_paths': excluded_paths, } _scan_task = initiate_scan.si(**kwargs) @@ -383,12 +393,14 @@ def start_multiple_scan(request, slug): .filter(default_engine=False) .count() ) + excluded_paths = ','.join(DEFAULT_EXCLUDED_PATHS) context = { 'scan_history_active': 'active', 'engines': engines, 'domain_list': list_of_domain_name, 'domain_ids': domain_ids, - 'custom_engine_count': custom_engine_count + 'custom_engine_count': custom_engine_count, + 'excluded_paths': excluded_paths } return render(request, 'startScan/start_multiple_scan_ui.html', context) diff --git a/web/static/custom/start-scan-wizard-init.js b/web/static/custom/start-scan-wizard-init.js new file mode 100644 index 000000000..ee1ce61db --- /dev/null +++ b/web/static/custom/start-scan-wizard-init.js @@ -0,0 +1,59 @@ +var buttonEnabled = true; +var globalTimeout = 0; + +$("#select_engine").steps({ + headerTag: "h4", + bodyTag: "div", + transitionEffect: "slide", + cssClass: "pill wizard", + enableKeyNavigation: false, + onStepChanging: updateButton, + labels: { finish: "Start Scan" }, + onInit: function (event, current) { + $(".actions ul li:nth-child(3) a").attr( + "onclick", + `$(this).closest('form').submit()` + ); + }, +}); +$("input[type=radio][name=scan_mode]").change(initTimer).keyup(initTimer); +disableNext(); +$('a[role="menuitem"]').addClass("text-white"); + +function initTimer() { + if (globalTimeout) clearTimeout(globalTimeout); + globalTimeout = setTimeout(updateButton, 400); +} +function disableNext() { + var nextButton = $(".actions ul li:nth-child(2) a"); + nextButton.attr("href", "#"); + buttonEnabled = $(".actions ul li:nth-child(2)") + .addClass("disabled") + .attr("aria-disabled", "true"); +} + +function enableNext() { + var nextButton = $(".actions ul li:nth-child(2) a"); + nextButton.attr("href", "#next"); + buttonEnabled = $(".actions ul li:nth-child(2)") + .removeClass("disabled") + .attr("aria-disabled", "false"); +} + +function updateButton() { + $('a[role="menuitem"]').addClass("btn btn-primary waves-effect waves-light"); + var text = $("input[type=radio][name=scan_mode]").val(); + if (text === "") { + disableNext(); + return false; + } else { + enableNext(); + return true; + } +} + +$("#excludedPaths").selectize({ + persist: false, + createOnBlur: true, + create: true, +}); From dcf264d206317a5e31e2da10163a19d1674fafc3 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 7 Aug 2024 22:31:38 +0530 Subject: [PATCH 070/211] feat: add feature for import/export,url filter for organization scan --- .../templates/organization/start_scan.html | 36 ++++--------------- web/startScan/views.py | 23 +++++++++--- 2 files changed, 25 insertions(+), 34 deletions(-) diff --git a/web/startScan/templates/organization/start_scan.html b/web/startScan/templates/organization/start_scan.html index 555ca9fe0..e968ad5fb 100644 --- a/web/startScan/templates/organization/start_scan.html +++ b/web/startScan/templates/organization/start_scan.html @@ -7,11 +7,13 @@ {% block custom_js_css_link %} + + {% endblock custom_js_css_link %} {% block breadcrumb_title %} -
  • Organization
    • +
  • Organization
  • Initiate Scan
  • {{organization.name}}
    • {% endblock breadcrumb_title %} @@ -21,36 +23,12 @@ {% endblock page_title %} {% block main_content %} -
    -
    -
    -
    -

    {{ domain_list|length }} Domains associated with organization {{organization.name}}

    - {% for domain in domain_list %} - {{domain.name}} - {% endfor %} -
    - {% csrf_token %} -
    -
    -

    Select Scan Engine

    - {% if custom_engine_count == 0 %} - - {% endif %} - {% include "startScan/_items/scanEngine_select.html" %} - -
    - - -
    -
    -
    -
    + {% include "startScan/_items/start_scan_wizard.html" %} {% endblock main_content %} {% block page_level_script %} + + + {% endblock page_level_script %} diff --git a/web/startScan/views.py b/web/startScan/views.py index 6cb33db49..386df7dc2 100644 --- a/web/startScan/views.py +++ b/web/startScan/views.py @@ -332,7 +332,6 @@ def start_multiple_scan(request, slug): subdomains_out = [s.rstrip() for s in subdomains_out if s] starting_point_url = request.POST['startingPointUrl'].strip() excluded_paths = request.POST['excludedPaths'] # string separated by , - # split excluded paths by , excluded_paths = [path.strip() for path in excluded_paths.split(',')] @@ -732,6 +731,15 @@ def start_organization_scan(request, id, slug): if request.method == "POST": engine_id = request.POST['scan_mode'] + subdomains_in = request.POST['importSubdomainTextArea'].split() + subdomains_in = [s.rstrip() for s in subdomains_in if s] + subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split() + subdomains_out = [s.rstrip() for s in subdomains_out if s] + starting_point_url = request.POST['startingPointUrl'].strip() + excluded_paths = request.POST['excludedPaths'] # string separated by , + # split excluded paths by , + excluded_paths = [path.strip() for path in excluded_paths.split(',')] + # Start Celery task for each organization's domains for domain in organization.get_domains(): scan_history_id = create_scan_object( @@ -748,9 +756,10 @@ def start_organization_scan(request, id, slug): 'scan_type': LIVE_SCAN, 'results_dir': '/usr/src/scan_results', 'initiated_by_id': request.user.id, - # TODO: Add this to multiple scan view - # 'imported_subdomains': subdomains_in, - # 'out_of_scope_subdomains': subdomains_out + 'imported_subdomains': subdomains_in, + 'out_of_scope_subdomains': subdomains_out, + 'starting_point_url': starting_point_url, + 'excluded_paths': excluded_paths, } initiate_scan.apply_async(kwargs=kwargs) scan.save() @@ -768,13 +777,17 @@ def start_organization_scan(request, id, slug): engine = EngineType.objects.order_by('engine_name') custom_engine_count = EngineType.objects.filter(default_engine=False).count() domain_list = organization.get_domains() + excluded_paths = ','.join(DEFAULT_EXCLUDED_PATHS) + context = { 'organization_data_active': 'true', 'list_organization_li': 'active', 'organization': organization, 'engines': engine, 'domain_list': domain_list, - 'custom_engine_count': custom_engine_count} + 'custom_engine_count': custom_engine_count, + 'excluded_paths': excluded_paths + } return render(request, 'organization/start_scan.html', context) From 5162982c1bafbb05e7a6e4ea4d1471ea6ba653d9 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Thu, 8 Aug 2024 07:36:10 +0530 Subject: [PATCH 071/211] fix ui for schedule scan --- .../startScan/js/schedule-scan-wizard.js | 89 +++++++++ .../startScan/js}/start-scan-wizard-init.js | 0 .../organization/schedule_scan_ui.html | 161 +--------------- .../templates/organization/start_scan.html | 2 +- ..._select.html => scanengine_accordion.html} | 0 .../_items/schedule_scan_wizard.html | 123 ++++++++++++ .../startScan/_items/start_scan_wizard.html | 2 +- .../templates/startScan/schedule_scan_ui.html | 180 +----------------- .../startScan/start_multiple_scan_ui.html | 2 +- .../templates/startScan/start_scan_ui.html | 4 +- 10 files changed, 225 insertions(+), 338 deletions(-) create mode 100644 web/startScan/static/startScan/js/schedule-scan-wizard.js rename web/{static/custom => startScan/static/startScan/js}/start-scan-wizard-init.js (100%) rename web/startScan/templates/startScan/_items/{scanEngine_select.html => scanengine_accordion.html} (100%) create mode 100644 web/startScan/templates/startScan/_items/schedule_scan_wizard.html diff --git a/web/startScan/static/startScan/js/schedule-scan-wizard.js b/web/startScan/static/startScan/js/schedule-scan-wizard.js new file mode 100644 index 000000000..1f2a8ca8c --- /dev/null +++ b/web/startScan/static/startScan/js/schedule-scan-wizard.js @@ -0,0 +1,89 @@ +function schedulerChanged(selectObject) { + selectedValue = selectObject.value; + if (selectedValue == "periodic") { + var clockedDiv = document.getElementById("clocked-div"); + clockedDiv.classList.remove("show"); + clockedDiv.classList.remove("active"); + var periodicDiv = document.getElementById("periodic-div"); + periodicDiv.classList.add("show"); + periodicDiv.classList.add("active"); + } else if (selectedValue == "clocked") { + var periodicDiv = document.getElementById("periodic-div"); + periodicDiv.classList.remove("show"); + periodicDiv.classList.remove("active"); + var clockedDiv = document.getElementById("clocked-div"); + clockedDiv.classList.add("show"); + clockedDiv.classList.add("active"); + } +} + +var buttonEnabled = true; +var globalTimeout = 0; + +function disableNext() { + var nextButton = $(".actions ul li:nth-child(2) a"); + nextButton.attr("href", "#"); + buttonEnabled = $(".actions ul li:nth-child(2)") + .addClass("disabled") + .attr("aria-disabled", "true"); +} + +function enableNext() { + var nextButton = $(".actions ul li:nth-child(2) a"); + nextButton.attr("href", "#next"); + buttonEnabled = $(".actions ul li:nth-child(2)") + .removeClass("disabled") + .attr("aria-disabled", "false"); +} + +function updateButton() { + var text = $("input[type=radio][name=scan_mode]").val(); + if (text === "") { + disableNext(); + return false; + } else { + enableNext(); + return true; + } +} + +function initTimer() { + if (globalTimeout) clearTimeout(globalTimeout); + globalTimeout = setTimeout(updateButton, 400); +} + +$("#schedule_scan_steps").steps({ + headerTag: "h3", + bodyTag: "div", + transitionEffect: "slide", + cssClass: "pill wizard", + enableKeyNavigation: false, + onStepChanging: updateButton, + labels: { finish: "Start Scan" }, + onInit: function (event, current) { + $('a[role="menuitem"]').addClass("text-white"); + $(".actions ul li:nth-child(3) a").attr( + "onclick", + `$(this).closest('form').submit()` + ); + flatpickr(document.getElementById("clockedTime"), { + enableTime: true, + dateFormat: "Y-m-d H:i", + }); + // $(".basic").select2({ + // minimumResultsForSearch: -1 + // }); + }, + onStepChanged: function (event, currentIndex, priorIndex) { + if (currentIndex == 1) { + $("input[type=radio][name=scan_mode]").change(initTimer).keyup(initTimer); + disableNext(); + } + }, +}); + +$("#excludedPaths").selectize({ + persist: false, + createOnBlur: true, + create: true, +}); diff --git a/web/static/custom/start-scan-wizard-init.js b/web/startScan/static/startScan/js/start-scan-wizard-init.js similarity index 100% rename from web/static/custom/start-scan-wizard-init.js rename to web/startScan/static/startScan/js/start-scan-wizard-init.js diff --git a/web/startScan/templates/organization/schedule_scan_ui.html b/web/startScan/templates/organization/schedule_scan_ui.html index 931c0ae46..997f5b8d3 100644 --- a/web/startScan/templates/organization/schedule_scan_ui.html +++ b/web/startScan/templates/organization/schedule_scan_ui.html @@ -10,6 +10,7 @@ + {% endblock custom_js_css_link %} {% block breadcrumb_title %} @@ -23,167 +24,13 @@ {% endblock page_title %} {% block main_content %} -
    -
    -
    -
    -

    {{ domain_list|length }} Domains associated with organization {{organization.name}}

    - {% for domain in domain_list %} - {{domain.name}} - {% endfor %} -
    - {% csrf_token %} -
    -

    Choose the scheduler

    -
    -
    - -
    -
    -
    -
    -
    Run scan every
    -
    -
    - -
    -
    - -
    -
    -
    -
    -
    -
    -
    Run scan exactly at
    -
    -
    -
    - -
    -
    -
    -
    -
    -
    -
    -

    Choose Scan Engine

    -
    -

    Select Scan Engine

    - {% if custom_engine_count == 0 %} - - {% endif %} - {% include "startScan/_items/scanEngine_select.html" %} -
    -
    -
    -
    -
    -
    -
    + {% include "startScan/_items/schedule_scan_wizard.html" %} {% endblock main_content %} {% block page_level_script %} - + + {% endblock page_level_script %} diff --git a/web/startScan/templates/organization/start_scan.html b/web/startScan/templates/organization/start_scan.html index e968ad5fb..8b388fa90 100644 --- a/web/startScan/templates/organization/start_scan.html +++ b/web/startScan/templates/organization/start_scan.html @@ -30,5 +30,5 @@ {% block page_level_script %} - + {% endblock page_level_script %} diff --git a/web/startScan/templates/startScan/_items/scanEngine_select.html b/web/startScan/templates/startScan/_items/scanengine_accordion.html similarity index 100% rename from web/startScan/templates/startScan/_items/scanEngine_select.html rename to web/startScan/templates/startScan/_items/scanengine_accordion.html diff --git a/web/startScan/templates/startScan/_items/schedule_scan_wizard.html b/web/startScan/templates/startScan/_items/schedule_scan_wizard.html new file mode 100644 index 000000000..d2dbf8816 --- /dev/null +++ b/web/startScan/templates/startScan/_items/schedule_scan_wizard.html @@ -0,0 +1,123 @@ +
    +
    +
    +
    + {% if domain_list %} +

    {{ domain_list|length }} Domains associated with organization {{organization.name}}

    {% for domain in domain_list %} + {{domain.name}} {% endfor %} +
    +
    {% endif %} +
    + {% csrf_token %} +
    +

    Choose the scheduler

    +
    +
    + +
    +
    +
    +
    +
    Run scan every
    +
    +
    + +
    +
    + +
    +
    +
    +
    +
    +
    +
    Run scan exactly at
    +
    +
    +
    + + +
    +
    +
    +
    +
    +
    +
    +

    Choose Scan Engine

    +
    +

    Select Scan Engine

    {% if custom_engine_count == 0 %} + + {% endif %} {% include "startScan/_items/scanengine_accordion.html" %} +
    +

    Import/Ignore Subdomains

    +
    +
    +

    Import Subdomains(Optional)

    + You can import subdomains for {{domain.name}} using your private recon tools. +
    + + + +
    +
    +

    Out of Scope Subdomains(Optional)

    + You can import subdomains for {{domain.name}} using your private recon tools. +
    + + + +
    +
    +

    URL Scope and Exclusions

    +
    +
    +

    Starting Point URL (Optional)

    + + + Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains. +
    + If a path is provided (e.g., /home), the scan will focus only on that path and its subpaths, + skipping subdomain scanning. For example, entering '/home' for {{domain.name}} will scan + https://{{domain.name}}/home, but not other parts of {{domain.name}} or its subdomains. +     +
    +
    +

    Excluded Paths (Optional)

    + + + Enter paths or regex patterns to exclude from the scan. Type a path or pattern and press Enter to add it. + Supports both exact path matching and regex patterns. Examples:
    + • /admin excludes paths starting with '/admin'
    + • /images/.*\.jpg excludes all .jpg files in the images directory
    + • /static/(?:css|js)/ excludes all contents of /static/css/ and /static/js/
    + Common exclusions:
    + • Static assets: /images/.*, /css/.*
    +
    + Note: Use regex patterns carefully. While exclusions can speed up scans, be cautious not to exclude critical + areas that may contain vulnerabilities. Test your patterns to ensure they match as intended. +     +
    +
    +
    +
    +
    +
    +
    +
    \ No newline at end of file diff --git a/web/startScan/templates/startScan/_items/start_scan_wizard.html b/web/startScan/templates/startScan/_items/start_scan_wizard.html index cecb29f75..1f2af3eda 100644 --- a/web/startScan/templates/startScan/_items/start_scan_wizard.html +++ b/web/startScan/templates/startScan/_items/start_scan_wizard.html @@ -22,7 +22,7 @@

    Select Scan Engine

    Create Custom Scan Engine
    {% endif %} - {% include "startScan/_items/scanEngine_select.html" %} + {% include "startScan/_items/scanengine_accordion.html" %}

    Import/Ignore Subdomains

    diff --git a/web/startScan/templates/startScan/schedule_scan_ui.html b/web/startScan/templates/startScan/schedule_scan_ui.html index 3ffbc1f6b..662ef9d3b 100644 --- a/web/startScan/templates/startScan/schedule_scan_ui.html +++ b/web/startScan/templates/startScan/schedule_scan_ui.html @@ -11,6 +11,7 @@ + {% endblock custom_js_css_link %} {% block breadcrumb_title %} @@ -24,186 +25,13 @@ {% endblock page_title %} {% block main_content %} -
    -
    -
    -
    -
    - {% csrf_token %} -
    -

    Choose the scheduler

    -
    -
    - -
    -
    -
    -
    -
    Run scan every
    -
    -
    - -
    -
    - -
    -
    -
    -
    -
    -
    -
    Run scan exactly at
    -
    -
    -
    - - -
    -
    -
    -
    -
    -
    -
    -

    Choose Scan Engine

    -
    -

    Select Scan Engine

    - {% if custom_engine_count == 0 %} - - {% endif %} - {% include "startScan/_items/scanEngine_select.html" %} -
    -

    Import/Ignore Subdomains

    -
    -
    -

    Import Subdomains(Optional)

    - You can import subdomains for {{domain.name}} using your private recon tools. -
    - - - -
    -
    -

    Out of Scope Subdomains(Optional)

    - You can import subdomains for {{domain.name}} using your private recon tools. -
    - - - -
    -
    -
    -
    -
    -
    -
    -
    + {% include "startScan/_items/schedule_scan_wizard.html" %} {% endblock main_content %} {% block page_level_script %} - + + {% endblock page_level_script %} diff --git a/web/startScan/templates/startScan/start_multiple_scan_ui.html b/web/startScan/templates/startScan/start_multiple_scan_ui.html index 0230b8f38..497bf7cd5 100644 --- a/web/startScan/templates/startScan/start_multiple_scan_ui.html +++ b/web/startScan/templates/startScan/start_multiple_scan_ui.html @@ -30,5 +30,5 @@ {% block page_level_script %} - + {% endblock page_level_script %} diff --git a/web/startScan/templates/startScan/start_scan_ui.html b/web/startScan/templates/startScan/start_scan_ui.html index 296a6c5f6..8260aad25 100644 --- a/web/startScan/templates/startScan/start_scan_ui.html +++ b/web/startScan/templates/startScan/start_scan_ui.html @@ -19,7 +19,7 @@ {% endblock breadcrumb_title %} {% block page_title %} - Initiating scan for {{domain.name}} +Initiating scan for {{domain.name}} {% endblock page_title %} {% block main_content %} @@ -30,5 +30,5 @@ {% block page_level_script %} - + {% endblock page_level_script %} From b8764ca419ba61e3a6099b95bb3e6b7a7d53d741 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Thu, 8 Aug 2024 07:44:38 +0530 Subject: [PATCH 072/211] feat: add starting point URL and excluded paths to schedule scan --- web/startScan/views.py | 45 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/web/startScan/views.py b/web/startScan/views.py index 386df7dc2..94b9ef21f 100644 --- a/web/startScan/views.py +++ b/web/startScan/views.py @@ -250,7 +250,7 @@ def all_endpoints(request, slug): } return render(request, 'startScan/endpoints.html', context) - +@has_permission_decorator(PERM_INITATE_SCANS_SUBSCANS, redirect_url=FOUR_OH_FOUR_URL) def start_scan_ui(request, slug, domain_id): domain = get_object_or_404(Domain, id=domain_id) if request.method == "POST": @@ -261,7 +261,6 @@ def start_scan_ui(request, slug, domain_id): subdomains_out = [s.rstrip() for s in subdomains_out if s] starting_point_url = request.POST['startingPointUrl'].strip() excluded_paths = request.POST['excludedPaths'] # string separated by , - # split excluded paths by , excluded_paths = [path.strip() for path in excluded_paths.split(',')] @@ -563,6 +562,10 @@ def schedule_scan(request, host_id, slug): subdomains_in = [s.rstrip() for s in subdomains_in if s] subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split() subdomains_out = [s.rstrip() for s in subdomains_out if s] + starting_point_url = request.POST['startingPointUrl'].strip() + excluded_paths = request.POST['excludedPaths'] # string separated by , + # split excluded paths by , + excluded_paths = [path.strip() for path in excluded_paths.split(',')] # Get engine type engine = get_object_or_404(EngineType, id=engine_type) @@ -593,6 +596,8 @@ def schedule_scan(request, host_id, slug): 'scan_type': SCHEDULED_SCAN, 'imported_subdomains': subdomains_in, 'out_of_scope_subdomains': subdomains_out, + 'starting_point_url': starting_point_url, + 'excluded_paths': excluded_paths, 'initiated_by_id': request.user.id } PeriodicTask.objects.create( @@ -612,6 +617,8 @@ def schedule_scan(request, host_id, slug): 'scan_type': SCHEDULED_SCAN, 'imported_subdomains': subdomains_in, 'out_of_scope_subdomains': subdomains_out, + 'starting_point_url': starting_point_url, + 'excluded_paths': excluded_paths, 'initiated_by_id': request.user.id } PeriodicTask.objects.create( @@ -635,11 +642,14 @@ def schedule_scan(request, host_id, slug): .filter(default_engine=False) .count() ) + excluded_paths = ','.join(DEFAULT_EXCLUDED_PATHS) context = { 'scan_history_active': 'active', 'domain': domain, 'engines': engines, - 'custom_engine_count': custom_engine_count} + 'custom_engine_count': custom_engine_count, + 'excluded_paths': excluded_paths + } return render(request, 'startScan/schedule_scan_ui.html', context) @@ -797,7 +807,18 @@ def schedule_organization_scan(request, slug, id): if request.method == "POST": engine_type = int(request.POST['scan_mode']) engine = get_object_or_404(EngineType, id=engine_type) + + # post vars scheduled_mode = request.POST['scheduled_mode'] + subdomains_in = request.POST['importSubdomainTextArea'].split() + subdomains_in = [s.rstrip() for s in subdomains_in if s] + subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split() + subdomains_out = [s.rstrip() for s in subdomains_out if s] + starting_point_url = request.POST['startingPointUrl'].strip() + excluded_paths = request.POST['excludedPaths'] # string separated by , + # split excluded paths by , + excluded_paths = [path.strip() for path in excluded_paths.split(',')] + for domain in organization.get_domains(): timestr = str(datetime.strftime(timezone.now(), '%Y_%m_%d_%H_%M_%S')) task_name = f'{engine.engine_name} for {domain.name}: {timestr}' @@ -828,8 +849,11 @@ def schedule_organization_scan(request, slug, id): 'engine_id': engine.id, 'scan_history_id': 0, 'scan_type': SCHEDULED_SCAN, - 'imported_subdomains': None, - 'initiated_by_id': request.user.id + 'initiated_by_id': request.user.id, + 'imported_subdomains': subdomains_in, + 'out_of_scope_subdomains': subdomains_out, + 'starting_point_url': starting_point_url, + 'excluded_paths': excluded_paths, }) PeriodicTask.objects.create( interval=schedule, @@ -849,8 +873,11 @@ def schedule_organization_scan(request, slug, id): 'engine_id': engine.id, 'scan_history_id': 0, 'scan_type': LIVE_SCAN, - 'imported_subdomains': None, - 'initiated_by_id': request.user.id + 'initiated_by_id': request.user.id, + 'imported_subdomains': subdomains_in, + 'out_of_scope_subdomains': subdomains_out, + 'starting_point_url': starting_point_url, + 'excluded_paths': excluded_paths, }) PeriodicTask.objects.create(clocked=clock, one_off=True, @@ -871,12 +898,14 @@ def schedule_organization_scan(request, slug, id): # GET request engine = EngineType.objects custom_engine_count = EngineType.objects.filter(default_engine=False).count() + excluded_paths = ','.join(DEFAULT_EXCLUDED_PATHS) context = { 'scan_history_active': 'active', 'organization': organization, 'domain_list': organization.get_domains(), 'engines': engine, - 'custom_engine_count': custom_engine_count + 'custom_engine_count': custom_engine_count, + 'excluded_paths': excluded_paths } return render(request, 'organization/schedule_scan_ui.html', context) From f237f43d952cb506c0ca8500b11e8c6535285c53 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Thu, 8 Aug 2024 07:45:53 +0530 Subject: [PATCH 073/211] fix: update label for finish button in schedule scan wizard --- web/startScan/static/startScan/js/schedule-scan-wizard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/startScan/static/startScan/js/schedule-scan-wizard.js b/web/startScan/static/startScan/js/schedule-scan-wizard.js index 1f2a8ca8c..ac67c8e96 100644 --- a/web/startScan/static/startScan/js/schedule-scan-wizard.js +++ b/web/startScan/static/startScan/js/schedule-scan-wizard.js @@ -59,7 +59,7 @@ $("#schedule_scan_steps").steps({ cssClass: "pill wizard", enableKeyNavigation: false, onStepChanging: updateButton, - labels: { finish: "Start Scan" }, + labels: { finish: "Schedule Scan" }, onInit: function (event, current) { $('a[role="menuitem"]').addClass("text-white"); $(".actions ul li:nth-child(3) a").attr( From 413825cf15619199b381d92545e0fa13b6549724 Mon Sep 17 00:00:00 2001 From: Damian Husted <43425216+DamianHusted@users.noreply.github.com> Date: Tue, 20 Aug 2024 09:29:27 +1200 Subject: [PATCH 074/211] Update of template.html with conditional statement TLDR; Added boolean operator to report template. This mitigates having a listed vulnerability with the "in" preposition, when no URL/URI is present. -- When generating reports, if a reported vulnerability does not have a specific URL/URI, the report will generate according to the following syntax (regardless of whether the URL is present or not): in This often results in a report with vulnerability headings appearing incomplete like: WAF detected in This change updates template.html file with a conditional check, avoiding this issue. The vulnerability headings without a URL omit the "in" preposition. --- web/templates/report/template.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web/templates/report/template.html b/web/templates/report/template.html index 9f353658b..4a7cf8b35 100644 --- a/web/templates/report/template.html +++ b/web/templates/report/template.html @@ -939,7 +939,10 @@

    Vulnerability Breakdown by Severity

    {{vulnerability.name}} -
    in {{vulnerabilities.grouper}}     + {% if vulnerabilities.grouper %} +
    in {{vulnerabilities.grouper}} + {% endif %} + {% if vulnerability.severity == -1 %} Unknown
    From b138ceffb7e7d07e339f822e4e8ed1b3124264f9 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 21 Aug 2024 11:23:27 +0530 Subject: [PATCH 075/211] Add util class to support both regex and string out of scope subdomains --- web/reNgine/utilities.py | 47 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/web/reNgine/utilities.py b/web/reNgine/utilities.py index c63fef975..a3101625f 100644 --- a/web/reNgine/utilities.py +++ b/web/reNgine/utilities.py @@ -1,3 +1,4 @@ +import re import os import validators @@ -113,4 +114,48 @@ def is_valid_url(url, validate_only_http_scheme=True): if validate_only_http_scheme: return url.startswith('http://') or url.startswith('https://') return True - return False \ No newline at end of file + return False + + +class SubdomainScopeChecker: + """ + SubdomainScopeChecker is a utility class to check if a subdomain is in scope or not. + it supports both regex and string matching. + """ + + def __init__(self, patterns): + self.regex_patterns = set() + self.plain_patterns = set() + self.load_patterns(patterns) + + def load_patterns(self, patterns): + """ + Load patterns into the checker. + + Args: + patterns (list): List of patterns to load. + Returns: + None + """ + for pattern in patterns: + # skip empty patterns + if not pattern: + continue + try: + self.regex_patterns.add(re.compile(pattern, re.IGNORECASE)) + except re.error: + self.plain_patterns.add(pattern.lower()) + + def is_out_of_scope(self, subdomain): + """ + Check if a subdomain is out of scope. + + Args: + subdomain (str): The subdomain to check. + Returns: + bool: True if the subdomain is out of scope, False otherwise. + """ + subdomain = subdomain.lower() # though we wont encounter this, but just in case + if subdomain in self.plain_patterns: + return True + return any(pattern.search(subdomain) for pattern in self.regex_patterns) From d44d043ba96e368a2a55db8ead88890c16ba187b Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 21 Aug 2024 11:23:49 +0530 Subject: [PATCH 076/211] add out of scope regex in ui --- .../startScan/_items/start_scan_wizard.html | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/web/startScan/templates/startScan/_items/start_scan_wizard.html b/web/startScan/templates/startScan/_items/start_scan_wizard.html index 1f2af3eda..cd0c8097f 100644 --- a/web/startScan/templates/startScan/_items/start_scan_wizard.html +++ b/web/startScan/templates/startScan/_items/start_scan_wizard.html @@ -28,23 +28,24 @@

    Import/Ignore Subdomains

    Import Subdomains(Optional)

    - You can import subdomains for {{domain.name}} using your private recon tools. +

    You can import subdomains for {{domain.name}} discovered through your private reconnaissance tools.


    -

    Out of Scope Subdomains(Optional)

    - You can import subdomains for {{domain.name}} using your private recon tools. -
    -
    From 9a285758024521fca5c4d92f6fd305fe66861fce Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 21 Aug 2024 11:23:59 +0530 Subject: [PATCH 077/211] Refactor subdomain out-of-scope checking logic --- web/reNgine/tasks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 33ecbae04..2e94b4daf 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -420,6 +420,7 @@ def subdomain_discovery( custom_subdomain_tools = [tool.name.lower() for tool in InstalledExternalTool.objects.filter(is_default=False).filter(is_subdomain_gathering=True)] send_subdomain_changes, send_interesting = False, False notif = Notification.objects.first() + subdomain_scope_checker = SubdomainScopeChecker(self.out_of_scope_subdomains) if notif: send_subdomain_changes = notif.send_subdomain_changes_notif send_interesting = notif.send_interesting_notif @@ -565,7 +566,7 @@ def subdomain_discovery( if valid_url: subdomain_name = urlparse(subdomain_name).netloc - if subdomain_name in self.out_of_scope_subdomains: + if subdomain_scope_checker.is_out_of_scope(subdomain_name): logger.error(f'Subdomain {subdomain_name} is out of scope. Skipping.') continue @@ -4421,6 +4422,7 @@ def save_subdomain(subdomain_name, ctx={}): scan_id = ctx.get('scan_history_id') subscan_id = ctx.get('subscan_id') out_of_scope_subdomains = ctx.get('out_of_scope_subdomains', []) + subdomain_checker = SubdomainScopeChecker(out_of_scope_subdomains) valid_domain = ( validators.domain(subdomain_name) or validators.ipv4(subdomain_name) or @@ -4430,7 +4432,7 @@ def save_subdomain(subdomain_name, ctx={}): logger.error(f'{subdomain_name} is not an invalid domain. Skipping.') return None, False - if subdomain_name in out_of_scope_subdomains: + if subdomain_checker.is_out_of_scope(subdomain_name): logger.error(f'{subdomain_name} is out-of-scope. Skipping.') return None, False From 4371ea341700f11b75ee2c055958fcdd67bc04f6 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 21 Aug 2024 12:07:41 +0530 Subject: [PATCH 078/211] feat: Add endpoint to delete multiple scheduled scans This commit adds a new endpoint `delete_multiple_scheduled_scans` to the `startScan` module. This endpoint allows users to delete multiple scheduled scans at once. --- .../startScan/schedule_scan_list.html | 107 ++++++++++++++++-- web/startScan/urls.py | 4 + web/startScan/views.py | 16 +++ 3 files changed, 119 insertions(+), 8 deletions(-) diff --git a/web/startScan/templates/startScan/schedule_scan_list.html b/web/startScan/templates/startScan/schedule_scan_list.html index 012b5427d..f575a27bc 100644 --- a/web/startScan/templates/startScan/schedule_scan_list.html +++ b/web/startScan/templates/startScan/schedule_scan_list.html @@ -23,9 +23,29 @@
    +
    +
    +
    +
    + {% if user|can:'initiate_scans_subscans' %} +
    + Delete Multiple Scheduled Scans +
    + {% endif %} +
    +
    +
    +
    +
    +
    +
    +
    +
    + {% csrf_token %} + @@ -38,6 +58,7 @@ {% for task in scheduled_tasks %} + {% with task_name=task.name|split:":" %} {% endwith %} @@ -79,6 +100,7 @@ {% endfor %}
    Serial Number Description Frequency Last Run
    {{ task.id }}{{ task_name.0 }}
    +
    @@ -88,20 +110,89 @@ // var e; $(document).ready(function() { var scheduled_scan_table = $('#scheduled_scan_table').DataTable({ + headerCallback: function(e, a, t, n, s) { + e.getElementsByTagName("th")[0].innerHTML='
    \nc
    \n' + }, "oLanguage": { - "oPaginate": { "sPrevious": '', "sNext": '' }, - "sInfo": "Showing page _PAGE_ of _PAGES_", - "sSearch": '', - "sSearchPlaceholder": "Search...", - "sLengthMenu": "Results : _MENU_", + "oPaginate": { "sPrevious": '', "sNext": '' }, + "sInfo": "Showing page _PAGE_ of _PAGES_", + "sSearch": '', + "sSearchPlaceholder": "Search...", + "sLengthMenu": "Results : _MENU_", }, + "columnDefs":[ + { + "targets":0, "width":"20px", "className":"", "orderable":!1, render:function(e, a, t, n) { + return'
    \nc
    ' + }, + + } + ], "lengthMenu": [5, 10, 20, 30, 50, 100], - "pageLength": 10, + "pageLength": 20, + "order": [[3, 'desc']], "dom": "<'dt--top-section'<'row'<'col-12 col-sm-6 d-flex justify-content-sm-start justify-content-center mt-sm-0 mt-3'f><'col-12 col-sm-6 d-flex justify-content-sm-end justify-content-center'l>>>" + - "<'table-responsive'tr>" + - "<'dt--bottom-section d-sm-flex justify-content-sm-between text-center'<'dt--pages-count mb-sm-0 mb-3'i><'dt--pagination'p>>", + "<'table-responsive'tr>" + + "<'dt--bottom-section d-sm-flex justify-content-sm-between text-center'<'dt--pages-count mb-sm-0 mb-3'i><'dt--pagination'p>>", }); multiCheck(scheduled_scan_table); }); +function mainCheckBoxSelected(checkbox) { + if (checkbox.checked) { + $("#delete_multiple_scheduled_button").removeClass("disabled"); + $(".targets_checkbox").prop('checked', true); + } else { + $("#delete_multiple_scheduled_button").addClass("disabled"); + $(".targets_checkbox").prop('checked', false); + } +} + +function checkedCount() { + // this function will count the number of boxes checked + item = document.getElementsByClassName("targets_checkbox"); + count = 0; + for (var i = 0; i < item.length; i++) { + if (item[i].checked) { + count++; + } + } + return count; +} + +function toggleMultipleTargetButton() { + if (checkedCount() > 0) { + $("#delete_multiple_scheduled_button").removeClass("disabled"); + } else { + $("#delete_multiple_scheduled_button").addClass("disabled"); + } +} + +function deleteMultipleScheduledScan() { + if (!checkedCount()) { + swal({ + title: '', + text: "Oops! You haven't selected any scheduled scan to delete.", + type: 'error', + padding: '2em' + }) + } else { + // atleast one target is selected + swal.queue([{ + title: 'Are you sure you want to delete ' + checkedCount() + ' Scheduled Scans?', + text: "This action is irreversible.\nThis will stop all the selected scheduled scans.", + type: 'warning', + showCancelButton: true, + confirmButtonText: 'Delete', + padding: '2em', + showLoaderOnConfirm: true, + preConfirm: function() { + deleteForm = document.getElementById("scheduled_scan_history_form"); + deleteForm.action = "../delete/multiple/scheduled"; + deleteForm.submit(); + } + }]) + } +} + {% endblock page_level_script %} diff --git a/web/startScan/urls.py b/web/startScan/urls.py index 7ff6a57d0..9ad4ab722 100644 --- a/web/startScan/urls.py +++ b/web/startScan/urls.py @@ -106,6 +106,10 @@ '/delete/multiple', views.delete_scans, name='delete_multiple_scans'), + path( + '/delete/multiple/scheduled', + views.delete_scheduled_scans, + name='delete_multiple_scheduled_scans'), path( '/stop/multiple', views.stop_scans, diff --git a/web/startScan/views.py b/web/startScan/views.py index 94b9ef21f..de1c6a378 100644 --- a/web/startScan/views.py +++ b/web/startScan/views.py @@ -685,6 +685,22 @@ def delete_scheduled_task(request, id): return JsonResponse(messageData) +@has_permission_decorator(PERM_MODIFY_SCAN_RESULTS, redirect_url=FOUR_OH_FOUR_URL) +def delete_scheduled_scans(request, slug): + if request.method == "POST": + for key, value in request.POST.items(): + if key == 'csrfmiddlewaretoken': + continue + print(value) + scan = get_object_or_404(PeriodicTask, id=value) + scan.delete() + messages.add_message( + request, + messages.INFO, + 'Multiple scheduled scans successfully deleted!') + return HttpResponseRedirect(reverse('scheduled_scan_view', kwargs={'slug': slug})) + + @has_permission_decorator(PERM_MODIFY_SCAN_RESULTS, redirect_url=FOUR_OH_FOUR_URL) def change_scheduled_task_status(request, id): if request.method == 'POST': From 4c3302771ecc661790a3ec67072f53f48aceb163 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 21 Aug 2024 12:10:21 +0530 Subject: [PATCH 079/211] fix 404 exception and task key in checkbox values --- web/startScan/views.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/web/startScan/views.py b/web/startScan/views.py index de1c6a378..b664818fd 100644 --- a/web/startScan/views.py +++ b/web/startScan/views.py @@ -689,11 +689,13 @@ def delete_scheduled_task(request, id): def delete_scheduled_scans(request, slug): if request.method == "POST": for key, value in request.POST.items(): - if key == 'csrfmiddlewaretoken': + if 'task' in key or key == 'csrfmiddlewaretoken': continue - print(value) - scan = get_object_or_404(PeriodicTask, id=value) - scan.delete() + try: + scan = get_object_or_404(PeriodicTask, id=value) + scan.delete() + except Exception as e: + logger.error(e) messages.add_message( request, messages.INFO, From cbdeca7b788df58f221f1f502e13cecf2bd27b49 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 21 Aug 2024 12:14:16 +0530 Subject: [PATCH 080/211] Add loading indicator when deleting scheduled scans --- web/startScan/templates/startScan/schedule_scan_list.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/web/startScan/templates/startScan/schedule_scan_list.html b/web/startScan/templates/startScan/schedule_scan_list.html index f575a27bc..974490bb0 100644 --- a/web/startScan/templates/startScan/schedule_scan_list.html +++ b/web/startScan/templates/startScan/schedule_scan_list.html @@ -186,6 +186,14 @@ padding: '2em', showLoaderOnConfirm: true, preConfirm: function() { + Swal.fire({ + title: 'Deleting...', + text: 'Please wait while the scheduled scans are being deleted.', + icon: 'info', + allowOutsideClick: false, + allowEscapeKey: false, + showConfirmButton: false + }); deleteForm = document.getElementById("scheduled_scan_history_form"); deleteForm.action = "../delete/multiple/scheduled"; deleteForm.submit(); From ab11e525e559d15a3a5fbd9f8d886a355bf2dc99 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 21 Aug 2024 12:16:40 +0530 Subject: [PATCH 081/211] Improve UI for specifying out of scope subdomains in scheduled scans --- .../startScan/_items/schedule_scan_wizard.html | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/web/startScan/templates/startScan/_items/schedule_scan_wizard.html b/web/startScan/templates/startScan/_items/schedule_scan_wizard.html index d2dbf8816..7edf9d05a 100644 --- a/web/startScan/templates/startScan/_items/schedule_scan_wizard.html +++ b/web/startScan/templates/startScan/_items/schedule_scan_wizard.html @@ -75,14 +75,15 @@

    Import Subdomains(Optional)

    -

    Out of Scope Subdomains(Optional)

    - You can import subdomains for {{domain.name}} using your private recon tools. -
    -

    URL Scope and Exclusions

    From 763384a4d8adfe5983491803587c02eba0a460bf Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 21 Aug 2024 12:18:33 +0530 Subject: [PATCH 082/211] feat: Add loading indicator when deleting multiple scans --- web/startScan/templates/startScan/history.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/web/startScan/templates/startScan/history.html b/web/startScan/templates/startScan/history.html index ff9bfba1f..3705f41d9 100644 --- a/web/startScan/templates/startScan/history.html +++ b/web/startScan/templates/startScan/history.html @@ -476,6 +476,14 @@ padding: '2em', showLoaderOnConfirm: true, preConfirm: function() { + Swal.fire({ + title: 'Deleting...', + text: 'Please wait while the scans are being deleted.', + icon: 'info', + allowOutsideClick: false, + allowEscapeKey: false, + showConfirmButton: false + }); deleteForm = document.getElementById("scan_history_form"); deleteForm.action = "../delete/multiple"; deleteForm.submit(); From 2f539a70d13d8126ec67802cf2aff99ca2bdff9c Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 21 Aug 2024 21:11:08 +0530 Subject: [PATCH 083/211] refactor starting point url to starting point path --- web/reNgine/celery_custom_task.py | 2 +- web/reNgine/tasks.py | 37 +++++++++++-------- .../migrations/0002_auto_20240821_1518.py | 29 +++++++++++++++ ...url_scanhistory_cfg_starting_point_path.py | 18 +++++++++ web/startScan/models.py | 14 +++++++ .../_items/schedule_scan_wizard.html | 4 +- .../startScan/_items/start_scan_wizard.html | 4 +- web/startScan/views.py | 24 ++++++------ 8 files changed, 99 insertions(+), 33 deletions(-) create mode 100644 web/startScan/migrations/0002_auto_20240821_1518.py create mode 100644 web/startScan/migrations/0003_rename_cfg_starting_point_url_scanhistory_cfg_starting_point_path.py diff --git a/web/reNgine/celery_custom_task.py b/web/reNgine/celery_custom_task.py index 2c6b11f25..863f77169 100644 --- a/web/reNgine/celery_custom_task.py +++ b/web/reNgine/celery_custom_task.py @@ -67,7 +67,7 @@ def __call__(self, *args, **kwargs): self.subscan_id = ctx.get('subscan_id') self.engine_id = ctx.get('engine_id') self.filename = ctx.get('filename') - self.starting_point_url = ctx.get('starting_point_url', '') + self.starting_point_path = ctx.get('starting_point_path', '') self.excluded_paths = ctx.get('excluded_paths', []) self.results_dir = ctx.get('results_dir', RENGINE_RESULTS) self.yaml_configuration = ctx.get('yaml_configuration', {}) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 33ecbae04..d23086c91 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -57,7 +57,7 @@ def initiate_scan( imported_subdomains=[], out_of_scope_subdomains=[], initiated_by_id=None, - starting_point_url='', + starting_point_path='', excluded_paths=[], ): """Initiate a new scan. @@ -70,7 +70,7 @@ def initiate_scan( results_dir (str): Results directory. imported_subdomains (list): Imported subdomains. out_of_scope_subdomains (list): Out-of-scope subdomains. - starting_point_url (str): URL path. Default: '' Defined where to start the scan. + starting_point_path (str): URL path. Default: '' Defined where to start the scan. initiated_by (int): User ID initiating the scan. excluded_paths (list): Excluded paths. Default: [], url paths to exclude from scan. """ @@ -92,7 +92,7 @@ def initiate_scan( domain.save() # Get path filter - starting_point_url = starting_point_url.rstrip('/') + starting_point_path = starting_point_path.rstrip('/') # for live scan scan history id is passed as scan_history_id # and no need to create scan_history object @@ -114,6 +114,11 @@ def initiate_scan( scan.tasks = engine.tasks scan.results_dir = f'{results_dir}/{domain.name}_{scan.id}' add_gf_patterns = gf_patterns and 'fetch_url' in engine.tasks + # add configs to scan object, cfg_ prefix is used to avoid conflicts with other scan object fields + scan.cfg_starting_point_path = starting_point_path + scan.cfg_excluded_paths = excluded_paths + scan.cfg_out_of_scope_subdomains = out_of_scope_subdomains + if add_gf_patterns: scan.used_gf_patterns = ','.join(gf_patterns) scan.save() @@ -127,7 +132,7 @@ def initiate_scan( 'engine_id': engine_id, 'domain_id': domain.id, 'results_dir': scan.results_dir, - 'starting_point_url': starting_point_url, + 'starting_point_path': starting_point_path, 'excluded_paths': excluded_paths, 'yaml_configuration': config, 'out_of_scope_subdomains': out_of_scope_subdomains @@ -152,7 +157,7 @@ def initiate_scan( # If enable_http_crawl is set, create an initial root HTTP endpoint so that # HTTP crawling can start somewhere - http_url = f'{domain.name}{starting_point_url}' if starting_point_url else domain.name + http_url = f'{domain.name}{starting_point_path}' if starting_point_path else domain.name endpoint, _ = save_endpoint( http_url, ctx=ctx, @@ -228,7 +233,7 @@ def initiate_subscan( engine_id=None, scan_type=None, results_dir=RENGINE_RESULTS, - starting_point_url='', + starting_point_path='', excluded_paths=[], ): """Initiate a new subscan. @@ -239,7 +244,7 @@ def initiate_subscan( engine_id (int): Engine ID. scan_type (int): Scan type (periodic, live). results_dir (str): Results directory. - starting_point_url (str): URL path. Default: '' + starting_point_path (str): URL path. Default: '' excluded_paths (list): Excluded paths. Default: [], url paths to exclude from scan. """ @@ -298,13 +303,13 @@ def initiate_subscan( 'subdomain_id': subdomain.id, 'yaml_configuration': config, 'results_dir': results_dir, - 'starting_point_url': starting_point_url, + 'starting_point_path': starting_point_path, 'excluded_paths': excluded_paths, } # Create initial endpoints in DB: find domain HTTP endpoint so that HTTP # crawling can start somewhere - base_url = f'{subdomain.name}{starting_point_url}' if starting_point_url else subdomain.name + base_url = f'{subdomain.name}{starting_point_path}' if starting_point_path else subdomain.name endpoint, _ = save_endpoint( base_url, crawl=enable_http_crawl, @@ -406,8 +411,8 @@ def subdomain_discovery( if not host: host = self.subdomain.name if self.subdomain else self.domain.name - if self.starting_point_url: - logger.warning(f'Ignoring subdomains scan as an URL path filter was passed ({self.starting_point_url}).') + if self.starting_point_path: + logger.warning(f'Ignoring subdomains scan as an URL path filter was passed ({self.starting_point_path}).') return # Config @@ -1929,7 +1934,7 @@ def fetch_url(self, urls=[], ctx={}, description=None): if base_url and urlpath: subdomain = urlparse(base_url) - url = f'{subdomain.scheme}://{subdomain.netloc}{self.starting_point_url}' + url = f'{subdomain.scheme}://{subdomain.netloc}{self.starting_point_path}' if not validators.url(url): logger.warning(f'Invalid URL "{url}". Skipping.') @@ -1938,8 +1943,8 @@ def fetch_url(self, urls=[], ctx={}, description=None): all_urls.append(url) # Filter out URLs if a path filter was passed - if self.starting_point_url: - all_urls = [url for url in all_urls if self.starting_point_url in url] + if self.starting_point_path: + all_urls = [url for url in all_urls if self.starting_point_path in url] # if exclude_paths is found, then remove urls matching those paths if self.excluded_paths: @@ -2839,8 +2844,8 @@ def http_crawl( input_path = f'{self.results_dir}/httpx_input.txt' history_file = f'{self.results_dir}/commands.txt' if urls: # direct passing URLs to check - if self.starting_point_url: - urls = [u for u in urls if self.starting_point_url in u] + if self.starting_point_path: + urls = [u for u in urls if self.starting_point_path in u] with open(input_path, 'w') as f: f.write('\n'.join(urls)) diff --git a/web/startScan/migrations/0002_auto_20240821_1518.py b/web/startScan/migrations/0002_auto_20240821_1518.py new file mode 100644 index 000000000..03dd158b3 --- /dev/null +++ b/web/startScan/migrations/0002_auto_20240821_1518.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.23 on 2024-08-21 15:18 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('startScan', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='scanhistory', + name='cfg_excluded_paths', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None), + ), + migrations.AddField( + model_name='scanhistory', + name='cfg_out_of_scope_subdomains', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None), + ), + migrations.AddField( + model_name='scanhistory', + name='cfg_starting_point_url', + field=models.CharField(blank=True, max_length=200, null=True), + ), + ] diff --git a/web/startScan/migrations/0003_rename_cfg_starting_point_url_scanhistory_cfg_starting_point_path.py b/web/startScan/migrations/0003_rename_cfg_starting_point_url_scanhistory_cfg_starting_point_path.py new file mode 100644 index 000000000..e5f005bf1 --- /dev/null +++ b/web/startScan/migrations/0003_rename_cfg_starting_point_url_scanhistory_cfg_starting_point_path.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.23 on 2024-08-21 15:40 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('startScan', '0002_auto_20240821_1518'), + ] + + operations = [ + migrations.RenameField( + model_name='scanhistory', + old_name='cfg_starting_point_url', + new_name='cfg_starting_point_path', + ), + ] diff --git a/web/startScan/models.py b/web/startScan/models.py index 0d44638a9..b93a0645c 100644 --- a/web/startScan/models.py +++ b/web/startScan/models.py @@ -48,6 +48,20 @@ class ScanHistory(models.Model): dorks = models.ManyToManyField('Dork', related_name='dorks', blank=True) initiated_by = models.ForeignKey(User, on_delete=models.CASCADE, related_name='initiated_scans', blank=True, null=True) aborted_by = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, related_name='aborted_scans') + # scan related configs, append config fields with cfg_ + cfg_out_of_scope_subdomains = ArrayField( + models.CharField(max_length=200), + blank=True, + null=True, + default=list + ) + cfg_starting_point_path = models.CharField(max_length=200, blank=True, null=True) + cfg_excluded_paths = ArrayField( + models.CharField(max_length=200), + blank=True, + null=True, + default=list + ) def __str__(self): diff --git a/web/startScan/templates/startScan/_items/schedule_scan_wizard.html b/web/startScan/templates/startScan/_items/schedule_scan_wizard.html index d2dbf8816..b4e43e0f6 100644 --- a/web/startScan/templates/startScan/_items/schedule_scan_wizard.html +++ b/web/startScan/templates/startScan/_items/schedule_scan_wizard.html @@ -88,8 +88,8 @@

    Out of Scope Subdomains(Optional)

    URL Scope and Exclusions

    -

    Starting Point URL (Optional)

    - +

    Starting Point Path (Optional)

    + Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains.
    diff --git a/web/startScan/templates/startScan/_items/start_scan_wizard.html b/web/startScan/templates/startScan/_items/start_scan_wizard.html index 1f2af3eda..1fb0934d7 100644 --- a/web/startScan/templates/startScan/_items/start_scan_wizard.html +++ b/web/startScan/templates/startScan/_items/start_scan_wizard.html @@ -51,8 +51,8 @@

    Out of Scope Subdomains(Optional)

    URL Scope and Exclusions

    -

    Starting Point URL (Optional)

    - +

    Starting Point Path (Optional)

    + Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains.
    diff --git a/web/startScan/views.py b/web/startScan/views.py index 94b9ef21f..d6908f9c6 100644 --- a/web/startScan/views.py +++ b/web/startScan/views.py @@ -259,7 +259,7 @@ def start_scan_ui(request, slug, domain_id): subdomains_in = [s.rstrip() for s in subdomains_in if s] subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split() subdomains_out = [s.rstrip() for s in subdomains_out if s] - starting_point_url = request.POST['startingPointUrl'].strip() + starting_point_path = request.POST['startingPointPath'].strip() excluded_paths = request.POST['excludedPaths'] # string separated by , # split excluded paths by , excluded_paths = [path.strip() for path in excluded_paths.split(',')] @@ -284,7 +284,7 @@ def start_scan_ui(request, slug, domain_id): 'results_dir': '/usr/src/scan_results', 'imported_subdomains': subdomains_in, 'out_of_scope_subdomains': subdomains_out, - 'starting_point_url': starting_point_url, + 'starting_point_path': starting_point_path, 'excluded_paths': excluded_paths, 'initiated_by_id': request.user.id } @@ -329,7 +329,7 @@ def start_multiple_scan(request, slug): subdomains_in = [s.rstrip() for s in subdomains_in if s] subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split() subdomains_out = [s.rstrip() for s in subdomains_out if s] - starting_point_url = request.POST['startingPointUrl'].strip() + starting_point_path = request.POST['startingPointPath'].strip() excluded_paths = request.POST['excludedPaths'] # string separated by , # split excluded paths by , excluded_paths = [path.strip() for path in excluded_paths.split(',')] @@ -354,7 +354,7 @@ def start_multiple_scan(request, slug): 'initiated_by_id': request.user.id, 'imported_subdomains': subdomains_in, 'out_of_scope_subdomains': subdomains_out, - 'starting_point_url': starting_point_url, + 'starting_point_path': starting_point_path, 'excluded_paths': excluded_paths, } @@ -562,7 +562,7 @@ def schedule_scan(request, host_id, slug): subdomains_in = [s.rstrip() for s in subdomains_in if s] subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split() subdomains_out = [s.rstrip() for s in subdomains_out if s] - starting_point_url = request.POST['startingPointUrl'].strip() + starting_point_path = request.POST['startingPointPath'].strip() excluded_paths = request.POST['excludedPaths'] # string separated by , # split excluded paths by , excluded_paths = [path.strip() for path in excluded_paths.split(',')] @@ -596,7 +596,7 @@ def schedule_scan(request, host_id, slug): 'scan_type': SCHEDULED_SCAN, 'imported_subdomains': subdomains_in, 'out_of_scope_subdomains': subdomains_out, - 'starting_point_url': starting_point_url, + 'starting_point_path': starting_point_path, 'excluded_paths': excluded_paths, 'initiated_by_id': request.user.id } @@ -617,7 +617,7 @@ def schedule_scan(request, host_id, slug): 'scan_type': SCHEDULED_SCAN, 'imported_subdomains': subdomains_in, 'out_of_scope_subdomains': subdomains_out, - 'starting_point_url': starting_point_url, + 'starting_point_path': starting_point_path, 'excluded_paths': excluded_paths, 'initiated_by_id': request.user.id } @@ -745,7 +745,7 @@ def start_organization_scan(request, id, slug): subdomains_in = [s.rstrip() for s in subdomains_in if s] subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split() subdomains_out = [s.rstrip() for s in subdomains_out if s] - starting_point_url = request.POST['startingPointUrl'].strip() + starting_point_path = request.POST['startingPointPath'].strip() excluded_paths = request.POST['excludedPaths'] # string separated by , # split excluded paths by , excluded_paths = [path.strip() for path in excluded_paths.split(',')] @@ -768,7 +768,7 @@ def start_organization_scan(request, id, slug): 'initiated_by_id': request.user.id, 'imported_subdomains': subdomains_in, 'out_of_scope_subdomains': subdomains_out, - 'starting_point_url': starting_point_url, + 'starting_point_path': starting_point_path, 'excluded_paths': excluded_paths, } initiate_scan.apply_async(kwargs=kwargs) @@ -814,7 +814,7 @@ def schedule_organization_scan(request, slug, id): subdomains_in = [s.rstrip() for s in subdomains_in if s] subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split() subdomains_out = [s.rstrip() for s in subdomains_out if s] - starting_point_url = request.POST['startingPointUrl'].strip() + starting_point_path = request.POST['startingPointPath'].strip() excluded_paths = request.POST['excludedPaths'] # string separated by , # split excluded paths by , excluded_paths = [path.strip() for path in excluded_paths.split(',')] @@ -852,7 +852,7 @@ def schedule_organization_scan(request, slug, id): 'initiated_by_id': request.user.id, 'imported_subdomains': subdomains_in, 'out_of_scope_subdomains': subdomains_out, - 'starting_point_url': starting_point_url, + 'starting_point_path': starting_point_path, 'excluded_paths': excluded_paths, }) PeriodicTask.objects.create( @@ -876,7 +876,7 @@ def schedule_organization_scan(request, slug, id): 'initiated_by_id': request.user.id, 'imported_subdomains': subdomains_in, 'out_of_scope_subdomains': subdomains_out, - 'starting_point_url': starting_point_url, + 'starting_point_path': starting_point_path, 'excluded_paths': excluded_paths, }) PeriodicTask.objects.create(clocked=clock, From 4af8226098ca1c21f6052f4336166b45b9909e52 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Thu, 22 Aug 2024 10:59:16 +0530 Subject: [PATCH 084/211] Add imported subdomains in scan config --- web/reNgine/tasks.py | 1 + web/startScan/models.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index d23086c91..35f8aa85f 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -118,6 +118,7 @@ def initiate_scan( scan.cfg_starting_point_path = starting_point_path scan.cfg_excluded_paths = excluded_paths scan.cfg_out_of_scope_subdomains = out_of_scope_subdomains + scan.cfg_imported_subdomains = imported_subdomains if add_gf_patterns: scan.used_gf_patterns = ','.join(gf_patterns) diff --git a/web/startScan/models.py b/web/startScan/models.py index b93a0645c..fcfbafa20 100644 --- a/web/startScan/models.py +++ b/web/startScan/models.py @@ -48,7 +48,7 @@ class ScanHistory(models.Model): dorks = models.ManyToManyField('Dork', related_name='dorks', blank=True) initiated_by = models.ForeignKey(User, on_delete=models.CASCADE, related_name='initiated_scans', blank=True, null=True) aborted_by = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, related_name='aborted_scans') - # scan related configs, append config fields with cfg_ + # scan related configs, prefix config fields with cfg_ cfg_out_of_scope_subdomains = ArrayField( models.CharField(max_length=200), blank=True, @@ -62,6 +62,12 @@ class ScanHistory(models.Model): null=True, default=list ) + cfg_imported_subdomains = ArrayField( + models.CharField(max_length=200), + blank=True, + null=True, + default=list + ) def __str__(self): From b5846c6199509ad4ff7df0f800380c78b54fe1db Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Thu, 22 Aug 2024 10:59:36 +0530 Subject: [PATCH 085/211] fix icon and log view as code --- web/startScan/static/startScan/js/detail_scan.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/startScan/static/startScan/js/detail_scan.js b/web/startScan/static/startScan/js/detail_scan.js index a7a972d7d..8e2da5b20 100644 --- a/web/startScan/static/startScan/js/detail_scan.js +++ b/web/startScan/static/startScan/js/detail_scan.js @@ -1039,12 +1039,12 @@ function create_log_element(log) { let logElement = document.createElement("p"); innerHTML = `

    -

    - ${log.command} +

    + ${log.command}

    ` if (log.output != ''){ - innerHTML += `
    ${log.output}
    `; + innerHTML += `
    ${log.output}
    `; } logElement.innerHTML = innerHTML; return logElement; From ce80595a0164e31b9eb11124c7e339fba433d296 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Thu, 22 Aug 2024 11:00:31 +0530 Subject: [PATCH 086/211] show scan config in ui --- ...004_scanhistory_cfg_imported_subdomains.py | 19 +++++++ .../templates/startScan/detail_scan.html | 56 ++++++++++++++++--- web/static/custom/custom.js | 2 +- 3 files changed, 67 insertions(+), 10 deletions(-) create mode 100644 web/startScan/migrations/0004_scanhistory_cfg_imported_subdomains.py diff --git a/web/startScan/migrations/0004_scanhistory_cfg_imported_subdomains.py b/web/startScan/migrations/0004_scanhistory_cfg_imported_subdomains.py new file mode 100644 index 000000000..992199df5 --- /dev/null +++ b/web/startScan/migrations/0004_scanhistory_cfg_imported_subdomains.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.23 on 2024-08-22 04:26 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('startScan', '0003_rename_cfg_starting_point_url_scanhistory_cfg_starting_point_path'), + ] + + operations = [ + migrations.AddField( + model_name='scanhistory', + name='cfg_imported_subdomains', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None), + ), + ] diff --git a/web/startScan/templates/startScan/detail_scan.html b/web/startScan/templates/startScan/detail_scan.html index c7fad04d1..f704affd4 100644 --- a/web/startScan/templates/startScan/detail_scan.html +++ b/web/startScan/templates/startScan/detail_scan.html @@ -146,11 +146,51 @@

    Scan failed due to ERROR: {{history.error_message}}

    {% endif %} {% endif %} -
    Scan Logs
    - Logs -
    Scan Engine
    +
    Scan Configurations
    + Starting Path: {% if history.cfg_starting_point_path %} {{history.cfg_starting_point_path}} {% else %}/ {% endif %}
    +
    +
    + +
    +
      + {% for subdomain in history.cfg_imported_subdomains %} +
    • {{subdomain}}
      • + {% endfor %} +
    +
    +
    +
    + +
    +
      + {% for subdomain in history.cfg_out_of_scope_subdomains %} +
    • {{subdomain}}
      • + {% endfor %} +
    +
    +
    +
    + +
    +
      + {% for path in history.cfg_excluded_paths %} +
    • {{path}}
      • + {% endfor %} +
    +
    +
    +
    +
    Scan Logs
    + Logs +
    Scan Engine
    {{history.scan_type.engine_name}} -
    Scan Duration
    +
    Scan Duration
    {% if history.scan_status == -1 %}

    Scan not yet started.

    {% elif history.scan_status == 0 %} @@ -166,9 +206,7 @@
    Scan Duration
    {% elif history.scan_status == 3 %} Aborted in {{ history.start_scan_date|timesince:history.stop_scan_date }} {% endif %} - - -

    Scan Progress

    +

    Scan Progress

    {% if history.scan_status == -1 %}
    @@ -200,7 +238,7 @@

    Tagged to Organization

    {% endfor %} {% endif %} -
    Scan Timeline
    +
    Timeline
      @@ -220,7 +258,7 @@
      {{activity.title}}

      Error: {{activity.error_message}}

      {% endif %} {% endif %} - Logs + Logs {% endfor %}
    diff --git a/web/static/custom/custom.js b/web/static/custom/custom.js index b5f58bf1e..9dfd313ef 100644 --- a/web/static/custom/custom.js +++ b/web/static/custom/custom.js @@ -3328,4 +3328,4 @@ function handleHashInUrl(){ }, 100); } } -} +} \ No newline at end of file From 8b7b577fac97cc121c1736603822f977bd511da3 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Thu, 22 Aug 2024 20:51:50 +0530 Subject: [PATCH 087/211] Add show config modal in scan history --- .../templates/startScan/history.html | 5 + web/static/custom/custom.css | 14 +- web/static/custom/custom.js | 246 +++++++++++------- 3 files changed, 165 insertions(+), 100 deletions(-) diff --git a/web/startScan/templates/startScan/history.html b/web/startScan/templates/startScan/history.html index ff9bfba1f..0f4fe1257 100644 --- a/web/startScan/templates/startScan/history.html +++ b/web/startScan/templates/startScan/history.html @@ -97,6 +97,10 @@

    Filters

    {{ scan_history.domain.name }}
    + {% if scan_history.cfg_starting_point_path %} + Path: {{scan_history.cfg_starting_point_path}} + {% endif %} +
    {% for organization in scan_history.domain.get_organization %} {{ organization.name }} {% endfor %} @@ -173,6 +177,7 @@

    Filters

    +  Show Configurations {% if user|can:'initiate_scans_subscans' %} {% if scan_history.scan_status == 0 or scan_history.scan_status == 2 or scan_history.scan_status == 3 %} diff --git a/web/static/custom/custom.css b/web/static/custom/custom.css index 03373be25..52835d872 100644 --- a/web/static/custom/custom.css +++ b/web/static/custom/custom.css @@ -394,4 +394,16 @@ mark{ .tab-content .card-body { padding-left: 0.8rem !important; padding-right: 0.8rem !important; -} \ No newline at end of file +} + +.scan-config-modal .config-list { + max-height: 300px; + overflow-y: auto; + padding-left: 20px; +} +.scan-config-modal .config-list li { + padding: 5px 0; +} +.scan-config-modal .badge { + margin-left: 5px; +} diff --git a/web/static/custom/custom.js b/web/static/custom/custom.js index 9dfd313ef..570d72332 100644 --- a/web/static/custom/custom.js +++ b/web/static/custom/custom.js @@ -1460,107 +1460,107 @@ function fetch_whois(domain_name, force_reload_whois=false) { } function get_target_whois(domain_name) { - const url = `/api/tools/whois/?format=json&target=${domain_name}`; - - Swal.fire({ - title: `Fetching WHOIS details for ${domain_name}...`, - allowOutsideClick: false, - showConfirmButton: false, - willOpen: () => { - Swal.showLoading(); - } - }); - - fetch(url, { - method: 'GET', - credentials: "same-origin", - headers: { - "X-CSRFToken": getCookie("csrftoken"), - 'Content-Type': 'application/json' - }, - }) - .then(response => { - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - return response.json(); - }) - .then(data => { - console.log(data); - if (data.status) { - Swal.close(); - display_whois_on_modal(data); - } else { - throw new Error(data.result || 'Failed to fetch WHOIS data'); - } - }) - .catch(error => { - console.error('Error:', error); - let errorMessage = error.message; - if (errorMessage.includes('Netlas limit exceeded')) { - errorMessage = 'Rate limit exceeded. Please try again later.'; - } else if (errorMessage.includes('Invalid domain')) { - errorMessage = 'Invalid domain or no WHOIS data available.'; - } - Swal.fire({ - title: 'Error!', - text: `Failed to fetch WHOIS records for ${domain_name}: ${errorMessage}`, - icon: 'error' - }); - }); + const url = `/api/tools/whois/?format=json&target=${domain_name}`; + + Swal.fire({ + title: `Fetching WHOIS details for ${domain_name}...`, + allowOutsideClick: false, + showConfirmButton: false, + willOpen: () => { + Swal.showLoading(); + } + }); + + fetch(url, { + method: 'GET', + credentials: "same-origin", + headers: { + "X-CSRFToken": getCookie("csrftoken"), + 'Content-Type': 'application/json' + }, + }) + .then(response => { + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + return response.json(); + }) + .then(data => { + console.log(data); + if (data.status) { + Swal.close(); + display_whois_on_modal(data); + } else { + throw new Error(data.result || 'Failed to fetch WHOIS data'); + } + }) + .catch(error => { + console.error('Error:', error); + let errorMessage = error.message; + if (errorMessage.includes('Netlas limit exceeded')) { + errorMessage = 'Rate limit exceeded. Please try again later.'; + } else if (errorMessage.includes('Invalid domain')) { + errorMessage = 'Invalid domain or no WHOIS data available.'; + } + Swal.fire({ + title: 'Error!', + text: `Failed to fetch WHOIS records for ${domain_name}: ${errorMessage}`, + icon: 'error' + }); + }); } function get_domain_whois(domain_name, show_add_target_btn = false) { - const url = `/api/tools/whois/?format=json&target=${domain_name}`; - - Swal.fire({ - title: `Fetching WHOIS details for ${domain_name}...`, - allowOutsideClick: false, - showConfirmButton: false, - willOpen: () => { - Swal.showLoading(); - } - }); - - $('.modal').modal('hide'); - - fetch(url, { - method: 'GET', - credentials: "same-origin", - headers: { - "X-CSRFToken": getCookie("csrftoken"), - 'Content-Type': 'application/json' - }, - }) - .then(response => { - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - return response.json(); - }) - .then(data => { - console.log(data); - Swal.close(); - if (data.status) { - display_whois_on_modal(data, show_add_target_btn); - } else { - throw new Error(data.result || 'Failed to fetch WHOIS data'); - } - }) - .catch(error => { - console.error('Error:', error); - let errorMessage = error.message; - if (errorMessage.includes('Netlas limit exceeded')) { - errorMessage = 'Rate limit exceeded. Please try again later.'; - } else if (errorMessage.includes('Invalid domain')) { - errorMessage = 'Invalid domain or no WHOIS data available.'; - } - Swal.fire({ - title: 'Error!', - text: `Failed to fetch WHOIS records for ${domain_name}: ${errorMessage}`, - icon: 'error' - }); - }); + const url = `/api/tools/whois/?format=json&target=${domain_name}`; + + Swal.fire({ + title: `Fetching WHOIS details for ${domain_name}...`, + allowOutsideClick: false, + showConfirmButton: false, + willOpen: () => { + Swal.showLoading(); + } + }); + + $('.modal').modal('hide'); + + fetch(url, { + method: 'GET', + credentials: "same-origin", + headers: { + "X-CSRFToken": getCookie("csrftoken"), + 'Content-Type': 'application/json' + }, + }) + .then(response => { + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + return response.json(); + }) + .then(data => { + console.log(data); + Swal.close(); + if (data.status) { + display_whois_on_modal(data, show_add_target_btn); + } else { + throw new Error(data.result || 'Failed to fetch WHOIS data'); + } + }) + .catch(error => { + console.error('Error:', error); + let errorMessage = error.message; + if (errorMessage.includes('Netlas limit exceeded')) { + errorMessage = 'Rate limit exceeded. Please try again later.'; + } else if (errorMessage.includes('Invalid domain')) { + errorMessage = 'Invalid domain or no WHOIS data available.'; + } + Swal.fire({ + title: 'Error!', + text: `Failed to fetch WHOIS records for ${domain_name}: ${errorMessage}`, + icon: 'error' + }); + }); } @@ -3328,4 +3328,52 @@ function handleHashInUrl(){ }, 100); } } -} \ No newline at end of file +} + +function show_scan_configuration(starting_path, out_of_scope_subdomains, excluded_paths, imported_subdomains) { + $('#modal_title').html('Scan Configuration'); + $('#modal-content').empty(); + var content = ` +
    +
    +
    Starting Path: ${starting_path || '/'}
    +
    +     +
    +
    +
      + ${out_of_scope_subdomains.map(domain => `
    • ${domain}
      • `).join('')} +
    +
    +
    +
      + ${imported_subdomains.map(domain => `
    • ${domain}
      • `).join('')} +
    +
    +
    +
      + ${excluded_paths.map(path => `
    • ${path}
      • `).join('')} +
    +
    +
    +
    + `; + $('#modal-content').append(content); + $('#modal_dialog').modal('show'); +} From 6481b2e9f43b7662e6c87b0442de4b3953b77f49 Mon Sep 17 00:00:00 2001 From: Damian Husted <43425216+DamianHusted@users.noreply.github.com> Date: Fri, 23 Aug 2024 10:39:11 +1200 Subject: [PATCH 088/211] Update to uninstall.sh TLDR; Script updated to abort uninstall process if anything other than the specified "y/Y/Yes/yes/YES" is entered. Fixes uninstall on user inputs like "no" or blank input. --- The uninstall.sh script does not abort if any user input is provided. The following user-input scenarios were tested, confirmed and fixed: - no input - "n" - "N" - "No" - "1" - "q" - " - ( - ' - < - [ - { - / --- scripts/uninstall.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh index cc177285d..761738b5c 100755 --- a/scripts/uninstall.sh +++ b/scripts/uninstall.sh @@ -30,8 +30,8 @@ read -p "$(echo -e ${WARNING}"Are you sure you want to proceed? (y/Y/yes/YES to # change answer to lowecase for comparison ANSWER_LC=$(echo "$CONFIRM" | tr '[:upper:]' '[:lower:]') -if [[ "$ANSWER_LC" != "y" && "$ANSWER_LC" != "yes" ]]; then - print_status "${YELLOW}Uninstall aborted by user.${RESET}" +if [ -z "$CONFIRM" ] || { [ "$CONFIRM" != "y" ] && [ "$CONFIRM" != "Y" ] && [ "$CONFIRM" != "yes" ] && [ "$CONFIRM" != "Yes" ] && [ "$CONFIRM" != "YES" ]; }; then + print_status "${WARNING}Uninstall aborted by user.${RESET}" exit 0 fi From 36b6da29a64c7c61bc32c80f05154360b4bd697c Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 23 Aug 2024 11:00:05 +0530 Subject: [PATCH 089/211] added prefill in views.py for rescans --- .../templates/startScan/history.html | 5 ++-- web/startScan/views.py | 29 +++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/web/startScan/templates/startScan/history.html b/web/startScan/templates/startScan/history.html index d15529372..bb78cf07c 100644 --- a/web/startScan/templates/startScan/history.html +++ b/web/startScan/templates/startScan/history.html @@ -180,8 +180,9 @@

    Filters

     Show Configurations {% if user|can:'initiate_scans_subscans' %} {% if scan_history.scan_status == 0 or scan_history.scan_status == 2 or scan_history.scan_status == 3 %} - -  Rescan + +  Rescan + {% endif %} {% if scan_history.scan_status == 1 or scan_history.scan_status == -1%} diff --git a/web/startScan/views.py b/web/startScan/views.py index 0f5f57abb..1528230b7 100644 --- a/web/startScan/views.py +++ b/web/startScan/views.py @@ -299,19 +299,44 @@ def start_scan_ui(request, slug, domain_id): return HttpResponseRedirect(reverse('scan_history', kwargs={'slug': slug})) # GET request + + is_rescan = request.GET.get('rescan', 'false').lower() == 'true' + history_id = request.GET.get('history_id', None) + + # default values + subdomains_in = [] + subdomains_out = [] + starting_point_path = None + excluded_paths = [] + selected_engine_id = None + + if is_rescan and history_id: + previous_scan = get_object_or_404(ScanHistory, id=history_id) + selected_engine_id = getattr(previous_scan.scan_type, 'id', None) + subdomains_in = getattr(previous_scan, 'cfg_imported_subdomains', None) + subdomains_out = getattr(previous_scan, 'cfg_out_of_scope_subdomains', None) + starting_point_path = getattr(previous_scan, 'cfg_starting_point_path', None) + excluded_paths = getattr(previous_scan, 'cfg_excluded_paths', None) + engines = EngineType.objects.order_by('engine_name') custom_engines_count = ( EngineType.objects .filter(default_engine=False) .count() ) - excluded_paths = ','.join(DEFAULT_EXCLUDED_PATHS) + excluded_paths = ','.join(DEFAULT_EXCLUDED_PATHS) if not excluded_paths else ','.join(excluded_paths) + + # context values context = { 'scan_history_active': 'active', 'domain': domain, 'engines': engines, 'custom_engines_count': custom_engines_count, - 'excluded_paths': excluded_paths + 'excluded_paths': excluded_paths, + 'subdomains_in': subdomains_in, + 'subdomains_out': subdomains_out, + 'starting_point_path': starting_point_path, + 'selected_engine_id': selected_engine_id, } return render(request, 'startScan/start_scan_ui.html', context) From d7dd06366312d717f54ff27bcbee6978d985e9a0 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 23 Aug 2024 11:00:22 +0530 Subject: [PATCH 090/211] modify wizard to accomodate prefilled engines --- .../startScan/js/start-scan-wizard-init.js | 48 ++++++++++++++----- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/web/startScan/static/startScan/js/start-scan-wizard-init.js b/web/startScan/static/startScan/js/start-scan-wizard-init.js index ee1ce61db..8c3403b6b 100644 --- a/web/startScan/static/startScan/js/start-scan-wizard-init.js +++ b/web/startScan/static/startScan/js/start-scan-wizard-init.js @@ -10,24 +10,32 @@ $("#select_engine").steps({ onStepChanging: updateButton, labels: { finish: "Start Scan" }, onInit: function (event, current) { - $(".actions ul li:nth-child(3) a").attr( - "onclick", - `$(this).closest('form').submit()` + $(".actions ul li:nth-child(3) a") + .attr("onclick", `$(this).closest('form').submit()`) + .addClass("text-white"); + $(".actions ul li:nth-child(1) a[href='#previous']").removeClass( + "btn btn-primary waves-effect waves-light" ); + disableNext(); + updateButton(); + updatePreviousButton(); }, }); + $("input[type=radio][name=scan_mode]").change(initTimer).keyup(initTimer); -disableNext(); -$('a[role="menuitem"]').addClass("text-white"); +// $('a[role="menuitem"]').addClass("text-white"); function initTimer() { if (globalTimeout) clearTimeout(globalTimeout); globalTimeout = setTimeout(updateButton, 400); } + function disableNext() { var nextButton = $(".actions ul li:nth-child(2) a"); nextButton.attr("href", "#"); - buttonEnabled = $(".actions ul li:nth-child(2)") + nextButton.removeClass("btn btn-primary waves-effect waves-light text-white"); + buttonEnabled = false; + $(".actions ul li:nth-child(2)") .addClass("disabled") .attr("aria-disabled", "true"); } @@ -35,23 +43,37 @@ function disableNext() { function enableNext() { var nextButton = $(".actions ul li:nth-child(2) a"); nextButton.attr("href", "#next"); - buttonEnabled = $(".actions ul li:nth-child(2)") + buttonEnabled = true; + nextButton.addClass("btn btn-primary waves-effect waves-light text-white"); + $(".actions ul li:nth-child(2)") .removeClass("disabled") .attr("aria-disabled", "false"); } function updateButton() { - $('a[role="menuitem"]').addClass("btn btn-primary waves-effect waves-light"); - var text = $("input[type=radio][name=scan_mode]").val(); - if (text === "") { + var selectedEngine = $("input[type=radio][name=scan_mode]:checked").val(); + if (selectedEngine) { + enableNext(); + } else { disableNext(); - return false; + } + updatePreviousButton(); + return buttonEnabled; +} + +function updatePreviousButton() { + var previousButton = $("a[href='#previous']"); + if (previousButton.parent().hasClass("disabled")) { + previousButton.removeClass("text-white"); } else { - enableNext(); - return true; + previousButton.addClass("text-white"); } } +$("#select_engine").on("steps.change", function (e, currentIndex, newIndex) { + setTimeout(updatePreviousButton, 0); +}); + $("#excludedPaths").selectize({ persist: false, createOnBlur: true, From df9afac63b623881ea4724e124a897db900ca05d Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 23 Aug 2024 11:00:36 +0530 Subject: [PATCH 091/211] prefill values for scan engine andf other scan configurations --- .../startScan/_items/scanengine_accordion.html | 2 +- .../startScan/_items/start_scan_wizard.html | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/web/startScan/templates/startScan/_items/scanengine_accordion.html b/web/startScan/templates/startScan/_items/scanengine_accordion.html index 745c315f3..b16dc753c 100644 --- a/web/startScan/templates/startScan/_items/scanengine_accordion.html +++ b/web/startScan/templates/startScan/_items/scanengine_accordion.html @@ -4,7 +4,7 @@

    Out of Scope Subdomains (Optional)

    @@ -45,7 +50,12 @@

    Out of Scope Subdomains (Optional)

  • For regex: ^.*outofscope.*\.com$, admin.* etc
    • - + {% if subdomains_out %} + + {% else %} + + {% endif %}
    @@ -53,7 +63,11 @@

    URL Scope and Exclusions

    Starting Point Path (Optional)

    - + {% if starting_point_path %} + + {% else %} + + {% endif %} Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains.
    From 855d4a53a3334b7832fe4858ab6d43ef006b6129 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sat, 24 Aug 2024 11:14:09 +0530 Subject: [PATCH 092/211] rename older to default template, and add new template modern --- web/templates/report/default.html | 1049 +++++++++++++++++++++++++++++ web/templates/report/modern.html | 388 +++++++++++ 2 files changed, 1437 insertions(+) create mode 100644 web/templates/report/default.html create mode 100644 web/templates/report/modern.html diff --git a/web/templates/report/default.html b/web/templates/report/default.html new file mode 100644 index 000000000..a129de3c3 --- /dev/null +++ b/web/templates/report/default.html @@ -0,0 +1,1049 @@ + + + + Report + + + + + +
    +

    {{report_name}} +
    + {{scan_object.domain.name}} +
    + {# generated date #} + {% now "F j, Y" %} +

    +
    + {{company_name}} + {{company_address}} +
    +
    + {{company_email}} + {{company_website}} +
    +
    + {% if show_rengine_banner %}Generated by reNgine + https://github.com/yogeshojha/rengine + {% endif %} +
    +
    + +
    +

     

    +

    Table of contents

    +
      + {% if show_executive_summary %} +
      • + {% endif %} + +
      • +
      • + + {% if interesting_subdomains and show_recon %} +
      • + {% endif %} + + {% if all_vulnerabilities.count > 0 and show_vuln %} +
      • + {% endif %} + + {% if show_recon %} +
      • + {% endif %} + + {% if all_vulnerabilities.count > 0 and show_vuln %} +
      • + {% endif %} +
    +
    + + {% if show_executive_summary %} +
    +

    Executive summary

    +
    + {{executive_summary_description | safe }} +
    + {% endif %} + +
    +

    Quick Summary

    +

    This section contains quick summary of scan performed on {{scan_object.domain.name}}

    +
    +
    + + {# recon section #} + {% if show_recon %} +

    Reconnaissance

    +
    +
    +

    Subdomains +
    + + {{scan_object.get_subdomain_count}} + +

    +
    +
    +

    Endpoints +
    + + {{scan_object.get_endpoint_count}} + +

    +
    +
    +

    Vulnerabilities +
    + + {{all_vulnerabilities_count}} + +

    +
    +
    + {% endif %} + + + {% if show_vuln %} +
    +
    +

    Vulnerability Summary

    +
    +
    +

    Critical +
    + + {{scan_object.get_critical_vulnerability_count}} + +

    +
    +
    +

    High +
    + + {{scan_object.get_high_vulnerability_count}} + +

    +
    +
    +

    Medium +
    + + {{scan_object.get_medium_vulnerability_count}} + +

    +
    +
    +

    Low +
    + + {{scan_object.get_low_vulnerability_count}} + +

    +
    +
    +

    Info +
    + + {% if is_ignore_info_vuln %} + 0 + {% else %} + {{scan_object.get_info_vulnerability_count}} + {% endif %} + +

    +
    +
    +

    Unknown +
    + + {{scan_object.get_unknown_vulnerability_count}} + +

    +
    +
    +
    + {% endif %} + +
    +

    Timeline of the Assessment

    +

    + Scan started on: {{scan_object.start_scan_date|date:"F j, Y h:i"}} +
    + Total time taken: + {% if scan_object.scan_status == 0 %} + {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} + {% elif scan_object.scan_status == 1 %} + {{ scan_object.get_elapsed_time }} + {% elif scan_object.scan_status == 2 %} + {% if scan_object.get_completed_time_in_sec < 60 %} + Completed in < 1 minutes {% else %} Completed in {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} {% endif %} {% elif scan_object.scan_status == 3 %} Aborted in + {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} {% endif %}
    + Report Generated on: {% now "F j, Y" %} +

    +
    + + {# show interesting_subdomains section only when show_recon result is there #} + {% if interesting_subdomains and show_recon %} +
    +

    Interesting Recon Data

    +

    Listed below are the {{interesting_subdomains.count}} interesting subdomains identified on {{scan_object.domain.name}}

    +
    +
    +
    + # +
    +
    + Subdomain +
    +
    + Page title +
    +
    + HTTP Status +
    +
    + {% for subdomain in interesting_subdomains %} +
    +
    + {{ forloop.counter }} +
    +
    + {{subdomain.name}} +
    +
    + {% if subdomain.page_title %} + {{subdomain.page_title}} + {% else %} +     + {% endif %} +
    +
    + {% if subdomain.http_status %} + {{subdomain.http_status}} + {% else %} +     + {% endif %} +
    +
    + {% endfor %} +
    +
    + {% endif %} + + {# vulnerability_summary only when vuln_report #} + {% if show_vuln %} +
    +

    Summary of Vulnerabilities Identified

    + {% if all_vulnerabilities.count > 0 %} +

    Listed below are the vulnerabilities identified on {{scan_object.domain.name}}

    +
    +
    +
    + # +
    +
    + Vulnerability Name +
    +
    + Times Identified +
    +
    + Severity +
    +
    + {% for vulnerability in unique_vulnerabilities %} +
    +
    + {{ forloop.counter }} +
    +
    + {{vulnerability.name}} +
    +
    + {{vulnerability.count}} +
    + {% if vulnerability.severity == -1 %} +
    + Unknown + {% elif vulnerability.severity == 0 %} +
    + Informational + {% elif vulnerability.severity == 1 %} +
    + Low + {% elif vulnerability.severity == 2 %} +
    + Medium + {% elif vulnerability.severity == 3 %} +
    + High + {% elif vulnerability.severity == 4 %} +
    + Critical + {% endif %} +
    +
    + {% endfor %} + {% else %} +

    No Vulnerabilities were Discovered.

    + {% endif %} +
    + +
    + {% endif %} + + {# show discovered assets only for show_recon report #} + {% if show_recon %} +
    +

    Discovered Assets

    +

    Subdomains

    +

    + During the reconnaissance phase, {{scan_object.get_subdomain_count}} subdomains were discovered. + Out of {{scan_object.get_subdomain_count}} subdomains, {{subdomain_alive_count}} returned HTTP status 200. + {{interesting_subdomains.count}} interesting subdomains were also identified based on the interesting keywords used. +

    +

    {{scan_object.get_subdomain_count}} subdomains identified on {{scan_object.domain.name}}

    +
    +
    +
    + Subdomain +
    +
    + Page title +
    +
    + HTTP Status +
    +
    + {% for subdomain in subdomains %} +
    +
    + {{subdomain.name}} +
    +
    + {% if subdomain.page_title %} + {{subdomain.page_title}} + {% endif %} +
    +
    + {{subdomain.http_status}} +
    +
    + {% endfor %} +
    + {% if ip_addresses.count %} +

    IP Addresses

    +

    {{ip_addresses.count}} IP Addresses were identified on {{scan_object.domain.name}}

    +
    +
    +
    + IP +
    +
    + Open Ports +
    +
    + Remarks +
    +
    + {% for ip in ip_addresses %} +
    +
    + {{ip.address}} +
    +
    + {% for port in ip.ports.all %} + {{port.number}}/{{port.service_name}}{% if not forloop.last %},{% endif %} + {% endfor %} +
    + {% if ip.is_cdn %} +
    + CDN IP Address + {% else %} +
    + {% endif %} +
    +
    + {% endfor %} +
    + {% endif %} +
    +
    + {% endif %} + + {# reconnaissance finding only when show_recon #} + {% if show_recon %} +
    +

    Reconnaissance Findings

    + {% for subdomain in subdomains %} + + + + + {% if subdomain.http_status == 200 %} + + {% elif subdomain.http_status >= 300 and subdomain.http_status < 400 %} + + {% elif subdomain.http_status >= 400 %} + + {% elif subdomain.http_status == 0 %} + + {% else %} + + {% endif %} + + {% if subdomain.page_title %} + + + + {% endif %} + {% if subdomain.ip_addresses.all %} + + + + {% endif %} + {% if subdomain.get_vulnerabilities_without_info %} + + + + {% endif %} +
    {{ forloop.counter }}.{{subdomain.name}}{{subdomain.http_status}}{{subdomain.http_status}}{{subdomain.http_status}}N/A{{subdomain.http_status}}
    Page Title: {{subdomain.page_title}}
    + IP Address: +
      + {% for ip in subdomain.ip_addresses.all %} +
    • {{ip.address}} + {% if ip.ports.all %} +
        +
      • Open Ports:   + {% for port in ip.ports.all %} + {{port.number}}/{{port.service_name}}{% if not forloop.last %},{% endif %} + {% endfor %} +
        • +
      + {% endif %} +
      • + {% endfor %} +
    +
    + Vulnerabilities + {% regroup subdomain.get_vulnerabilities_without_info by name as vuln_list %} + +
    + {% endfor %} +
    + {% endif %} + + {% if all_vulnerabilities.count > 0 and show_vuln %} +
    +

    Vulnerabilities Discovered

    +

    + This section reports the security issues found during the audit. +
    + A Total of {{scan_object.get_vulnerability_count}} were discovered in {{scan_object.domain.name}}, + {{scan_object.get_critical_vulnerability_count}} of them were Critical, + {{scan_object.get_high_vulnerability_count}} of them were High Severity, + {{scan_object.get_medium_vulnerability_count}} of them were Medium severity, + {% if is_ignore_info_vuln %}0{% else %}{{scan_object.get_info_vulnerability_count}}{% endif %} of them were Low severity, and + {{scan_object.get_info_vulnerability_count}} of them were Informational. + {{scan_object.get_unknown_vulnerability_count}} of them were Unknown Severity. +

    +

    Vulnerability Breakdown by Severity

    +
    +
    +

    Critical +
    + + {{scan_object.get_critical_vulnerability_count}} + +

    +
    +
    +

    High +
    + + {{scan_object.get_high_vulnerability_count}} + +

    +
    +
    +

    Medium +
    + + {{scan_object.get_medium_vulnerability_count}} + +

    +
    +
    +

    Low +
    + + {{scan_object.get_low_vulnerability_count}} + +

    +
    +
    +

    Info +
    + + {% if is_ignore_info_vuln %} + 0 + {% else %} + {{scan_object.get_info_vulnerability_count}} + {% endif %} + +

    +
    +
    +

    Unknown +
    + + {{scan_object.get_unknown_vulnerability_count}} + +

    +
    +
    +
    + + + {# start vulnerability #} + {% if show_vuln %} +
    + {% regroup all_vulnerabilities by get_path as grouped_vulnerabilities %} + {% for vulnerabilities in grouped_vulnerabilities %} + {% for vulnerability in vulnerabilities.list %} +
    +

    + {{vulnerability.name}} + {% if vulnerabilities.grouper %} +
    in {{vulnerabilities.grouper}} + {% endif %} +     + {% if vulnerability.severity == -1 %} + Unknown +
    + {% elif vulnerability.severity == 0 %} + INFO +
    + {% elif vulnerability.severity == 1 %} + LOW +
    + {% elif vulnerability.severity == 2 %} + MEDIUM +
    + {% elif vulnerability.severity == 3 %} + HIGH +
    + {% elif vulnerability.severity == 4 %} + CRITICAL +
    + {% endif %} +

    + + Vulnerability Source: {{vulnerability.source|upper}}
    + {% if vulnerability.cvss_metrics or vulnerability.cvss_score or vulnerability.cve_ids.all or vulnerability.cve_ids.all %} + Vulnerability Classification
    + {% if vulnerability.cvss_metrics %} + CVSS Metrics: {{vulnerability.cvss_metrics}} + {% endif %} + {% if vulnerability.cvss_score %} +
    + CVSS Score: {{vulnerability.cvss_score}} + {% endif %} + {% if vulnerability.cve_ids.all %} +
    + CVE IDs
    +     {% for cve in vulnerability.cve_ids.all %} {{cve}}{% if not forloop.last %}, {% endif %} {% endfor %} + {% endif %} + {% if vulnerability.cwe_ids.all %} +
    + CWE IDs
    +     {% for cwe in vulnerability.cwe_ids.all %} {{cwe}}{% if not forloop.last %}, {% endif %} {% endfor %} + {% endif %} +
    + {% endif %} + {% if vulnerability.description %} +
    + Description
    + {{vulnerability.description|linebreaks}} + {% endif %} + {% if vulnerability.impact %} +
    + Impact
    + {{vulnerability.impact|linebreaks}} + {% endif %} + {% if vulnerability.remediation %} +
    + Remediation
    + {{vulnerability.remediation|linebreaks}} + {% endif %} +
    + Vulnerable URL(s)
    + + + + {% if vulnerability.references.all %} + References
    +
      + {% for ref in vulnerability.references.all %} +
    • + {{ref}} +
      • + {% endfor %} +
    + {% endif %} +
    +
    +
    + {% endfor %} + {% endfor %} +
    + {% endif %} + + {% endif %} +
    +

    END OF REPORT

    +
    + + + diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html new file mode 100644 index 000000000..fb3295ad6 --- /dev/null +++ b/web/templates/report/modern.html @@ -0,0 +1,388 @@ + + + + + Penetration Testing Report + + + + + + {% comment %} cover page {% endcomment %} +
    +
    +
    +
    +
    +
    +

    {{report_name}}

    +
    {{scan_object.domain.name}}
    +
    {% now "F j, Y" %}
    +
    +
    +
    + {{company_name}} | {{company_address}} | {{company_email}} | {{company_website}} + {% if show_rengine_banner %} +
    + Generated by reNgine
    + https://github.com/yogeshojha/rengine +
    + {% endif %} +
    +
    + + {% comment %} table of contents {% endcomment %} + + + {% if show_executive_summary %} +
    +

    Executive Summary

    +
    + {{executive_summary_description | safe }} +
    + {% endif %} + +
    +

    Quick Summary

    +

    This section contains quick summary of scan performed on {{scan_object.domain.name}}

    +
    +
    + +
    +
    +

    END OF REPORT

    +
    +
    +
    + + + \ No newline at end of file From 323aadd81b1688e38f166184b6f3e73561320b32 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sat, 24 Aug 2024 12:53:55 +0530 Subject: [PATCH 093/211] added quick summary grid boxes --- web/templates/report/modern.html | 874 ++++++++++++++++++------------- 1 file changed, 509 insertions(+), 365 deletions(-) diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index fb3295ad6..bda33e450 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -1,388 +1,532 @@ - - + + Penetration Testing Report - + - - - + + {% comment %} cover page {% endcomment %}
    -
    -
    -
    -
    -
    -

    {{report_name}}

    -
    {{scan_object.domain.name}}
    -
    {% now "F j, Y" %}
    -
    -
    -
    - {{company_name}} | {{company_address}} | {{company_email}} | {{company_website}} - {% if show_rengine_banner %} -
    - Generated by reNgine
    - https://github.com/yogeshojha/rengine -
    - {% endif %} +
    +
    +
    +
    +
    +

    {{report_name}}

    +
    {{scan_object.domain.name}}
    +
    {% now "F j, Y" %}
    +
    +
    +
    + {{company_name}} | {{company_address}} | {{company_email}} | + {{company_website}} {% if show_rengine_banner %} +
    + Generated by reNgine
    + https://github.com/yogeshojha/rengine
    + {% endif %} +
    {% comment %} table of contents {% endcomment %}
    -

    Table of Contents

    -
    - {% if show_executive_summary %} -
    - Executive Summary - -
    - {% endif %} -
    - Quick Summary - -
    -
    - Assessment Timeline - -
    - {% if show_vuln %} -
    - Vulnerability Summary - -
    - {% endif %} - {% if show_recon %} -
    - Reconnaissance Results - -
    - {% endif %} - {% if show_vuln %} -
    - Vulnerabilities Discovered - -
    - {% endif %} +

    Table of Contents

    +
    + {% if show_executive_summary %} +
    + Executive Summary + +
    + {% endif %} +
    + Quick Summary + +
    +
    + Assessment Timeline + +
    + {% if show_vuln %} +
    + Vulnerability Summary +
    + {% endif %} {% if show_recon %} +
    + Reconnaissance Results + +
    + {% endif %} {% if show_vuln %} +
    + Vulnerabilities Discovered + +
    + {% endif %} +
    {% if show_executive_summary %} -
    -

    Executive Summary

    -
    - {{executive_summary_description | safe }} -
    +
    +

    Executive Summary

    +
    + {{executive_summary_description | safe }} +
    {% endif %} -
    +

    Quick Summary

    -

    This section contains quick summary of scan performed on {{scan_object.domain.name}}

    -
    +

    + This section contains quick summary of scan performed on + {{scan_object.domain.name}} +

    +
    + {% if show_recon %} +
    +

    Reconnaissance

    +
    +
    +

    Subdomains

    +
    {{scan_object.get_subdomain_count}}
    +
    +
    +

    Endpoints

    +
    {{scan_object.get_endpoint_count}}
    +
    +
    +

    Vulnerabilities

    +
    {{all_vulnerabilities_count}}
    +
    +
    +
    + {% endif %} + {% if show_vuln %} +
    +

    Vulnerability Summary

    +
    +
    +

    Critical

    +
    {{scan_object.get_critical_vulnerability_count}}
    +
    +
    +

    High

    +
    {{scan_object.get_high_vulnerability_count}}
    +
    +
    +

    Medium

    +
    {{scan_object.get_medium_vulnerability_count}}
    +
    +
    +
    +
    +

    Low

    +
    {{scan_object.get_low_vulnerability_count}}
    +
    +
    +

    Info

    +
    + {% if is_ignore_info_vuln %} + 0 + {% else %} + {{scan_object.get_info_vulnerability_count}} + {% endif %} +
    +
    +
    +

    Unknown

    +
    {{scan_object.get_unknown_vulnerability_count}}
    +
    +
    +
    + {% endif %}
    -
    -

    END OF REPORT

    -
    -
    +
    +

    END OF REPORT

    +
    +
    - - - \ No newline at end of file + + From 005d215f54b802ac8850380542e15973b085313b Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sat, 24 Aug 2024 13:26:18 +0530 Subject: [PATCH 094/211] added scan timeline --- web/templates/report/modern.html | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index bda33e450..c77feab83 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -520,6 +520,25 @@

    Unknown

    {% endif %} + +
    +

    Assessment Timeline

    +

    + Scan started on: {{scan_object.start_scan_date|date:"F j, Y h:i"}} +
    + Total time taken: + {% if scan_object.scan_status == 0 %} + {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} + {% elif scan_object.scan_status == 1 %} + {{ scan_object.get_elapsed_time }} + {% elif scan_object.scan_status == 2 %} + {% if scan_object.get_completed_time_in_sec < 60 %} + Completed in < 1 minutes {% else %} Completed in {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} {% endif %} {% elif scan_object.scan_status == 3 %} Aborted in + {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} {% endif %}
    + Report Generated on: {% now "F j, Y" %} +

    +
    +
    From e4aa74315a4e36db8790884d62e80e5a8c574284 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sat, 24 Aug 2024 15:11:05 +0530 Subject: [PATCH 095/211] feat: Add subdomain HTTP status breakdown chart to modern report template --- web/reNgine/charts.py | 104 +++++++++++++++++++++++++++++++ web/templates/report/modern.html | 41 ++++++++++-- 2 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 web/reNgine/charts.py diff --git a/web/reNgine/charts.py b/web/reNgine/charts.py new file mode 100644 index 000000000..df215a4aa --- /dev/null +++ b/web/reNgine/charts.py @@ -0,0 +1,104 @@ +import base64 +import colorsys + +import plotly.graph_objs as go +from plotly.io import to_image +from django.db.models import Count + +from startScan.models import * + + + +""" + This file is used to generate the charts for the pdf report. +""" + +def generate_subdomain_chart_by_http_status(subdomains): + """ + Generates a donut chart using plotly for the subdomains based on the http status. + Args: + sobdomains: List of subdomains. + Returns: + Image as base64 encoded string. + """ + http_statuses = ( + subdomains + .exclude(http_status=0) + .values('http_status') + .annotate(count=Count('http_status')) + .order_by('-count') + ) + http_status_count = [{'http_status': entry['http_status'], 'count': entry['count']} for entry in http_statuses] + + labels = [f"{entry['http_status']} ({entry['count']})" for entry in http_status_count] + sizes = [entry['count'] for entry in http_status_count] + colors = [get_color_by_http_status(entry['http_status']) for entry in http_status_count] + + fig = go.Figure(data=[go.Pie( + labels=labels, + values=sizes, + marker=dict(colors=colors), + hole=0.4, + textinfo="value", + textfont=dict(size=18), + hoverinfo="none" + )]) + + fig.update_layout( + title_text="", + annotations=[dict(text='HTTP Status', x=0.5, y=0.5, font_size=14, showarrow=False)], + showlegend=True, + margin=dict(t=50, b=50, l=50, r=50), + width=450, + height=450, + ) + + img_bytes = to_image(fig, format="png") + img_base64 = base64.b64encode(img_bytes).decode('utf-8') + return img_base64 + + +def generate_color(base_color, offset): + r, g, b = int(base_color[1:3], 16), int(base_color[3:5], 16), int(base_color[5:7], 16) + factor = 1 + (offset * 0.03) + r, g, b = [min(255, int(c * factor)) for c in (r, g, b)] + return f"#{r:02x}{g:02x}{b:02x}" + + +def get_color_by_http_status(http_status): + """ + Returns the color based on the http status. + Args: + http_status: HTTP status code. + Returns: + Color code. + """ + + status = int(http_status) + + colors = { + 200: "#36a2eb", + 300: "#4bc0c0", + 400: "#ff6384", + 401: "#ff9f40", + 403: "#f27474", + 404: "#ffa1b5", + 429: "#bf7bff", + 500: "#9966ff", + 502: "#8a4fff", + 503: "#c39bd3", + } + + + if status in colors: + return colors[status] + elif 200 <= status < 300: + return generate_color(colors[200], status - 200) + elif 300 <= status < 400: + return generate_color(colors[300], status - 300) + elif 400 <= status < 500: + return generate_color(colors[400], status - 400) + elif 500 <= status < 600: + return generate_color(colors[500], status - 500) + else: + return "#c9cbcf" \ No newline at end of file diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index c77feab83..6e7f22bb7 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -190,7 +190,7 @@ {% comment %} table of content page {% endcomment %} .toc-modern { background-color: white; - padding: 20mm; + padding: 15mm; font-family: 'Inter', sans-serif; } @@ -239,17 +239,29 @@ #executive-summary-page { justify-content: space-between; background-color: white; - padding: 20mm; + padding: 15mm; font-family: 'Inter', sans-serif; } #quick-summary-page { justify-content: space-between; background-color: white; - padding: 20mm; + padding: 15mm; font-family: 'Inter', sans-serif; } + #summary-of-finding-page { + justify-content: space-between; + background-color: white; + padding: 15mm; + font-family: 'Inter', sans-serif; + } + + #summary-of-finding-page h2 { + font-size: 24pt; + color: {{primary_color}}; + } + #executive-summary-page h2 { font-size: 24pt; color: {{primary_color}}; @@ -373,6 +385,13 @@ .info .number, .info::before { color: #3498db; border-right-color: #3498db; } .unknown .number, .unknown::before { color: #95a5a6; border-right-color: #95a5a6; } .primary { color: {{primary_color}}; } + + .center-img { + display: block; + margin-left: auto; + margin-right: auto; + width: 50%; + } @@ -522,6 +541,8 @@

    Unknown

    {% endif %}
    +
    +

    Assessment Timeline

    Scan started on: {{scan_object.start_scan_date|date:"F j, Y h:i"}} @@ -538,7 +559,17 @@

    Assessment Timeline

    Report Generated on: {% now "F j, Y" %}

    - +
    + +
    +

    Summary of Findings

    +
    +

    + This section provides a summary of the findings. +

    +
    +

    Subdomain Breakdown by HTTP Status

    +
    @@ -547,5 +578,7 @@

    END OF REPORT

    + + From 72489043af0251f4608f00a4f9a72cfc3135e4b0 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sat, 24 Aug 2024 20:34:16 +0530 Subject: [PATCH 096/211] add vuln chart --- web/reNgine/charts.py | 108 ++++++++++++++++++++++++++----- web/startScan/views.py | 11 +++- web/templates/report/modern.html | 11 +++- 3 files changed, 109 insertions(+), 21 deletions(-) diff --git a/web/reNgine/charts.py b/web/reNgine/charts.py index df215a4aa..87ecd44fa 100644 --- a/web/reNgine/charts.py +++ b/web/reNgine/charts.py @@ -4,6 +4,7 @@ import plotly.graph_objs as go from plotly.io import to_image from django.db.models import Count +from reNgine.definitions import NUCLEI_SEVERITY_MAP from startScan.models import * @@ -15,11 +16,12 @@ def generate_subdomain_chart_by_http_status(subdomains): """ - Generates a donut chart using plotly for the subdomains based on the http status. - Args: - sobdomains: List of subdomains. - Returns: - Image as base64 encoded string. + Generates a donut chart using plotly for the subdomains based on the http status. + Includes label, count, and percentage inside the chart segments and in the legend. + Args: + subdomains: QuerySet of subdomains. + Returns: + Image as base64 encoded string. """ http_statuses = ( subdomains @@ -30,34 +32,106 @@ def generate_subdomain_chart_by_http_status(subdomains): ) http_status_count = [{'http_status': entry['http_status'], 'count': entry['count']} for entry in http_statuses] - labels = [f"{entry['http_status']} ({entry['count']})" for entry in http_status_count] + total = sum(entry['count'] for entry in http_status_count) + + labels = [str(entry['http_status']) for entry in http_status_count] sizes = [entry['count'] for entry in http_status_count] colors = [get_color_by_http_status(entry['http_status']) for entry in http_status_count] + text = [f"{label}
    {size}
    ({size/total:.1%})" for label, size in zip(labels, sizes)] + fig = go.Figure(data=[go.Pie( - labels=labels, - values=sizes, - marker=dict(colors=colors), - hole=0.4, - textinfo="value", - textfont=dict(size=18), - hoverinfo="none" - )]) + labels=labels, + values=sizes, + marker=dict(colors=colors), + hole=0.4, + textinfo="text", + text=text, + textposition="inside", + textfont=dict(size=10), + hoverinfo="label+percent+value" + )]) fig.update_layout( title_text="", annotations=[dict(text='HTTP Status', x=0.5, y=0.5, font_size=14, showarrow=False)], showlegend=True, - margin=dict(t=50, b=50, l=50, r=50), - width=450, - height=450, + margin=dict(t=60, b=60, l=60, r=60), + width=700, + height=700, + ) + + img_bytes = to_image(fig, format="png") + img_base64 = base64.b64encode(img_bytes).decode('utf-8') + return img_base64 + + + +def get_color_by_severity(severity_int): + """ + Returns a color based on the severity level using a modern color scheme. + """ + color_map = { + 4: '#FF4D6A', + 3: '#FF9F43', + 2: '#FFCA3A', + 1: '#4ADE80', + 0: '#4ECDC4', + -1: '#A8A9AD', + } + return color_map.get(severity_int, '#A8A9AD') # Default to gray if severity is unknown + +def generate_vulnerability_chart_by_severity(vulnerabilities): + """ + Generates a donut chart using plotly for the vulnerabilities based on the severity. + Args: + vulnerabilities: QuerySet of Vulnerability objects. + Returns: + Image as base64 encoded string. + """ + severity_counts = ( + vulnerabilities + .values('severity') + .annotate(count=Count('severity')) + .order_by('-severity') + ) + + total = sum(entry['count'] for entry in severity_counts) + + labels = [NUCLEI_REVERSE_SEVERITY_MAP[entry['severity']].capitalize() for entry in severity_counts] + values = [entry['count'] for entry in severity_counts] + colors = [get_color_by_severity(entry['severity']) for entry in severity_counts] + + text = [f"{label}
    {value}
    ({value/total:.1%})" for label, value in zip(labels, values)] + + fig = go.Figure(data=[go.Pie( + labels=labels, + values=values, + marker=dict(colors=colors), + hole=0.4, + textinfo="text", + text=text, + textposition="inside", + textfont=dict(size=12), + hoverinfo="label+percent+value", + )]) + + fig.update_layout( + title_text="", + annotations=[dict(text='Severity', x=0.5, y=0.5, font_size=14, showarrow=False)], + showlegend=True, + margin=dict(t=60, b=60, l=60, r=60), + width=700, + height=700, ) + img_bytes = to_image(fig, format="png") img_base64 = base64.b64encode(img_bytes).decode('utf-8') return img_base64 + def generate_color(base_color, offset): r, g, b = int(base_color[1:3], 16), int(base_color[3:5], 16), int(base_color[5:7], 16) factor = 1 + (offset * 0.03) diff --git a/web/startScan/views.py b/web/startScan/views.py index 0f5f57abb..3e88d2a03 100644 --- a/web/startScan/views.py +++ b/web/startScan/views.py @@ -1,7 +1,7 @@ import markdown from celery import group -from weasyprint import HTML +from weasyprint import HTML, CSS from datetime import datetime from django.contrib import messages from django.db.models import Count @@ -14,6 +14,7 @@ from rolepermissions.decorators import has_permission_decorator from reNgine.celery import app +from reNgine.charts import * from reNgine.common_func import * from reNgine.definitions import ABORTED_TASK, SUCCESS_TASK from reNgine.tasks import create_scan_activity, initiate_scan, run_command @@ -1074,13 +1075,17 @@ def create_report(request, id): data['primary_color'] = primary_color data['secondary_color'] = secondary_color - template = get_template('report/template.html') + data['subdomain_http_status_chart'] = generate_subdomain_chart_by_http_status(subdomains) + data['vulns_severity_chart'] = generate_vulnerability_chart_by_severity(vulns) if vulns else '' + + template = get_template('report/modern.html') html = template.render(data) pdf = HTML(string=html).write_pdf() + # pdf = HTML(string=html).write_pdf(stylesheets=[CSS(string='@page { size: A4; margin: 0; }')]) if 'download' in request.GET: response = HttpResponse(pdf, content_type='application/octet-stream') else: response = HttpResponse(pdf, content_type='application/pdf') - return response + return response \ No newline at end of file diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index 6e7f22bb7..2505ce6e5 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -568,8 +568,17 @@

    Summary of Findings

    This section provides a summary of the findings.

    -

    Subdomain Breakdown by HTTP Status

    +

    Subdomains Breakdown by HTTP Status

    + {% if show_vuln and unique_vulnerabilities %} +

    Vulnerabilities Breakdown by Severity

    + + {% endif %} + + +
    +

    Vulnerabilities Breakdown by Severity

    +
    From 9ab1d9f4ea2f781fb5963a817299051a7295ea58 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 25 Aug 2024 12:28:12 +0530 Subject: [PATCH 097/211] add vuln list table --- web/reNgine/charts.py | 16 +++++ web/templates/report/modern.html | 116 +++++++++++++++++++++++++++++-- 2 files changed, 128 insertions(+), 4 deletions(-) diff --git a/web/reNgine/charts.py b/web/reNgine/charts.py index 87ecd44fa..546f09a62 100644 --- a/web/reNgine/charts.py +++ b/web/reNgine/charts.py @@ -59,6 +59,14 @@ def generate_subdomain_chart_by_http_status(subdomains): margin=dict(t=60, b=60, l=60, r=60), width=700, height=700, + legend=dict( + font=dict(size=18), + orientation="v", + yanchor="middle", + y=0.5, + xanchor="left", + x=1.05 + ), ) img_bytes = to_image(fig, format="png") @@ -123,6 +131,14 @@ def generate_vulnerability_chart_by_severity(vulnerabilities): margin=dict(t=60, b=60, l=60, r=60), width=700, height=700, + legend=dict( + font=dict(size=18), + orientation="v", + yanchor="middle", + y=0.5, + xanchor="left", + x=1.05 + ), ) diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index 2505ce6e5..58c06b977 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -257,6 +257,13 @@ font-family: 'Inter', sans-serif; } + #vulnerability-breakdown-chart-page { + justify-content: space-between; + background-color: white; + padding: 15mm; + font-family: 'Inter', sans-serif; + } + #summary-of-finding-page h2 { font-size: 24pt; color: {{primary_color}}; @@ -272,6 +279,11 @@ color: {{primary_color}}; } + #vulnerability-breakdown-chart-page h2 { + font-size: 24pt; + color: {{primary_color}}; + } + @page end-of-report { background: {{primary_color}}; margin: 0; @@ -386,6 +398,63 @@ .unknown .number, .unknown::before { color: #95a5a6; border-right-color: #95a5a6; } .primary { color: {{primary_color}}; } + {% comment %} table related {% endcomment %} + .vuln-summary-table-container { + width: 100%; + margin: auto auto; + background-color: #ffffff; + border-radius: 4mm; + overflow: hidden; + } + .vuln-summary-table { + width: 100%; + border-collapse: separate; + border-spacing: 0.5mm; + background-color: #ffffff; + } + .vuln-summary-table th, .vuln-summary-table td { + padding: 2.5mm 3mm; + text-align: center; + vertical-align: middle; + } + .vuln-summary-table th { + background-color: #4a5568; + color: white; + font-weight: bold; + } + .vuln-summary-table .vulnerability-td { + text-align: left !important; + } + .vulnerability-td a { + color: inherit; + text-decoration: none; + font-weight: inherit; + } + .number-td { + width: 5mm; + text-align: center; + color: rgba(0, 0, 0, 0.5); + font-weight: normal; + } + .vulnerability-td { + width: 100mm; + text-align: left; + } + .instances-td { + width: 10mm; + text-align: center; + } + .severity-td { + width: 10mm; + text-align: center; + font-weight: bold; + } + .unknown-td { background-color: #f7f7f7; color: #777777; } + .info-td { background-color: #e3f2fd; color: #1565c0; } + .low-td { background-color: #e8f5e9; color: #2e7d32; } + .medium-td { background-color: #fff3e0; color: #ef6c00; } + .high-td { background-color: #fbe9e7; color: #d84315; } + .critical-td { background-color: #ffebee; color: #c62828; } .center-img { display: block; margin-left: auto; @@ -440,10 +509,10 @@

    Table of Contents

    {% if show_vuln %}
    - Vulnerability Summary - +
    {% endif %} {% if show_recon %}
    @@ -577,8 +646,47 @@

    Vulnerabilities Breakdown by Severity

    -

    Vulnerabilities Breakdown by Severity

    - +

    Summary of Vulnerabilities Identified

    +

    Listed below are the vulnerabilities identified on {{scan_object.domain.name}}

    +
    + + + + + + + + {% for vulnerability in unique_vulnerabilities %} + + + + + + + {% endfor %} +
    #Vulnerability NameInstancesSeverity
    {{ forloop.counter }}{{vulnerability.name}}{{vulnerability.count}} + {% if vulnerability.severity == -1 %} + Unknown + {% elif vulnerability.severity == 0 %} + Informational + {% elif vulnerability.severity == 1 %} + Low + {% elif vulnerability.severity == 2 %} + Medium + {% elif vulnerability.severity == 3 %} + High + {% elif vulnerability.severity == 4 %} + Critical + {% endif %} +
    +
    +
    From 3e8f4be2a9da2f5ffb173a94446b82a4ca4fa934 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 25 Aug 2024 13:20:13 +0530 Subject: [PATCH 098/211] fix page margin --- web/templates/report/modern.html | 127 +++++++++++++++++-------------- 1 file changed, 70 insertions(+), 57 deletions(-) diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index 58c06b977..cb77e1181 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -10,7 +10,7 @@ - - - {% comment %} cover page {% endcomment %} -
    -
    -
    -
    -
    -
    -

    {{report_name}}

    -
    {{scan_object.domain.name}}
    -
    {% now "F j, Y" %}
    -
    -
    -
    - {{company_name}} | {{company_address}} | {{company_email}} | - {{company_website}} {% if show_rengine_banner %} -
    - Generated by reNgine
    - https://github.com/yogeshojha/rengine -
    - {% endif %} -
    -
    - - {% comment %} table of contents {% endcomment %} -
    -
    -

    Table of Contents

    -
    - {% if show_executive_summary %} -
    - Executive Summary - + + + + {% comment %} cover page {% endcomment %} +
    +
    +
    +
    +
    +
    +

    {{ report_name }}

    +
    {{ scan_object.domain.name }}
    +
    {% now "F j, Y" %}
    +
    - {% endif %} -
    - Quick Summary - +
    + {{ company_name }} | {{ company_address }} | {{ company_email }} | + {{ company_website }} + {% if show_rengine_banner %} +
    + Generated by reNgine +
    + https://github.com/yogeshojha/rengine +
    + {% endif %}
    -
    - Assessment Timeline - +
    + {% comment %} table of contents {% endcomment %} + + {% if show_executive_summary %} +
    +

    Executive Summary

    +
    + {{ executive_summary_description | safe }} +
    + {% endif %} +
    +

    Quick Summary

    +

    + This section contains quick summary of scan performed on + {{ scan_object.domain.name }} +

    +
    + {% if show_recon %} +
    +

    Reconnaissance

    +
    +
    +

    Subdomains

    +
    {{ scan_object.get_subdomain_count }}
    +
    +
    +

    Endpoints

    +
    {{ scan_object.get_endpoint_count }}
    +
    +
    +

    Vulnerabilities

    +
    {{ all_vulnerabilities_count }}
    +
    +
    +
    + {% endif %} {% if show_vuln %} -
    - Vulnerability Summary - -
    - {% endif %} {% if show_recon %} -
    - Reconnaissance Results - +
    +

    Vulnerability Summary

    +
    +
    +

    Critical

    +
    {{ scan_object.get_critical_vulnerability_count }}
    +
    +
    +

    High

    +
    {{ scan_object.get_high_vulnerability_count }}
    +
    +
    +

    Medium

    +
    {{ scan_object.get_medium_vulnerability_count }}
    +
    +
    +
    +
    +

    Low

    +
    {{ scan_object.get_low_vulnerability_count }}
    +
    +
    +

    Info

    +
    + {% if is_ignore_info_vuln %} + 0 + {% else %} + {{ scan_object.get_info_vulnerability_count }} + {% endif %} +
    +
    +
    +

    Unknown

    +
    {{ scan_object.get_unknown_vulnerability_count }}
    +
    +
    +
    + {% endif %} +
    +
    +
    +

    Assessment Timeline

    +

    + Scan started on: {{ scan_object.start_scan_date|date:"F j, Y h:i" }} +
    + Total time taken: + {% if scan_object.scan_status == 0 %} + {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} + {% elif scan_object.scan_status == 1 %} + {{ scan_object.get_elapsed_time }} + {% elif scan_object.scan_status == 2 %} + {% if scan_object.get_completed_time_in_sec < 60 %} + Completed in < 1 minutes + {% else %} + Completed in {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} + {% endif %} + {% elif scan_object.scan_status == 3 %} + Aborted in + {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} + {% endif %} +
    + Report Generated on: {% now "F j, Y" %} +

    - {% endif %} {% if show_vuln %} -
    - Vulnerabilities Discovered - +
    +
    +

    Summary of Findings

    +
    +

    This section provides a summary of the findings.

    +

    Subdomains Breakdown by HTTP Status

    + + {% if show_vuln and unique_vulnerabilities %} +

    Vulnerabilities Breakdown by Severity

    + {% endif %} -
    -
    -
    - - {% if show_executive_summary %} -
    -

    Executive Summary

    -
    - {{executive_summary_description | safe }} -
    - {% endif %} - -
    -

    Quick Summary

    -

    - This section contains quick summary of scan performed on - {{scan_object.domain.name}} -

    -
    - {% if show_recon %} -
    -

    Reconnaissance

    -
    -
    -

    Subdomains

    -
    {{scan_object.get_subdomain_count}}
    -
    -
    -

    Endpoints

    -
    {{scan_object.get_endpoint_count}}
    -
    -
    -

    Vulnerabilities

    -
    {{all_vulnerabilities_count}}
    -
    -
    -
    - {% endif %} - {% if show_vuln %} -
    -

    Vulnerability Summary

    -
    -
    -

    Critical

    -
    {{scan_object.get_critical_vulnerability_count}}
    -
    -
    -

    High

    -
    {{scan_object.get_high_vulnerability_count}}
    -
    -
    -

    Medium

    -
    {{scan_object.get_medium_vulnerability_count}}
    -
    -
    -
    -
    -

    Low

    -
    {{scan_object.get_low_vulnerability_count}}
    -
    -
    -

    Info

    -
    - {% if is_ignore_info_vuln %} - 0 - {% else %} - {{scan_object.get_info_vulnerability_count}} - {% endif %} -
    -
    -
    -

    Unknown

    -
    {{scan_object.get_unknown_vulnerability_count}}
    -
    -
    -
    - {% endif %} - -
    -
    -
    -

    Assessment Timeline

    -

    - Scan started on: {{scan_object.start_scan_date|date:"F j, Y h:i"}} -
    - Total time taken: - {% if scan_object.scan_status == 0 %} - {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} - {% elif scan_object.scan_status == 1 %} - {{ scan_object.get_elapsed_time }} - {% elif scan_object.scan_status == 2 %} - {% if scan_object.get_completed_time_in_sec < 60 %} - Completed in < 1 minutes {% else %} Completed in {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} {% endif %} {% elif scan_object.scan_status == 3 %} Aborted in - {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} {% endif %}
    - Report Generated on: {% now "F j, Y" %} -

    -
    -
    - -
    -

    Summary of Findings

    -
    +
    +
    +

    Summary of Vulnerabilities Identified

    - This section provides a summary of the findings. + Listed below are the vulnerabilities identified on {{ scan_object.domain.name }}

    -
    -

    Subdomains Breakdown by HTTP Status

    - - {% if show_vuln and unique_vulnerabilities %} -

    Vulnerabilities Breakdown by Severity

    - - {% endif %} - - -
    -

    Summary of Vulnerabilities Identified

    -

    Listed below are the vulnerabilities identified on {{scan_object.domain.name}}

    -
    - - - - - - - - {% for vulnerability in unique_vulnerabilities %} - - - - - - - {% endfor %} -
    #Vulnerability NameInstancesSeverity
    {{ forloop.counter }}{{vulnerability.name}}{{vulnerability.count}} - {% if vulnerability.severity == -1 %} - Unknown - {% elif vulnerability.severity == 0 %} - Informational - {% elif vulnerability.severity == 1 %} - Low - {% elif vulnerability.severity == 2 %} - Medium - {% elif vulnerability.severity == 3 %} - High - {% elif vulnerability.severity == 4 %} - Critical - {% endif %} -
    -
    - -
    - -
    -
    -

    END OF REPORT

    -
    -
    -
    - - - +
    + + + + + + + + {% for vulnerability in unique_vulnerabilities %} + + + + + + + {% endfor %} +
    #Vulnerability NameInstancesSeverity
    {{ forloop.counter }} + {{ vulnerability.name }} + {{ vulnerability.count }} + {% if vulnerability.severity == -1 %} + Unknown + {% elif vulnerability.severity == 0 %} + Informational + {% elif vulnerability.severity == 1 %} + Low + {% elif vulnerability.severity == 2 %} + Medium + {% elif vulnerability.severity == 3 %} + High + {% elif vulnerability.severity == 4 %} + Critical + {% endif %} +
    +
    + +
    +
    +

    END OF REPORT

    +
    +
    +
    + From 8a44b11db87ad656bc6c15684b3d80bede332cb2 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 25 Aug 2024 13:53:23 +0530 Subject: [PATCH 100/211] show interesting subdomain --- web/templates/report/modern.html | 124 +++++++++++++++++++++++-------- 1 file changed, 94 insertions(+), 30 deletions(-) diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index 7af7e3bc9..ed50ef394 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -469,7 +469,43 @@ margin-left: auto; margin-right: auto; width: 50%; - } + } + + {% comment %} interesting-subdomain-table {% endcomment %} + .interesting-subdomain-table-container { + width: 100%; + margin: auto auto; + background-color: #ffffff; + border-radius: 4mm; + overflow: hidden; + } + .interesting-subdomain-table { + width: 100%; + border-collapse: separate; + border-spacing: 0.5mm; + background-color: #ffffff; + } + .interesting-subdomain-table th, .interesting-subdomain-table td { + padding: 2.5mm 3mm; + text-align: center; + vertical-align: middle; + } + .interesting-subdomain-table th { + background-color: #696969; + color: white; + font-weight: bold; + } + .interesting-subdomain-table .page-title-td { + text-align: left !important; + } + .interesting-subdomain-table .subdomain-name-td { + text-align: left !important; + } + .status-td { + width: 10mm; + text-align: center; + font-weight: bold; + } @@ -650,47 +686,75 @@

    Vulnerabilities Breakdown by Severity

    src="data:image/png;base64,{{ vulns_severity_chart }}" /> {% endif %} -
    -

    Summary of Vulnerabilities Identified

    +
    +

    Interesting Subdomains

    - Listed below are the vulnerabilities identified on {{ scan_object.domain.name }} + Listed below are the interesting subdomains identified on {{ scan_object.domain.name }}

    -
    - +
    +
    - - - + + + - {% for vulnerability in unique_vulnerabilities %} - + {% for subdomain in interesting_subdomains %} + - - - + + {% endfor %}
    #Vulnerability NameInstancesSeveritySubdomainPage TitleHTTP Status
    {{ forloop.counter }} - {{ vulnerability.name }} - {{ vulnerability.count }} - {% if vulnerability.severity == -1 %} - Unknown - {% elif vulnerability.severity == 0 %} - Informational - {% elif vulnerability.severity == 1 %} - Low - {% elif vulnerability.severity == 2 %} - Medium - {% elif vulnerability.severity == 3 %} - High - {% elif vulnerability.severity == 4 %} - Critical - {% endif %} + {{ subdomain.name }}{{ subdomain.ip_address }} + {% if subdomain.http_status %}{{ subdomain.http_status }}{% endif %}
    + {% if show_vuln %} +
    +

    Summary of Vulnerabilities Identified

    +

    + Listed below are the vulnerabilities identified on {{ scan_object.domain.name }} +

    +
    + + + + + + + + {% for vulnerability in unique_vulnerabilities %} + + + + + + + {% endfor %} +
    #Vulnerability NameInstancesSeverity
    {{ forloop.counter }} + {{ vulnerability.name }} + {{ vulnerability.count }} + {% if vulnerability.severity == -1 %} + Unknown + {% elif vulnerability.severity == 0 %} + Informational + {% elif vulnerability.severity == 1 %} + Low + {% elif vulnerability.severity == 2 %} + Medium + {% elif vulnerability.severity == 3 %} + High + {% elif vulnerability.severity == 4 %} + Critical + {% endif %} +
    +
    +
    + {% endif %}

    END OF REPORT

    From d0cab4144ff5383517dde5f7dcff92e517cb1f04 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 25 Aug 2024 19:33:21 +0530 Subject: [PATCH 101/211] Add summary of findings section to modern report template --- web/templates/report/modern.html | 91 ++++++++++++++++---------------- 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index ed50ef394..644340b57 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -552,6 +552,10 @@

    Table of Contents

    Assessment Timeline
    +
    + Summary of Findings + +
    {% if show_vuln %}
    Vulnerability Summary @@ -685,9 +689,7 @@

    Vulnerabilities Breakdown by Severity

    {% endif %} -
    -
    -

    Interesting Subdomains

    +

    Interesting Subdomains

    Listed below are the interesting subdomains identified on {{ scan_object.domain.name }}

    @@ -700,10 +702,12 @@

    Interesting Subdomains

    HTTP Status {% for subdomain in interesting_subdomains %} - + {{ forloop.counter }} {{ subdomain.name }} - {{ subdomain.ip_address }} + + {% if subdomain.page_title %}{{ subdomain.page_title }}{% endif %} + {% if subdomain.http_status %}{{ subdomain.http_status }}{% endif %} @@ -713,47 +717,44 @@

    Interesting Subdomains

    {% if show_vuln %} -
    -

    Summary of Vulnerabilities Identified

    -

    - Listed below are the vulnerabilities identified on {{ scan_object.domain.name }} -

    -
    - - - - - - +

    Summary of Vulnerabilities Identified

    +

    + Listed below are the vulnerabilities identified on {{ scan_object.domain.name }} +

    +
    +
    #Vulnerability NameInstancesSeverity
    + + + + + + + {% for vulnerability in unique_vulnerabilities %} + + + + + - {% for vulnerability in unique_vulnerabilities %} - - - - - - - {% endfor %} -
    #Vulnerability NameInstancesSeverity
    {{ forloop.counter }} + {{ vulnerability.name }} + {{ vulnerability.count }} + {% if vulnerability.severity == -1 %} + Unknown + {% elif vulnerability.severity == 0 %} + Informational + {% elif vulnerability.severity == 1 %} + Low + {% elif vulnerability.severity == 2 %} + Medium + {% elif vulnerability.severity == 3 %} + High + {% elif vulnerability.severity == 4 %} + Critical + {% endif %} +
    {{ forloop.counter }} - {{ vulnerability.name }} - {{ vulnerability.count }} - {% if vulnerability.severity == -1 %} - Unknown - {% elif vulnerability.severity == 0 %} - Informational - {% elif vulnerability.severity == 1 %} - Low - {% elif vulnerability.severity == 2 %} - Medium - {% elif vulnerability.severity == 3 %} - High - {% elif vulnerability.severity == 4 %} - Critical - {% endif %} -
    -
    -
    + {% endfor %} + +
    {% endif %}
    From e763db4c9c33afef2df37307b98215f2fceb1f09 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 25 Aug 2024 19:58:26 +0530 Subject: [PATCH 102/211] Added discovered assets --- web/templates/report/modern.html | 144 +++++++++++++++++++++++-------- 1 file changed, 108 insertions(+), 36 deletions(-) diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index 644340b57..e9f877955 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -718,43 +718,115 @@

    Interesting Subdomains

    {% if show_vuln %}

    Summary of Vulnerabilities Identified

    -

    - Listed below are the vulnerabilities identified on {{ scan_object.domain.name }} -

    -
    - - - - - - - - {% for vulnerability in unique_vulnerabilities %} - - - - - + {% if all_vulnerabilities.count == 0 %} +

    + No vulnerabilities were identified on {{ scan_object.domain.name }} +

    + {% else %} +

    + Listed below are the vulnerabilities identified on {{ scan_object.domain.name }} +

    +
    +
    #Vulnerability NameInstancesSeverity
    {{ forloop.counter }} - {{ vulnerability.name }} - {{ vulnerability.count }} - {% if vulnerability.severity == -1 %} - Unknown - {% elif vulnerability.severity == 0 %} - Informational - {% elif vulnerability.severity == 1 %} - Low - {% elif vulnerability.severity == 2 %} - Medium - {% elif vulnerability.severity == 3 %} - High - {% elif vulnerability.severity == 4 %} - Critical - {% endif %} -
    + + + + + - {% endfor %} -
    #Vulnerability NameInstancesSeverity
    -
    + {% for vulnerability in unique_vulnerabilities %} + + {{ forloop.counter }} + + {{ vulnerability.name }} + + {{ vulnerability.count }} + + {% if vulnerability.severity == -1 %} + Unknown + {% elif vulnerability.severity == 0 %} + Informational + {% elif vulnerability.severity == 1 %} + Low + {% elif vulnerability.severity == 2 %} + Medium + {% elif vulnerability.severity == 3 %} + High + {% elif vulnerability.severity == 4 %} + Critical + {% endif %} + + + {% endfor %} + +
    + {% endif %} + {% endif %} + {% if show_recon %} +
    +

    Discovered Assets

    +

    This section provides a list of assets discovered during the reconnaissance phase.

    +

    Subdomains

    +

    During the reconnaissance phase, our subdomain enumeration process revealed:

    +
      +
    1. + Total Subdomains: {{ scan_object.get_subdomain_count }} +
        +
      • This extensive list provides a comprehensive view of the target's online footprint.
        • +
      +
      2. +
    2. + Active Subdomains: {{ subdomain_alive_count }} +
        +
      • These subdomains returned an HTTP status 200 (OK), indicating live web assets.
        • +
      +
      3. +
    3. + Interesting Subdomains: {{ interesting_subdomains.count }} +
        +
      • High-priority subdomains identified through keyword analysis (e.g., admin, api, test), suggesting a focused investigation.
        • +
      +
      4. +
    +

    + {{ scan_object.get_subdomain_count }} subdomains identified on {{ scan_object.domain.name }} +

    +
    + + + + + + + {% comment %} if show vuln is there we may also show total vulnerbaility count {% endcomment %} + {% if show_vuln %} + + {% endif %} + + {% for subdomain in subdomains %} + + + + + + {% if show_vuln %} + + {% endif %} + + {% endfor %} +
    #SubdomainPage TitleHTTP StatusVulnerabilities Count
    {{ forloop.counter }}{{ subdomain.name }} + {% if subdomain.page_title %}{{ subdomain.page_title }}{% endif %} + + {% if subdomain.http_status %}{{ subdomain.http_status }}{% endif %} + + {% if subdomain.get_total_vulnerability_count %} + {{ subdomain.get_total_vulnerability_count }} + {% else %} + 0 + {% endif %} +
    +
    +
    {% endif %}
    From 3f3da99c184f52df9152997d444362dea16baff4 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 25 Aug 2024 20:17:32 +0530 Subject: [PATCH 103/211] show ip assets --- web/templates/report/modern.html | 96 ++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 29 deletions(-) diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index e9f877955..209c19964 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -272,22 +272,7 @@ font-family: 'Inter', sans-serif; } - #summary-of-finding-page h2 { - font-size: 24pt; - color: {{primary_color}}; - } - - #executive-summary-page h2 { - font-size: 24pt; - color: {{primary_color}}; - } - - #quick-summary-page h2 { - font-size: 24pt; - color: {{primary_color}}; - } - - #vulnerability-breakdown-chart-page h2 { + .page-title { font-size: 24pt; color: {{primary_color}}; } @@ -470,6 +455,13 @@ margin-right: auto; width: 50%; } + {% comment %} stripped table {% endcomment %} + .table-stripped tr:nth-child(odd) { + background-color: #f9f9f9; + } + .table-stripped tr:nth-child(even) { + background-color: #f2f2f2; + } {% comment %} interesting-subdomain-table {% endcomment %} .interesting-subdomain-table-container { @@ -505,6 +497,10 @@ width: 10mm; text-align: center; font-weight: bold; + } + .normal-td { + width: 10mm; + text-align: center; } @@ -784,7 +780,9 @@

    Subdomains

  • Interesting Subdomains: {{ interesting_subdomains.count }}
      -
    • High-priority subdomains identified through keyword analysis (e.g., admin, api, test), suggesting a focused investigation.
      • +
    • + High-priority subdomains identified through keyword analysis (e.g., admin, api, test), suggesting a focused investigation. +
    • @@ -799,9 +797,7 @@

    Page Title HTTP Status {% comment %} if show vuln is there we may also show total vulnerbaility count {% endcomment %} - {% if show_vuln %} - Vulnerabilities Count - {% endif %} + {% if show_vuln %}Vulnerabilities Count{% endif %} {% for subdomain in subdomains %} @@ -826,13 +822,55 @@

    {% endfor %}

    + {% if ip_addresses.count > 0 %} +

    IP Assets

    +

    In addition to subdomains, various IP assets associated with the target infrastructure were also identified:

    +
      +
    1. + Total IP Addresses: {{ ip_addresses.count }} +
        +
      • + This represents the range of unique IP addresses associated with the discovered subdomains and other network assets. +
        • +
      +
      2. +
    +
    + + + + + + + + + {% for ip in ip_addresses %} + + + + + + + {% endfor %} +
    #IPOpen PortsGeo LocationRemarks
    {{ forloop.counter }}{{ ip.address }} + {% for port in ip.ports.all %} + {{ port.number }}/{{ port.service_name }} + {% if not forloop.last %},{% endif %} + {% endfor %} + + {% if ip.geo_iso %}{{ ip.geo_iso }}{% endif %} + + {% if ip.is_cdn %}CDN{% endif %} +
    +
    + {% endif %} +
    + {% endif %} +
    +
    +

    END OF REPORT

    +
    +
    - {% endif %} -
    -
    -

    END OF REPORT

    -
    -
    -
    - - + + From 3046c2e436bfc687ec60758b1dadf504e6908feb Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 25 Aug 2024 20:52:48 +0530 Subject: [PATCH 104/211] Added Recon result cards --- web/templates/report/modern.html | 145 ++++++++++++++++++++++++++----- 1 file changed, 121 insertions(+), 24 deletions(-) diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index 209c19964..602434415 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -502,6 +502,42 @@ width: 10mm; text-align: center; } + /* recon finding card table */ + .subdomain-card { + width: 100%; + margin-bottom: 3mm; + border: 1px solid #ddd; + border-radius: 4mm; + overflow: hidden; + box-shadow: 0 1mm 2mm rgba(0, 0, 0, 0.1); + } + .subdomain-card-header { + background-color: #f0f0f0; + padding: 2mm; + border-bottom: 1px solid #ddd; + display: flex; + justify-content: space-between; + align-items: center; + } + .subdomain-card-name { + font-weight: bold; + color: #2c3e50; + font-size: 11pt; + } + .subdomain-card-http-status { + padding: 1mm 2mm; + border-radius: 1mm; + font-size: 13pt; + } + .subdomain-card-content { + padding: 3mm; + } + .subdomain-card-page-title { + font-style: italic !important; + color: #7f8c8d; + margin-bottom: 1mm; + font-size: 10pt; + } @@ -686,31 +722,37 @@

    Vulnerabilities Breakdown by Severity

    src="data:image/png;base64,{{ vulns_severity_chart }}" /> {% endif %}

    Interesting Subdomains

    -

    - Listed below are the interesting subdomains identified on {{ scan_object.domain.name }} -

    -
    - - - - - - - - {% for subdomain in interesting_subdomains %} - - - - - + {% if interesting_subdomains %} +

    + Listed below are the interesting subdomains identified on {{ scan_object.domain.name }} +

    +
    +
    #SubdomainPage TitleHTTP Status
    {{ forloop.counter }}{{ subdomain.name }} - {% if subdomain.page_title %}{{ subdomain.page_title }}{% endif %} - - {% if subdomain.http_status %}{{ subdomain.http_status }}{% endif %} -
    + + + + + - {% endfor %} -
    #SubdomainPage TitleHTTP Status
    -
    + {% for subdomain in interesting_subdomains %} + + {{ forloop.counter }} + {{ subdomain.name }} + + {% if subdomain.page_title %}{{ subdomain.page_title }}{% endif %} + + + {% if subdomain.http_status %}{{ subdomain.http_status }}{% endif %} + + + {% endfor %} + +

    + {% else %} +

    + No interesting subdomains were identified on {{ scan_object.domain.name }} +

    + {% endif %} {% if show_vuln %}

    Summary of Vulnerabilities Identified

    @@ -866,6 +908,61 @@

    IP Assets

    {% endif %} {% endif %} + {% if show_recon %} +
    +

    Reconnaissance Findings

    +

    + This section contains list of all the subdomains identified during the reconnaissance phase. +

    + {% for subdomain in subdomains %} +
    +
    + {{ subdomain.name }} + {% if subdomain.http_status %} + {{ subdomain.http_status }} + {% endif %} +
    +
    + {% if subdomain.page_title %} +
    {{ subdomain.page_title }}
    + {% endif %} + {% if subdomain.ip_addresses.all %} +
    +

    IP Addresses:

    +
      + {% for ip in subdomain.ip_addresses.all %} +
    • + {{ip.address}} + {% if ip.ports.all %} +
        + {% for port in ip.ports.all %} +
      • {{ port.number }}/{{ port.service_name }}
        • + {% endfor %} +
      + {% endif %} +
      • + {% endfor %} +
    +
    + {% endif %} + {% if subdomain.get_vulnerabilities_without_info %} +
    +

    Vulnerabilities:

    + {% regroup subdomain.get_vulnerabilities_without_info by name as vuln_list %} + +
    + {% endif %} +
    +
    + {% endfor %} +
    + {% endif %}

    END OF REPORT

    From d9ed236537b3c972423554217fd8fe11ec20c808 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 28 Aug 2024 08:39:23 +0530 Subject: [PATCH 105/211] add vuln summary section with badges --- web/templates/report/modern.html | 298 ++++++++++++++++++++++++++----- 1 file changed, 250 insertions(+), 48 deletions(-) diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index 602434415..10c54d42c 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -449,6 +449,16 @@ .medium-td { background-color: #fff3e0; color: #ef6c00; } .high-td { background-color: #fbe9e7; color: #d84315; } .critical-td { background-color: #ffebee; color: #c62828; } + + .unknown-severity-badge { background-color: rgba(200, 200, 200, 0.2); } + .info-severity-badge { background-color: rgba(144, 202, 249, 0.2); } + .low-severity-badge { background-color: rgba(165, 214, 167, 0.2); } + .medium-severity-badge { background-color: rgba(255, 204, 128, 0.2); } + .high-severity-badge { background-color: rgba(239, 154, 154, 0.2); } + .critical-severity-badge { background-color: rgba(239, 154, 154, 0.2); } + + + .center-img { display: block; margin-left: auto; @@ -538,6 +548,96 @@ margin-bottom: 1mm; font-size: 10pt; } + {% comment %} vuln finding related css {% endcomment %} + .vuln-card { + background-color: #ffffff; + margin-bottom: 7mm; + border-radius: 4mm; + box-shadow: 0 2mm 4mm rgba(0,0,0,0.1); + overflow: hidden; + } + .vuln-card-header { + padding: 1mm 4mm; + position: relative; + display: flex; + align-items: center; + justify-content: space-between; + } + .card-header h4 { + font-size: 14pt; + margin: 0; + font-weight: 500; + } + .vuln-severity { + font-size: 11pt; + font-weight: bold; + padding: 1mm 3mm; + border-radius: 2mm; + } + .vuln-badge { + display: inline-block; + padding: 1mm 3mm; + border-radius: 2mm; + font-size: 9pt; + font-weight: bold; + text-transform: uppercase; + margin-right: 3mm; + margin-top: 1mm; + } + .badge-source { + background-color: #2ecc71; + color: white; + } + .badge-cvss { + background-color: #e74c3c; + color: white; + } + .badge-cve { + background-color: #3498db; + color: white; + } + + .badge-cwe { + background-color: #9b59b6; + color: white; + } + .vuln-card-body { + padding: 5mm 2mm; + } + .vuln-section { + margin-bottom: 4mm; + padding-bottom: 4mm; + border-bottom: 1pt solid #e0e0e0; + } + .vuln-section:last-child { + border-bottom: none; + } + .section-title { + font-weight: bold; + color: #2c3e50; + font-size: 11pt; + margin-bottom: 2mm; + text-transform: uppercase; + } + .vuln-cvss-details { + background-color: #f8f9fa; + padding: 4mm; + border-radius: 2mm; + font-family: monospace; + font-size: 9pt; + } + .vulnerable-url-li { + background-color: #fff3cd; + padding: 2mm 4mm; + margin-bottom: 2mm; + border-radius: 2mm; + font-family: monospace; + font-size: 9pt; + } + .references-li a { + color: #3498db; + text-decoration: none; + } @@ -908,61 +1008,163 @@

    IP Assets

    {% endif %}
    {% endif %} - {% if show_recon %} + {% if show_recon %}
    -

    Reconnaissance Findings

    -

    - This section contains list of all the subdomains identified during the reconnaissance phase. -

    - {% for subdomain in subdomains %} -
    -
    - {{ subdomain.name }} - {% if subdomain.http_status %} - {{ subdomain.http_status }} - {% endif %} -
    -
    - {% if subdomain.page_title %} -
    {{ subdomain.page_title }}
    - {% endif %} - {% if subdomain.ip_addresses.all %} -
    -

    IP Addresses:

    -
      - {% for ip in subdomain.ip_addresses.all %} -
    • - {{ip.address}} - {% if ip.ports.all %} +

      Reconnaissance Findings

      +

      This section contains list of all the subdomains identified during the reconnaissance phase.

      + {% for subdomain in subdomains %} +
      +
      + {{ subdomain.name }} + {% if subdomain.http_status %} + {{ subdomain.http_status }} + {% endif %} +
      +
      + {% if subdomain.page_title %}
      {{ subdomain.page_title }}
      {% endif %} + {% if subdomain.ip_addresses.all %} +
      +

      IP Addresses:

        - {% for port in ip.ports.all %} -
      • {{ port.number }}/{{ port.service_name }}
        • + {% for ip in subdomain.ip_addresses.all %} +
      • + {{ ip.address }} + {% if ip.ports.all %} +
          + {% for port in ip.ports.all %}
        • {{ port.number }}/{{ port.service_name }}
          • {% endfor %} +
        + {% endif %} +
        • {% endfor %}
      - {% endif %} -
      • - {% endfor %} -
    -
    - {% endif %} - {% if subdomain.get_vulnerabilities_without_info %} -
    -

    Vulnerabilities:

    - {% regroup subdomain.get_vulnerabilities_without_info by name as vuln_list %} - +
    + {% endif %} + {% if show_vuln %} + {% if subdomain.get_vulnerabilities_without_info %} +
    +

    Vulnerabilities:

    + {% regroup subdomain.get_vulnerabilities_without_info by name as vuln_list %} + +
    + {% endif %} + {% endif %}
    - {% endif %}
    -
    - {% endfor %} + {% endfor %} - {% endif %} + {% endif %} + {% if show_vuln %} +
    +

    Vulnerabilities Discovered

    +

    + This section details the security vulnerabilities identified during our penetration testing engagement. Each finding is documented with its description, potential impact, and recommended remediation steps. +
    + Vulnerabilities are categorized by severity (Critical, High, Medium, Low, Info) to prioritize remediation efforts. This assessment is based on the potential impact to confidentiality, integrity, and availability of the systems and data. +
    + The information presented here is crucial for understanding your current security posture and should guide your remediation strategy to enhance overall security.

    +
    + {% regroup all_vulnerabilities by get_path as grouped_vulnerabilities %} + {% for vulnerabilities in grouped_vulnerabilities %} + {% for vulnerability in vulnerabilities.list %} +
    +
    +

    + {{ vulnerability.name }} + {% if vulnerabilities.grouper %} +
    + in {{ vulnerabilities.grouper }} + {% endif %} +

    +
    + {% if vulnerability.severity == -1 %} + Unknown + {% elif vulnerability.severity == 0 %} + INFO + {% elif vulnerability.severity == 1 %} + LOW + {% elif vulnerability.severity == 2 %} + MEDIUM + {% elif vulnerability.severity == 3 %} + HIGH + {% elif vulnerability.severity == 4 %} + CRITICAL + {% endif %} +
    +
    +
    +
    + {{vulnerability.source|upper}} + {% if vulnerability.cvss_score %} + CVSS: {{vulnerability.cvss_score}} + {% endif %} + {% if vulnerability.cve_ids.all %} + {% for cve in vulnerability.cve_ids.all %}{{cve}}{% endfor %} + {% endif %} + {% if vulnerability.cwe_ids.all %} + {% for cwe in vulnerability.cwe_ids.all %}{{cwe}}{% endfor %} + {% endif %} +
    +
    +

    CVSS

    +
    CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H
    +
    +
    +

    Description

    +

    + A SQL injection vulnerability was discovered in the login form, allowing potential attackers to manipulate database queries. This vulnerability arises from improper input validation and could potentially affect multiple areas of the application where user input interacts with database operations. +

    +
    +
    +

    Impact

    +

    + This vulnerability could lead to unauthorized access, data theft, or manipulation of the database. An attacker could potentially: +

    +
      +
    • Bypass authentication mechanisms
      • +
    • Read, modify, or delete sensitive data
      • +
    • Execute administration operations on the database
      • +
    • Issue commands to the operating system
      • +
    +
    +
    +

    Remediation

    +

    To address this vulnerability, implement the following measures:

    +
      +
    • Use prepared statements or parameterized queries to prevent SQL injection attacks
      • +
    • Implement proper input validation and sanitization for all user inputs
      • +
    • Apply the principle of least privilege to database accounts used by the application
      • +
    • Regularly update and patch the database management system
      • +
    +
    +
    +

    Vulnerable URLs

    +
    https://example.com/login.php
    +
    https://example.com/user.php?id=1
    +
    +
    +

    References

    + +
    +
    +
    + {% endfor %} + {% endfor %} +
    + {% endif %}

    END OF REPORT

    From fd30b2745a17c9c41bee598dda816037350a92b2 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 28 Aug 2024 08:55:11 +0530 Subject: [PATCH 106/211] finish all the href links inside page --- web/templates/report/modern.html | 78 +++++++++++++++----------------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index 10c54d42c..a471ccdb7 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -582,7 +582,7 @@ font-weight: bold; text-transform: uppercase; margin-right: 3mm; - margin-top: 1mm; + margin-bottom: 2mm; } .badge-source { background-color: #2ecc71; @@ -638,6 +638,10 @@ color: #3498db; text-decoration: none; } + .no-style-a{ + color: #3498db; + text-decoration: none; + } @@ -688,16 +692,10 @@

    Table of Contents

    Summary of Findings
    - {% if show_vuln %} -
    - Vulnerability Summary - -
    - {% endif %} {% if show_recon %}
    - Reconnaissance Results - + Reconnaissance Findings +
    {% endif %} {% if show_vuln %} @@ -944,7 +942,7 @@

    {% for subdomain in subdomains %} {{ forloop.counter }} - {{ subdomain.name }} + {{ subdomain.name }} {% if subdomain.page_title %}{{ subdomain.page_title }}{% endif %} @@ -1013,7 +1011,7 @@

    IP Assets

    Reconnaissance Findings

    This section contains list of all the subdomains identified during the reconnaissance phase.

    {% for subdomain in subdomains %} -
    +
    {{ subdomain.name }} {% if subdomain.http_status %} @@ -1073,7 +1071,7 @@

    Vulnerabilities Discovered

    {% regroup all_vulnerabilities by get_path as grouped_vulnerabilities %} {% for vulnerabilities in grouped_vulnerabilities %} {% for vulnerability in vulnerabilities.list %} -
    +

    {{ vulnerability.name }} @@ -1102,63 +1100,59 @@

    {{vulnerability.source|upper}} {% if vulnerability.cvss_score %} - CVSS: {{vulnerability.cvss_score}} + CVSS: {{vulnerability.cvss_score|upper}} {% endif %} {% if vulnerability.cve_ids.all %} - {% for cve in vulnerability.cve_ids.all %}{{cve}}{% endfor %} + {% for cve in vulnerability.cve_ids.all %}{{cve|upper}}{% endfor %} {% endif %} {% if vulnerability.cwe_ids.all %} - {% for cwe in vulnerability.cwe_ids.all %}{{cwe}}{% endfor %} + {% for cwe in vulnerability.cwe_ids.all %}{{cwe|upper}}{% endfor %} {% endif %}
    + {% if vulnerability.cvss_metrics %}

    CVSS

    -
    CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H
    +
    {{vulnerability.cvss_metrics}}
    + {% endif %} + {% if vulnerability.description %}

    Description

    - A SQL injection vulnerability was discovered in the login form, allowing potential attackers to manipulate database queries. This vulnerability arises from improper input validation and could potentially affect multiple areas of the application where user input interacts with database operations. -

    -
    -
    -

    Impact

    -

    - This vulnerability could lead to unauthorized access, data theft, or manipulation of the database. An attacker could potentially: + {{vulnerability.description|linebreaks}}

    -
      -
    • Bypass authentication mechanisms
      • -
    • Read, modify, or delete sensitive data
      • -
    • Execute administration operations on the database
      • -
    • Issue commands to the operating system
      • -
    + {% endif %} + {% if vulnerability.impact %} +
    +

    Impact

    +

    + {{vulnerability.impact|linebreaks}} +

    +
    + {% endif %} + {% if vulnerability.remediation %}

    Remediation

    -

    To address this vulnerability, implement the following measures:

    -
      -
    • Use prepared statements or parameterized queries to prevent SQL injection attacks
      • -
    • Implement proper input validation and sanitization for all user inputs
      • -
    • Apply the principle of least privilege to database accounts used by the application
      • -
    • Regularly update and patch the database management system
      • -
    +

    {{vulnerability.remediation|linebreaks}}

    + {% endif %}

    Vulnerable URLs

    -
    https://example.com/login.php
    -
    https://example.com/user.php?id=1
    +
    {{vulnerability.http_url}}
    + {% if vulnerability.references.all %}

    References

    + {% endif %}

    {% endfor %} From 30f84f77be3cb4ba380d7f981d35b80902c11769 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 28 Aug 2024 09:05:07 +0530 Subject: [PATCH 107/211] Update dependencies for charts and sort interesting subdomains by HTTP status code util function --- web/reNgine/utilities.py | 13 + web/requirements.txt | 2 + web/startScan/views.py | 26 +- web/templates/report/modern.html | 2 +- web/templates/report/template.html | 1050 ---------------------------- 5 files changed, 41 insertions(+), 1052 deletions(-) delete mode 100644 web/templates/report/template.html diff --git a/web/reNgine/utilities.py b/web/reNgine/utilities.py index a3101625f..58a4503b0 100644 --- a/web/reNgine/utilities.py +++ b/web/reNgine/utilities.py @@ -159,3 +159,16 @@ def is_out_of_scope(self, subdomain): if subdomain in self.plain_patterns: return True return any(pattern.search(subdomain) for pattern in self.regex_patterns) + + +def sorting_key(subdomain): + # sort subdomains based on their http status code with priority 200 < 300 < 400 < rest + status = subdomain['http_status'] + if 200 <= status <= 299: + return 1 + elif 300 <= status <= 399: + return 2 + elif 400 <= status <= 499: + return 3 + else: + return 4 \ No newline at end of file diff --git a/web/requirements.txt b/web/requirements.txt index 9b53e163f..6fff50162 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -40,3 +40,5 @@ weasyprint==53.3 wafw00f==2.2.0 xmltodict==0.13.0 django-environ==0.11.2 +plotly==5.23.0 +kaleido \ No newline at end of file diff --git a/web/startScan/views.py b/web/startScan/views.py index 3e88d2a03..4e163060b 100644 --- a/web/startScan/views.py +++ b/web/startScan/views.py @@ -4,7 +4,7 @@ from weasyprint import HTML, CSS from datetime import datetime from django.contrib import messages -from django.db.models import Count +from django.db.models import Count, Case, When, IntegerField from django.http import HttpResponse, HttpResponseRedirect, JsonResponse from django.shortcuts import get_object_or_404, render from django.template.loader import get_template @@ -13,6 +13,7 @@ from django_celery_beat.models import (ClockedSchedule, IntervalSchedule, PeriodicTask) from rolepermissions.decorators import has_permission_decorator + from reNgine.celery import app from reNgine.charts import * from reNgine.common_func import * @@ -1017,6 +1018,29 @@ def create_report(request, id): .count() ) interesting_subdomains = get_interesting_subdomains(scan_history=id) + interesting_subdomains = interesting_subdomains.annotate( + sort_order=Case( + When(http_status__gte=200, http_status__lt=300, then=1), + When(http_status__gte=300, http_status__lt=400, then=2), + When(http_status__gte=400, http_status__lt=500, then=3), + default=4, + output_field=IntegerField(), + ) + ).order_by('sort_order', 'http_status') + + subdomains = subdomains.annotate( + sort_order=Case( + When(http_status__gte=200, http_status__lt=300, then=1), + When(http_status__gte=300, http_status__lt=400, then=2), + When(http_status__gte=400, http_status__lt=500, then=3), + default=4, + output_field=IntegerField(), + ) + ).order_by('sort_order', 'http_status') + + + + ip_addresses = ( IpAddress.objects .filter(ip_addresses__in=subdomains) diff --git a/web/templates/report/modern.html b/web/templates/report/modern.html index a471ccdb7..eb13345ed 100644 --- a/web/templates/report/modern.html +++ b/web/templates/report/modern.html @@ -2,7 +2,7 @@ - Penetration Testing Report + Report - - - -
    -

    {{report_name}} -
    - {{scan_object.domain.name}} -
    - {# generated date #} - {% now "F j, Y" %} -

    -
    - {{company_name}} - {{company_address}} -
    -
    - {{company_email}} - {{company_website}} -
    -
    - {% if show_rengine_banner %}Generated by reNgine - https://github.com/yogeshojha/rengine - {% endif %} -
    -
    - -
    -

     

    -

    Table of contents

    -
      - {% if show_executive_summary %} -
      • - {% endif %} - -
      • -
      • - - {% if interesting_subdomains and show_recon %} -
      • - {% endif %} - - {% if all_vulnerabilities.count > 0 and show_vuln %} -
      • - {% endif %} - - {% if show_recon %} -
      • - {% endif %} - - {% if all_vulnerabilities.count > 0 and show_vuln %} -
      • - {% endif %} -
    -
    - - {% if show_executive_summary %} -
    -

    Executive summary

    -
    - {{executive_summary_description | safe }} -
    - {% endif %} - -
    -

    Quick Summary

    -

    This section contains quick summary of scan performed on {{scan_object.domain.name}}

    -
    -
    - - {# recon section #} - {% if show_recon %} -

    Reconnaissance

    -
    -
    -

    Subdomains -
    - - {{scan_object.get_subdomain_count}} - -

    -
    -
    -

    Endpoints -
    - - {{scan_object.get_endpoint_count}} - -

    -
    -
    -

    Vulnerabilities -
    - - {{all_vulnerabilities_count}} - -

    -
    -
    - {% endif %} - - - {% if show_vuln %} -
    -
    -

    Vulnerability Summary

    -
    -
    -

    Critical -
    - - {{scan_object.get_critical_vulnerability_count}} - -

    -
    -
    -

    High -
    - - {{scan_object.get_high_vulnerability_count}} - -

    -
    -
    -

    Medium -
    - - {{scan_object.get_medium_vulnerability_count}} - -

    -
    -
    -

    Low -
    - - {{scan_object.get_low_vulnerability_count}} - -

    -
    -
    -

    Info -
    - - {% if is_ignore_info_vuln %} - 0 - {% else %} - {{scan_object.get_info_vulnerability_count}} - {% endif %} - -

    -
    -
    -

    Unknown -
    - - {{scan_object.get_unknown_vulnerability_count}} - -

    -
    -
    -
    - {% endif %} - -
    -

    Timeline of the Assessment

    -

    - Scan started on: {{scan_object.start_scan_date|date:"F j, Y h:i"}} -
    - Total time taken: - {% if scan_object.scan_status == 0 %} - {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} - {% elif scan_object.scan_status == 1 %} - {{ scan_object.get_elapsed_time }} - {% elif scan_object.scan_status == 2 %} - {% if scan_object.get_completed_time_in_sec < 60 %} - Completed in < 1 minutes {% else %} Completed in {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} {% endif %} {% elif scan_object.scan_status == 3 %} Aborted in - {{ scan_object.start_scan_date|timesince:scan_object.stop_scan_date }} {% endif %}
    - Report Generated on: {% now "F j, Y" %} -

    -
    - - {# show interesting_subdomains section only when show_recon result is there #} - {% if interesting_subdomains and show_recon %} -
    -

    Interesting Recon Data

    -

    Listed below are the {{interesting_subdomains.count}} interesting subdomains identified on {{scan_object.domain.name}}

    -
    -
    -
    - # -
    -
    - Subdomain -
    -
    - Page title -
    -
    - HTTP Status -
    -
    - {% for subdomain in interesting_subdomains %} -
    -
    - {{ forloop.counter }} -
    -
    - {{subdomain.name}} -
    -
    - {% if subdomain.page_title %} - {{subdomain.page_title}} - {% else %} -     - {% endif %} -
    -
    - {% if subdomain.http_status %} - {{subdomain.http_status}} - {% else %} -     - {% endif %} -
    -
    - {% endfor %} -
    -
    - {% endif %} - - {# vulnerability_summary only when vuln_report #} - {% if show_vuln %} -
    -

    Summary of Vulnerabilities Identified

    - {% if all_vulnerabilities.count > 0 %} -

    Listed below are the vulnerabilities identified on {{scan_object.domain.name}}

    -
    -
    -
    - # -
    -
    - Vulnerability Name -
    -
    - Times Identified -
    -
    - Severity -
    -
    - {% for vulnerability in unique_vulnerabilities %} -
    -
    - {{ forloop.counter }} -
    -
    - {{vulnerability.name}} -
    -
    - {{vulnerability.count}} -
    - {% if vulnerability.severity == -1 %} -
    - Unknown - {% elif vulnerability.severity == 0 %} -
    - Informational - {% elif vulnerability.severity == 1 %} -
    - Low - {% elif vulnerability.severity == 2 %} -
    - Medium - {% elif vulnerability.severity == 3 %} -
    - High - {% elif vulnerability.severity == 4 %} -
    - Critical - {% endif %} -
    -
    - {% endfor %} - {% else %} -

    No Vulnerabilities were Discovered.

    - {% endif %} -
    - -
    - {% endif %} - - {# show discovered assets only for show_recon report #} - {% if show_recon %} -
    -

    Discovered Assets

    -

    Subdomains

    -

    - During the reconnaissance phase, {{scan_object.get_subdomain_count}} subdomains were discovered. - Out of {{scan_object.get_subdomain_count}} subdomains, {{subdomain_alive_count}} returned HTTP status 200. - {{interesting_subdomains.count}} interesting subdomains were also identified based on the interesting keywords used. -

    -

    {{scan_object.get_subdomain_count}} subdomains identified on {{scan_object.domain.name}}

    -
    -
    -
    - Subdomain -
    -
    - Page title -
    -
    - HTTP Status -
    -
    - {% for subdomain in subdomains %} -
    -
    - {{subdomain.name}} -
    -
    - {% if subdomain.page_title %} - {{subdomain.page_title}} - {% endif %} -
    -
    - {{subdomain.http_status}} -
    -
    - {% endfor %} -
    - {% if ip_addresses.count %} -

    IP Addresses

    -

    {{ip_addresses.count}} IP Addresses were identified on {{scan_object.domain.name}}

    -
    -
    -
    - IP -
    -
    - Open Ports -
    -
    - Remarks -
    -
    - {% for ip in ip_addresses %} -
    -
    - {{ip.address}} -
    -
    - {% for port in ip.ports.all %} - {{port.number}}/{{port.service_name}}{% if not forloop.last %},{% endif %} - {% endfor %} -
    - {% if ip.is_cdn %} -
    - CDN IP Address - {% else %} -
    - {% endif %} -
    -
    - {% endfor %} -
    - {% endif %} -
    -
    - {% endif %} - - {# reconnaissance finding only when show_recon #} - {% if show_recon %} -
    -

    Reconnaissance Findings

    - {% for subdomain in subdomains %} - - - - - {% if subdomain.http_status == 200 %} - - {% elif subdomain.http_status >= 300 and subdomain.http_status < 400 %} - - {% elif subdomain.http_status >= 400 %} - - {% elif subdomain.http_status == 0 %} - - {% else %} - - {% endif %} - - {% if subdomain.page_title %} - - - - {% endif %} - {% if subdomain.ip_addresses.all %} - - - - {% endif %} - {% if subdomain.get_vulnerabilities_without_info %} - - - - {% endif %} -
    {{ forloop.counter }}.{{subdomain.name}}{{subdomain.http_status}}{{subdomain.http_status}}{{subdomain.http_status}}N/A{{subdomain.http_status}}
    Page Title: {{subdomain.page_title}}
    - IP Address: -
      - {% for ip in subdomain.ip_addresses.all %} -
    • {{ip.address}} - {% if ip.ports.all %} -
        -
      • Open Ports:   - {% for port in ip.ports.all %} - {{port.number}}/{{port.service_name}}{% if not forloop.last %},{% endif %} - {% endfor %} -
        • -
      - {% endif %} -
      • - {% endfor %} -
    -
    - Vulnerabilities - {% regroup subdomain.get_vulnerabilities_without_info by name as vuln_list %} - -
    - {% endfor %} -
    - {% endif %} - - {% if all_vulnerabilities.count > 0 and show_vuln %} -
    -

    Vulnerabilities Discovered

    -

    - This section reports the security issues found during the audit. -
    - A Total of {{scan_object.get_vulnerability_count}} were discovered in {{scan_object.domain.name}}, - {{scan_object.get_critical_vulnerability_count}} of them were Critical, - {{scan_object.get_high_vulnerability_count}} of them were High Severity, - {{scan_object.get_medium_vulnerability_count}} of them were Medium severity, - {% if is_ignore_info_vuln %}0{% else %}{{scan_object.get_info_vulnerability_count}}{% endif %} of them were Low severity, and - {{scan_object.get_info_vulnerability_count}} of them were Informational. - {{scan_object.get_unknown_vulnerability_count}} of them were Unknown Severity. -

    -

    Vulnerability Breakdown by Severity

    -
    -
    -

    Critical -
    - - {{scan_object.get_critical_vulnerability_count}} - -

    -
    -
    -

    High -
    - - {{scan_object.get_high_vulnerability_count}} - -

    -
    -
    -

    Medium -
    - - {{scan_object.get_medium_vulnerability_count}} - -

    -
    -
    -

    Low -
    - - {{scan_object.get_low_vulnerability_count}} - -

    -
    -
    -

    Info -
    - - {% if is_ignore_info_vuln %} - 0 - {% else %} - {{scan_object.get_info_vulnerability_count}} - {% endif %} - -

    -
    -
    -

    Unknown -
    - - {{scan_object.get_unknown_vulnerability_count}} - -

    -
    -
    -
    - - - {# start vulnerability #} - {% if show_vuln %} -
    - {% regroup all_vulnerabilities by get_path as grouped_vulnerabilities %} - {% for vulnerabilities in grouped_vulnerabilities %} - {% for vulnerability in vulnerabilities.list %} -
    -

    - {{vulnerability.name}} - {% if vulnerabilities.grouper %} -
    in {{vulnerabilities.grouper}} - {% endif %} -     - {% if vulnerability.severity == -1 %} - Unknown -
    - {% elif vulnerability.severity == 0 %} - INFO -
    - {% elif vulnerability.severity == 1 %} - LOW -
    - {% elif vulnerability.severity == 2 %} - MEDIUM -
    - {% elif vulnerability.severity == 3 %} - HIGH -
    - {% elif vulnerability.severity == 4 %} - CRITICAL -
    - {% endif %} -

    - - Vulnerability Source: {{vulnerability.source|upper}}
    - {% if vulnerability.cvss_metrics or vulnerability.cvss_score or vulnerability.cve_ids.all or vulnerability.cve_ids.all %} - Vulnerability Classification
    - {% if vulnerability.cvss_metrics %} - CVSS Metrics: {{vulnerability.cvss_metrics}} - {% endif %} - {% if vulnerability.cvss_score %} -
    - CVSS Score: {{vulnerability.cvss_score}} - {% endif %} - {% if vulnerability.cve_ids.all %} -
    - CVE IDs
    -     {% for cve in vulnerability.cve_ids.all %} {{cve}}{% if not forloop.last %}, {% endif %} {% endfor %} - {% endif %} - {% if vulnerability.cwe_ids.all %} -
    - CWE IDs
    -     {% for cwe in vulnerability.cwe_ids.all %} {{cwe}}{% if not forloop.last %}, {% endif %} {% endfor %} - {% endif %} -
    - {% endif %} - {% if vulnerability.description %} -
    - Description
    - {{vulnerability.description|linebreaks}} - {% endif %} - {% if vulnerability.impact %} -
    - Impact
    - {{vulnerability.impact|linebreaks}} - {% endif %} - {% if vulnerability.remediation %} -
    - Remediation
    - {{vulnerability.remediation|linebreaks}} - {% endif %} -
    - Vulnerable URL(s)
    - - - - {% if vulnerability.references.all %} - References
    -
      - {% for ref in vulnerability.references.all %} -
    • - {{ref}} -
      • - {% endfor %} -
    - {% endif %} -
    -
    -
    - {% endfor %} - {% endfor %} -
    - {% endif %} - - {% endif %} -
    -

    END OF REPORT

    -
    - - - From 0a634e72b70eba3fbbb372910aad4fa8646095d7 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 28 Aug 2024 19:34:36 +0530 Subject: [PATCH 108/211] remove unused svg files --- web/templates/report/heading.svg | 26 ----------------------- web/templates/report/style.svg | 29 -------------------------- web/templates/report/table-content.svg | 21 ------------------- 3 files changed, 76 deletions(-) delete mode 100644 web/templates/report/heading.svg delete mode 100644 web/templates/report/style.svg delete mode 100644 web/templates/report/table-content.svg diff --git a/web/templates/report/heading.svg b/web/templates/report/heading.svg deleted file mode 100644 index 7e6bbb79a..000000000 --- a/web/templates/report/heading.svg +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/web/templates/report/style.svg b/web/templates/report/style.svg deleted file mode 100644 index 3930090fe..000000000 --- a/web/templates/report/style.svg +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/web/templates/report/table-content.svg b/web/templates/report/table-content.svg deleted file mode 100644 index 9961a203b..000000000 --- a/web/templates/report/table-content.svg +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - From e3e6409d700c28fd328234d796ebc6cc93b4fbfd Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 28 Aug 2024 20:37:30 +0530 Subject: [PATCH 109/211] change modal for downloading report --- .../templates/startScan/history.html | 183 +++++++++++------- 1 file changed, 109 insertions(+), 74 deletions(-) diff --git a/web/startScan/templates/startScan/history.html b/web/startScan/templates/startScan/history.html index d15529372..7ed670269 100644 --- a/web/startScan/templates/startScan/history.html +++ b/web/startScan/templates/startScan/history.html @@ -197,7 +197,7 @@

    Filters

    {% endif %} {% if scan.scan_status != -1%} - +  Scan Report {% endif %}
    @@ -213,30 +213,87 @@

    Filters

    -     +
    + +

    Chaos keys will be used for subdomain enumeration and recon data for Public Bug Bounty Programs.

    +
    + {% if chaos_key %} + + {% else %} + + {% endif %} +
    + +
    +
    + This is optional but recommended. Get your API key from https://cloud.projectdiscovery.io +
    +
    -
    @@ -70,4 +85,4 @@ {% block page_level_script %} -{% endblock page_level_script %} +{% endblock page_level_script %} \ No newline at end of file From 596dddf9390e59403c646d35d16fa8bc8d62f003 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sat, 31 Aug 2024 19:26:57 +0530 Subject: [PATCH 134/211] add and save chaos key funcs --- web/api/views.py | 6 ++++- web/dashboard/admin.py | 1 + web/dashboard/migrations/0008_chaosapikey.py | 20 ++++++++++++++++ web/dashboard/models.py | 8 +++++++ web/dashboard/views.py | 24 +++++++++++++++----- web/reNgine/common_func.py | 6 +++++ web/scanEngine/views.py | 12 ++++++++++ web/static/custom/custom.css | 5 ++++ 8 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 web/dashboard/migrations/0008_chaosapikey.py diff --git a/web/api/views.py b/web/api/views.py index 740967753..1bcd909d2 100644 --- a/web/api/views.py +++ b/web/api/views.py @@ -1123,7 +1123,11 @@ def get(self, request): version_number = None _, stdout = run_command(tool.version_lookup_command) - version_number = re.search(re.compile(tool.version_match_regex), str(stdout)) + if tool.version_match_regex: + version_number = re.search(re.compile(tool.version_match_regex), str(stdout)) + else: + version_match_regex = r'(?i:v)?(\d+(?:\.\d+){2,})' + version_number = re.search(version_match_regex, str(stdout)) if not version_number: return Response({'status': False, 'message': 'Invalid version lookup command.'}) diff --git a/web/dashboard/admin.py b/web/dashboard/admin.py index 881b18acb..19aa70806 100644 --- a/web/dashboard/admin.py +++ b/web/dashboard/admin.py @@ -5,4 +5,5 @@ admin.site.register(Project) admin.site.register(OpenAiAPIKey) admin.site.register(NetlasAPIKey) +admin.site.register(ChaosAPIKey) admin.site.register(InAppNotification) \ No newline at end of file diff --git a/web/dashboard/migrations/0008_chaosapikey.py b/web/dashboard/migrations/0008_chaosapikey.py new file mode 100644 index 000000000..0ffdbb929 --- /dev/null +++ b/web/dashboard/migrations/0008_chaosapikey.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.23 on 2024-08-31 12:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dashboard', '0007_auto_20240831_0608'), + ] + + operations = [ + migrations.CreateModel( + name='ChaosAPIKey', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('key', models.CharField(max_length=500)), + ], + ), + ] diff --git a/web/dashboard/models.py b/web/dashboard/models.py index a43152b42..a59bc1e2a 100644 --- a/web/dashboard/models.py +++ b/web/dashboard/models.py @@ -41,6 +41,14 @@ class NetlasAPIKey(models.Model): def __str__(self): return self.key + + +class ChaosAPIKey(models.Model): + id = models.AutoField(primary_key=True) + key = models.CharField(max_length=500) + + def __str__(self): + return self.key class InAppNotification(models.Model): diff --git a/web/dashboard/views.py b/web/dashboard/views.py index 11c688bfc..4573b1f2e 100644 --- a/web/dashboard/views.py +++ b/web/dashboard/views.py @@ -319,6 +319,13 @@ def onboarding(request): context = {} error = '' + # check is any projects exists, then redirect to project list else onboarding + # project = Project.objects.first() + + # if project: + # slug = project.slug + # return HttpResponseRedirect(reverse('dashboardIndex', kwargs={'slug': slug})) + if request.method == "POST": project_name = request.POST.get('project_name') slug = slugify(project_name) @@ -327,6 +334,7 @@ def onboarding(request): create_user_role = request.POST.get('create_user_role') key_openai = request.POST.get('key_openai') key_netlas = request.POST.get('key_netlas') + key_chaos = request.POST.get('key_chaos') insert_date = timezone.now() @@ -369,15 +377,19 @@ def onboarding(request): else: NetlasAPIKey.objects.create(key=key_netlas) + if key_chaos: + chaos_api_key = ChaosAPIKey.objects.first() + if chaos_api_key: + chaos_api_key.key = key_chaos + chaos_api_key.save() + else: + ChaosAPIKey.objects.create(key=key_chaos) + context['error'] = error - # check is any projects exists, then redirect to project list else onboarding - project = Project.objects.first() + context['openai_key'] = OpenAiAPIKey.objects.first() context['netlas_key'] = NetlasAPIKey.objects.first() - - if project: - slug = project.slug - return HttpResponseRedirect(reverse('dashboardIndex', kwargs={'slug': slug})) + context['chaos_key'] = ChaosAPIKey.objects.first() return render(request, 'dashboard/onboarding.html', context) diff --git a/web/reNgine/common_func.py b/web/reNgine/common_func.py index ce50b0e5b..dae1e8f2d 100644 --- a/web/reNgine/common_func.py +++ b/web/reNgine/common_func.py @@ -1034,6 +1034,12 @@ def get_netlas_key(): netlas_key = NetlasAPIKey.objects.all() return netlas_key[0] if netlas_key else None + +def get_chaos_key(): + chaos_key = ChaosAPIKey.objects.all() + return chaos_key[0] if chaos_key else None + + def parse_llm_vulnerability_report(report): report = report.replace('**', '') data = {} diff --git a/web/scanEngine/views.py b/web/scanEngine/views.py index 08433ec44..80845d984 100644 --- a/web/scanEngine/views.py +++ b/web/scanEngine/views.py @@ -503,6 +503,7 @@ def api_vault(request, slug): if request.method == "POST": key_openai = request.POST.get('key_openai') key_netlas = request.POST.get('key_netlas') + key_chaos = request.POST.get('key_chaos') if key_openai: @@ -521,10 +522,21 @@ def api_vault(request, slug): else: NetlasAPIKey.objects.create(key=key_netlas) + if key_chaos: + chaos_api_key = ChaosAPIKey.objects.first() + if chaos_api_key: + chaos_api_key.key = key_chaos + chaos_api_key.save() + else: + ChaosAPIKey.objects.create(key=key_chaos) + openai_key = OpenAiAPIKey.objects.first() netlas_key = NetlasAPIKey.objects.first() + chaos_key = ChaosAPIKey.objects.first() + context['openai_key'] = openai_key context['netlas_key'] = netlas_key + context['chaos_key'] = chaos_key return render(request, 'scanEngine/settings/api.html', context) diff --git a/web/static/custom/custom.css b/web/static/custom/custom.css index 47fee0e0b..0f28c7916 100644 --- a/web/static/custom/custom.css +++ b/web/static/custom/custom.css @@ -581,4 +581,9 @@ mark{ .notification-bell-icon.buzz { animation: notification-buzz 0.8s ease-out; +} + + +.optional-text { + margin-top: 4px; } \ No newline at end of file From b841b00873c4a7fdde89cd0ecdeb46e6a7f9aa54 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sat, 31 Aug 2024 19:27:08 +0530 Subject: [PATCH 135/211] use chaos as subdomain enum tool --- web/reNgine/tasks.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 81c328f7f..5e934803f 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -497,6 +497,15 @@ def subdomain_discovery( cmd_extract = f"grep -oE '([a-zA-Z0-9]([-a-zA-Z0-9]*[a-zA-Z0-9])?\.)+{host}'" cmd += f' | {cmd_extract} > {results_file}' + elif tool == 'chaos': + # we need to find api key if not ignore + chaos_key = get_chaos_key() + if not chaos_key: + logger.error('Chaos API key not found. Skipping.') + continue + results_file = self.results_dir + '/subdomains_chaos.txt' + cmd = f'chaos -d {host} -silent -key {chaos_key} -o {results_file}' + elif tool in custom_subdomain_tools: tool_query = InstalledExternalTool.objects.filter(name__icontains=tool.lower()) if not tool_query.exists(): From e75603f0dfe82c4d550a6f28d205e4dae9c63975 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sat, 31 Aug 2024 19:27:23 +0530 Subject: [PATCH 136/211] remove debug code --- web/dashboard/views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/dashboard/views.py b/web/dashboard/views.py index 4573b1f2e..8802314b8 100644 --- a/web/dashboard/views.py +++ b/web/dashboard/views.py @@ -320,11 +320,11 @@ def onboarding(request): error = '' # check is any projects exists, then redirect to project list else onboarding - # project = Project.objects.first() + project = Project.objects.first() - # if project: - # slug = project.slug - # return HttpResponseRedirect(reverse('dashboardIndex', kwargs={'slug': slug})) + if project: + slug = project.slug + return HttpResponseRedirect(reverse('dashboardIndex', kwargs={'slug': slug})) if request.method == "POST": project_name = request.POST.get('project_name') From beab91f6a7211a0b95267d5c31e205e333b1ad23 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sat, 31 Aug 2024 19:34:02 +0530 Subject: [PATCH 137/211] Added chaos as external tool --- web/fixtures/external_tools.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/web/fixtures/external_tools.yaml b/web/fixtures/external_tools.yaml index 0c2994b64..9c56b4d24 100644 --- a/web/fixtures/external_tools.yaml +++ b/web/fixtures/external_tools.yaml @@ -329,3 +329,20 @@ is_github_cloned: false github_clone_path: null subdomain_gathering_command: null +- model: scanEngine.installedexternaltool + pk: 19 + fields: + logo_url: null + name: chaos + description: Go client to communicate with Project Discovery's Chaos dataset API. + github_url: https://github.com/projectdiscovery/chaos-client + license_url: https://github.com/projectdiscovery/chaos-client/blob/main/LICENSE.md + version_lookup_command: chaos -version + update_command: chaos -up + install_command: go install -v github.com/projectdiscovery/chaos-client/cmd/chaos@latest + version_match_regex: (?i:v)?(\d+(?:\.\d+){2,}) + is_default: true + is_subdomain_gathering: true + is_github_cloned: false + github_clone_path: null + subdomain_gathering_command: null From 6fc9e3e3955c8d1fe9de5ecb8042dd4118dbe1a0 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sat, 31 Aug 2024 19:44:29 +0530 Subject: [PATCH 138/211] Added chaos in default_scan_engines fixture --- web/fixtures/default_scan_engines.yaml | 64 +++++++++++++------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/web/fixtures/default_scan_engines.yaml b/web/fixtures/default_scan_engines.yaml index 6194b4585..6d481f64d 100644 --- a/web/fixtures/default_scan_engines.yaml +++ b/web/fixtures/default_scan_engines.yaml @@ -2,11 +2,11 @@ pk: 1 fields: engine_name: Full Scan - yaml_configuration: "subdomain_discovery: {\r\n 'uses_tools': ['subfinder', 'ctfr', - 'sublist3r', 'tlsx', 'oneforall', 'netlas'],\r\n 'enable_http_crawl': true,\r\n - \ 'threads': 30,\r\n 'timeout': 5,\r\n}\r\nhttp_crawl: {}\r\nport_scan: {\r\n - \ 'enable_http_crawl': true,\r\n 'timeout': 5,\r\n # 'exclude_ports': [],\r\n - \ # 'exclude_subdomains': [],\r\n 'ports': ['top-100'],\r\n 'rate_limit': + yaml_configuration: "subdomain_discovery: {\r\n 'uses_tools': ['subfinder', 'chaos', + 'ctfr', 'sublist3r', 'tlsx', 'oneforall', 'netlas'],\r\n 'enable_http_crawl': + true,\r\n 'threads': 30,\r\n 'timeout': 5,\r\n}\r\nhttp_crawl: {}\r\nport_scan: + {\r\n 'enable_http_crawl': true,\r\n 'timeout': 5,\r\n # 'exclude_ports': + [],\r\n # 'exclude_subdomains': [],\r\n 'ports': ['top-100'],\r\n 'rate_limit': 150,\r\n 'threads': 30,\r\n 'passive': false,\r\n # 'use_naabu_config': false,\r\n \ # 'enable_nmap': true,\r\n # 'nmap_cmd': '',\r\n # 'nmap_script': '',\r\n \ # 'nmap_script_args': ''\r\n}\r\nosint: {\r\n 'discover': [\r\n 'emails',\r\n @@ -26,14 +26,15 @@ 'page_title'],\r\n 'enable_http_crawl': true,\r\n 'gf_patterns': ['debug_logic', 'idor', 'interestingEXT', 'interestingparams', 'interestingsubs', 'lfi', 'rce', 'redirect', 'sqli', 'ssrf', 'ssti', 'xss'],\r\n 'ignore_file_extensions': ['png', - 'jpg', 'jpeg', 'gif', 'mp4', 'mpeg', 'mp3'],\r\n 'threads': 30\r\n}\r\nvulnerability_scan: {\r\n - \ 'run_nuclei': true,\r\n 'run_dalfox': true,\r\n 'run_crlfuzz': true,\r\n + 'jpg', 'jpeg', 'gif', 'mp4', 'mpeg', 'mp3'],\r\n 'threads': 30\r\n}\r\nvulnerability_scan: + {\r\n 'run_nuclei': true,\r\n 'run_dalfox': true,\r\n 'run_crlfuzz': true,\r\n \ 'enable_http_crawl': true,\r\n 'concurrency': 50,\r\n 'intensity': 'normal',\r\n \ 'rate_limit': 150,\r\n 'retries': 1,\r\n 'timeout': 5,\r\n 'fetch_gpt_report': - true,\r\n 'nuclei': {\r\n 'use_nuclei_config': false,\r\n 'severities': ['unknown', - 'info', 'low', 'medium', 'high', 'critical']\r\n }\r\n}\r\nwaf_detection: {\r\n\r\n}\r\nscreenshot: - {\r\n 'enable_http_crawl': true,\r\n 'intensity': 'normal',\r\n 'timeout': - 10,\r\n 'threads': 40\r\n}\r\n\r\n# custom_headers: [\"Cookie: Test\"]" + true,\r\n 'nuclei': {\r\n 'use_nuclei_config': false,\r\n 'severities': + ['unknown', 'info', 'low', 'medium', 'high', 'critical']\r\n }\r\n}\r\nwaf_detection: + {\r\n\r\n}\r\nscreenshot: {\r\n 'enable_http_crawl': true,\r\n 'intensity': + 'normal',\r\n 'timeout': 10,\r\n 'threads': 40\r\n}\r\n\r\n# custom_headers: + [\"Cookie: Test\"]" default_engine: true - model: scanEngine.enginetype pk: 2 @@ -41,8 +42,8 @@ engine_name: Subdomain Scan yaml_configuration: "subdomain_discovery: {\r\n 'uses_tools': [\r\n 'subfinder', \r\n 'ctfr', \r\n 'sublist3r', \r\n 'tlsx', \r\n 'oneforall', \r\n - \ 'netlas'\r\n ],\r\n 'enable_http_crawl': true,\r\n 'threads': 30,\r\n - \ 'timeout': 5,\r\n}\r\nhttp_crawl: {}" + \ 'netlas', \r\n 'chaos'\r\n ],\r\n 'enable_http_crawl': true,\r\n 'threads': + 30,\r\n 'timeout': 5,\r\n}\r\nhttp_crawl: {}" default_engine: true - model: scanEngine.enginetype pk: 3 @@ -60,11 +61,11 @@ pk: 4 fields: engine_name: Vulnerability Scan - yaml_configuration: "subdomain_discovery: {\r\n 'uses_tools': ['subfinder', 'ctfr', - 'sublist3r', 'tlsx', 'oneforall', 'netlas'],\r\n 'enable_http_crawl': true,\r\n - \ 'threads': 30,\r\n 'timeout': 5,\r\n}\r\nhttp_crawl: {}\r\nosint: {\r\n 'discover': - [\r\n 'emails',\r\n 'metainfo',\r\n 'employees'\r\n ],\r\n - \ 'dorks': [\r\n 'login_pages',\r\n 'admin_panels',\r\n 'dashboard_pages',\r\n + yaml_configuration: "subdomain_discovery: {\r\n 'uses_tools': ['subfinder', 'chaos', + 'ctfr', 'sublist3r', 'tlsx', 'oneforall', 'netlas'],\r\n 'enable_http_crawl': + true,\r\n 'threads': 30,\r\n 'timeout': 5,\r\n}\r\nhttp_crawl: {}\r\nosint: + {\r\n 'discover': [\r\n 'emails',\r\n 'metainfo',\r\n 'employees'\r\n + \ ],\r\n 'dorks': [\r\n 'login_pages',\r\n 'admin_panels',\r\n 'dashboard_pages',\r\n \ 'stackoverflow',\r\n 'social_media',\r\n 'project_management',\r\n \ 'code_sharing',\r\n 'config_files',\r\n 'jenkins',\r\n 'wordpress_files',\r\n \ 'php_error',\r\n 'exposed_documents',\r\n 'db_files',\r\n 'git_exposed'\r\n @@ -72,8 +73,8 @@ {\r\n 'run_nuclei': true,\r\n 'run_dalfox': true,\r\n 'run_crlfuzz': true,\r\n \ 'enable_http_crawl': true,\r\n 'concurrency': 50,\r\n 'intensity': 'normal',\r\n \ 'rate_limit': 150,\r\n 'retries': 1,\r\n 'timeout': 5,\r\n 'fetch_gpt_report': - true,\r\n 'nuclei': {\r\n 'use_nuclei_config': false,\r\n 'severities': ['unknown', - 'info', 'low', 'medium', 'high', 'critical']\r\n }\r\n}" + true,\r\n 'nuclei': {\r\n 'use_nuclei_config': false,\r\n 'severities': + ['unknown', 'info', 'low', 'medium', 'high', 'critical']\r\n }\r\n}" default_engine: true - model: scanEngine.enginetype pk: 5 @@ -90,15 +91,16 @@ pk: 6 fields: engine_name: reNgine Recommended - yaml_configuration: "subdomain_discovery: {\r\n 'uses_tools': ['subfinder', 'ctfr', - 'sublist3r', 'tlsx', 'oneforall', 'netlas'],\r\n 'enable_http_crawl': true,\r\n - \ 'threads': 30,\r\n 'timeout': 5,\r\n}\r\nhttp_crawl: {}\r\nosint: {\r\n 'discover': - [\r\n 'emails',\r\n 'metainfo'\r\n ],\r\n 'dorks': [\r\n 'login_pages',\r\n - \ 'admin_panels',\r\n 'dashboard_pages',\r\n 'config_files',\r\n 'exposed_documents',\r\n - \ ],\r\n 'intensity': 'normal',\r\n 'documents_limit': 50\r\n}\r\nvulnerability_scan: - {\r\n 'run_nuclei': true,\r\n 'run_dalfox': true,\r\n 'run_crlfuzz': true,\r\n - \ 'enable_http_crawl': false,\r\n 'concurrency': 50,\r\n 'intensity': 'normal',\r\n - \ 'rate_limit': 150,\r\n 'retries': 1,\r\n 'timeout': 5,\r\n 'fetch_gpt_report': - true,\r\n 'nuclei': {\r\n 'use_nuclei_config': false,\r\n 'severities': ['low', - 'medium', 'high', 'critical']\r\n }\r\n}" + yaml_configuration: "subdomain_discovery: {\r\n 'uses_tools': ['subfinder', 'chaos', + 'ctfr', 'sublist3r', 'tlsx', 'oneforall', 'netlas'],\r\n 'enable_http_crawl': + true,\r\n 'threads': 30,\r\n 'timeout': 5,\r\n}\r\nhttp_crawl: {}\r\nosint: + {\r\n 'discover': [\r\n 'emails',\r\n 'metainfo'\r\n ],\r\n 'dorks': + [\r\n 'login_pages',\r\n 'admin_panels',\r\n 'dashboard_pages',\r\n + \ 'config_files',\r\n 'exposed_documents',\r\n ],\r\n 'intensity': 'normal',\r\n + \ 'documents_limit': 50\r\n}\r\nvulnerability_scan: {\r\n 'run_nuclei': true,\r\n + \ 'run_dalfox': true,\r\n 'run_crlfuzz': true,\r\n 'enable_http_crawl': false,\r\n + \ 'concurrency': 50,\r\n 'intensity': 'normal',\r\n 'rate_limit': 150,\r\n + \ 'retries': 1,\r\n 'timeout': 5,\r\n 'fetch_gpt_report': true,\r\n 'nuclei': + {\r\n 'use_nuclei_config': false,\r\n 'severities': ['low', 'medium', + 'high', 'critical']\r\n }\r\n}" default_engine: true From 5b15a30e9522691c581d92bd7c9b46cf399ab76e Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sat, 31 Aug 2024 19:57:40 +0530 Subject: [PATCH 139/211] disable fetching gpt report by default --- web/fixtures/default_scan_engines.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/fixtures/default_scan_engines.yaml b/web/fixtures/default_scan_engines.yaml index 6d481f64d..825bcd6a3 100644 --- a/web/fixtures/default_scan_engines.yaml +++ b/web/fixtures/default_scan_engines.yaml @@ -30,7 +30,7 @@ {\r\n 'run_nuclei': true,\r\n 'run_dalfox': true,\r\n 'run_crlfuzz': true,\r\n \ 'enable_http_crawl': true,\r\n 'concurrency': 50,\r\n 'intensity': 'normal',\r\n \ 'rate_limit': 150,\r\n 'retries': 1,\r\n 'timeout': 5,\r\n 'fetch_gpt_report': - true,\r\n 'nuclei': {\r\n 'use_nuclei_config': false,\r\n 'severities': + false,\r\n 'nuclei': {\r\n 'use_nuclei_config': false,\r\n 'severities': ['unknown', 'info', 'low', 'medium', 'high', 'critical']\r\n }\r\n}\r\nwaf_detection: {\r\n\r\n}\r\nscreenshot: {\r\n 'enable_http_crawl': true,\r\n 'intensity': 'normal',\r\n 'timeout': 10,\r\n 'threads': 40\r\n}\r\n\r\n# custom_headers: @@ -73,7 +73,7 @@ {\r\n 'run_nuclei': true,\r\n 'run_dalfox': true,\r\n 'run_crlfuzz': true,\r\n \ 'enable_http_crawl': true,\r\n 'concurrency': 50,\r\n 'intensity': 'normal',\r\n \ 'rate_limit': 150,\r\n 'retries': 1,\r\n 'timeout': 5,\r\n 'fetch_gpt_report': - true,\r\n 'nuclei': {\r\n 'use_nuclei_config': false,\r\n 'severities': + false,\r\n 'nuclei': {\r\n 'use_nuclei_config': false,\r\n 'severities': ['unknown', 'info', 'low', 'medium', 'high', 'critical']\r\n }\r\n}" default_engine: true - model: scanEngine.enginetype @@ -100,7 +100,7 @@ \ 'documents_limit': 50\r\n}\r\nvulnerability_scan: {\r\n 'run_nuclei': true,\r\n \ 'run_dalfox': true,\r\n 'run_crlfuzz': true,\r\n 'enable_http_crawl': false,\r\n \ 'concurrency': 50,\r\n 'intensity': 'normal',\r\n 'rate_limit': 150,\r\n - \ 'retries': 1,\r\n 'timeout': 5,\r\n 'fetch_gpt_report': true,\r\n 'nuclei': + \ 'retries': 1,\r\n 'timeout': 5,\r\n 'fetch_gpt_report': false,\r\n 'nuclei': {\r\n 'use_nuclei_config': false,\r\n 'severities': ['low', 'medium', 'high', 'critical']\r\n }\r\n}" default_engine: true From 5685a135d924502e9b8b7e28a176691fcdabeb90 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sat, 31 Aug 2024 20:35:21 +0530 Subject: [PATCH 140/211] update chaos during dockerbuild process --- web/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/Dockerfile b/web/Dockerfile index 621357706..66d709e75 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -104,6 +104,9 @@ RUN printf "\ # Update Nuclei and Nuclei-Templates RUN nuclei -update-templates +# update chaos +RUN chaos -update + # Copy requirements COPY ./requirements.txt /tmp/requirements.txt RUN pip3 install --upgrade setuptools==72.1.0 From ba162e5bea7a997ec6efae29707909563639ca47 Mon Sep 17 00:00:00 2001 From: Sean Keane Date: Thu, 4 Jul 2024 07:02:29 -0500 Subject: [PATCH 141/211] adding logic for a new feature to sync hackerone bookmarks --- docker-compose.yml | 4 +- web/celery-entrypoint.sh | 2 + web/reNgine/tasks.py | 153 ++++++++++++++++++ .../scanEngine/settings/hackerone.html | 28 ++-- .../templates/organization/list.html | 1 + web/targetApp/urls.py | 4 + web/targetApp/views.py | 11 +- 7 files changed, 190 insertions(+), 13 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2daccd382..1f580d997 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: - tool_config:/root/.config - static_volume:/usr/src/app/staticfiles/ environment: - - DEBUG=0 + - DEBUG=1 - CELERY_BROKER=redis://redis:6379/0 - CELERY_BACKEND=redis://redis:6379/0 - DOMAIN_NAME=${DOMAIN_NAME} @@ -86,7 +86,7 @@ services: restart: always image: docker.pkg.github.com/yogeshojha/rengine/rengine:latest environment: - - DEBUG=0 + - DEBUG=1 - CELERY_BROKER=redis://redis:6379/0 - CELERY_BACKEND=redis://redis:6379/0 - DOMAIN_NAME=${DOMAIN_NAME} diff --git a/web/celery-entrypoint.sh b/web/celery-entrypoint.sh index 6d7968fff..5f5dfe14d 100755 --- a/web/celery-entrypoint.sh +++ b/web/celery-entrypoint.sh @@ -187,6 +187,8 @@ watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/re watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q query_reverse_whois_queue -n query_reverse_whois_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q query_ip_history_queue -n query_ip_history_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=$loglevel -Q llm_queue -n llm_worker & +watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=$loglevel -Q gpt_queue -n gpt_worker & +watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q h1_sync_queue -n h1_sync_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q dorking_queue -n dorking_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q osint_discovery_queue -n osint_discovery_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q h8mail_queue -n h8mail_worker & diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 66a45498d..d377201c5 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -20,6 +20,7 @@ from django.db.models import Count from dotted_dict import DottedDict from django.utils import timezone +from django.shortcuts import get_object_or_404 from pycvesearch import CVESearch from metafinder.extractor import extract_metadata_from_google_search @@ -4622,3 +4623,155 @@ def llm_vulnerability_description(vulnerability_id): vuln.save() return response + +def fetch_h1_bookmarked(): + """ + Fetches bookmarked programs from HackerOne API using pagination. + + Returns: + list: A list of bookmarked program objects. + """ + ALLOWED_SCOPE_TYPES = ["WILDCARD", "DOMAIN", "IP_ADDRESS", "CIDR", "URL"] + + bookmarked_programs = [] + next_link = True + programs_url = "https://api.hackerone.com/v1/hackers/programs?size=100" + + hackerone_query = Hackerone.objects.all() + + if hackerone_query.exists(): + hackerone = Hackerone.objects.first() + + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + + while next_link: + try: + response = requests.get(programs_url, auth=(hackerone.username, hackerone.api_key), headers=headers) + response.raise_for_status() # Raise an exception for non-200 status codes + + response_json = response.json() + for program in response_json["data"]: + if program["attributes"]["bookmarked"]: + program_url = f"https://api.hackerone.com/v1/hackers/programs/{program['attributes']['handle']}" + program_response = requests.get(program_url, auth=(hackerone.username, hackerone.api_key), headers=headers) + program_response.raise_for_status() + + program_json = program_response.json() + program_scopes = [] + for scope in program_json["relationships"]["structured_scopes"]["data"]: + if ( + scope["attributes"]["asset_type"] in ALLOWED_SCOPE_TYPES + and scope["attributes"]["eligible_for_submission"] is True + ): + program_scopes.append(scope["attributes"]) + + program["scopes"] = program_scopes + bookmarked_programs.append(program) + + if "links" in response_json and "next" in response_json["links"]: + programs_url = response_json["links"]["next"] + else: + next_link = False + + except requests.exceptions.RequestException as e: + logger.error(f"Error fetching HackerOne programs: {e}") + break + + return bookmarked_programs + +@app.task(name='sync_h1_bookmarked', bind=False, queue='h1_sync_queue') +def sync_h1_bookmarked(): + """ + Sync HackerOne bookmarked programs to organizations + """ + try: + logger.info('Starting HackerOne Bookmark Sync') + + # Fetch bookmarked programs and project details + bookmarked_programs = fetch_h1_bookmarked() + project = Project.objects.get(slug="default") + bookmarked_handles = {program['attributes']['handle'] + for program in bookmarked_programs} + + # Get current organizations and their handles + current_organizations = Organization.objects.filter(project=project) + current_handles = {org.name for org in current_organizations} + + # Delete organizations not in the bookmarked programs + handles_to_delete = current_handles - bookmarked_handles + + # Delete organizations + for handle in handles_to_delete: + try: + org_to_delete = get_object_or_404(Organization, name=handle, project=project) + # Check if organization was added via H1 Bookmark Sync + if(org_to_delete.description == 'Added via H1 Bookmark Sync'): + for domain in org_to_delete.get_domains(): + domain.delete() + org_to_delete.delete() + logger.info(f'Deleted organization: {handle}') + except Exception as e: + logger.error(f"Error deleting organization {handle}: {e}") + + # Process bookmarked programs and create domains + for program in bookmarked_programs: + + if program['attributes']['handle'] not in current_handles: + domains = [] + + for scope in program["scopes"]: + domain_name = None + description = '' + ip_address_cidr = None + + if scope["asset_type"] == "WILDCARD": + domain_name = scope["asset_identifier"].replace('*.', '') + elif scope["asset_type"] == "DOMAIN": + domain_name = scope["asset_identifier"] + elif scope["asset_type"] in ["IP_ADDRESS", "CIDR"]: + domain_name = scope["asset_identifier"] + ip_address_cidr = scope["asset_identifier"] + elif scope["asset_type"] == "URL": + parsed_url = urlparse(scope["asset_identifier"]) + domain_name = parsed_url.netloc + + if domain_name: + try: + domain, created = Domain.objects.get_or_create( + name=domain_name, + description=description, + h1_team_handle=program['attributes']['handle'], + project=project, + ip_address_cidr=ip_address_cidr + ) + domain.insert_date = timezone.now() + domain.save() + + domains.append(domain) + except Exception as e: + logger.error(f"Error creating/updating domain {domain_name}: {e}") + + try: + organization, created = Organization.objects.get_or_create( + name=program['attributes']['handle'], + project=project, + defaults={'description': 'Added via H1 Bookmark Sync', 'insert_date': timezone.now()} + ) + + if not created: + organization.insert_date = timezone.now() + + for domain in domains: + organization.domains.add(domain) + + organization.save() + logger.info(f"Added organization: program['attributes']['handle']") + except Exception as e: + logger.error(f"Error creating/updating organization {program['attributes']['handle']}: {e}") + + logger.info('Completed HackerOne Bookmark Sync') + except Exception as e: + logger.error(f"Error in sync_h1_bookmarked task: {e}") \ No newline at end of file diff --git a/web/scanEngine/templates/scanEngine/settings/hackerone.html b/web/scanEngine/templates/scanEngine/settings/hackerone.html index 1cbd18142..a5f4f1b21 100644 --- a/web/scanEngine/templates/scanEngine/settings/hackerone.html +++ b/web/scanEngine/templates/scanEngine/settings/hackerone.html @@ -24,16 +24,16 @@
    -

    Hackerone Automatic Vulnerability Report Settings

    - - + +

    + HackerOne API Setting can be used for the following: +

      +
    • Synchroinze HackerOne bookmarks with reEngine organizations
      • +
    • Automatically Reports vulnerabilities
      • +
    +

    +

    Hackerone API Settings

    - reNgine Automatically Reports vulnerabilities to your bug bounty programs on Hackerone, if any vulnerabilities are identified. -
    A valid Hackerone API token and username is required.
    More details on how to generate your hackerone api token is provided by Hackerone Documentation @@ -53,6 +53,15 @@

    Hackerone Automatic Vulnerability Report Settings

      Test my hackerone api key +

    Hackerone Automatic Vulnerability Report Settings

    +

    + reNgine Automatically Reports vulnerabilities to your bug bounty programs on Hackerone, if any vulnerabilities are identified. +

    +

    Report Vulnerability to hackerone when

    @@ -90,7 +99,6 @@

    Vulnerability Report Template

    {% endblock main_content %} - {% block page_level_script %} diff --git a/web/targetApp/templates/organization/list.html b/web/targetApp/templates/organization/list.html index e204337b1..a7edd761a 100644 --- a/web/targetApp/templates/organization/list.html +++ b/web/targetApp/templates/organization/list.html @@ -28,6 +28,7 @@
    {% if user|can:'modify_targets' %} + Sync Organizations Add New Organization {% endif %}
    diff --git a/web/targetApp/urls.py b/web/targetApp/urls.py index 5d7f64142..d0e2d8a91 100644 --- a/web/targetApp/urls.py +++ b/web/targetApp/urls.py @@ -31,6 +31,10 @@ '/list/organization', views.list_organization, name='list_organization'), + path( + '/sync/organization', + views.sync_organization, + name='sync_organization'), path( 'delete/target/', views.delete_target, diff --git a/web/targetApp/views.py b/web/targetApp/views.py index e5c6a16fc..e9dd175a1 100644 --- a/web/targetApp/views.py +++ b/web/targetApp/views.py @@ -3,8 +3,10 @@ import ipaddress import logging import validators +import pytz from datetime import timedelta +from django.utils import timezone from urllib.parse import urlparse from django import http from django.conf import settings @@ -15,9 +17,10 @@ from django.utils import timezone from django.utils.safestring import mark_safe from rolepermissions.decorators import has_permission_decorator +from django_celery_beat.models import (PeriodicTask, ClockedSchedule) from reNgine.common_func import * -from reNgine.tasks import run_command, sanitize_url +from reNgine.tasks import run_command, sanitize_url, sync_h1_bookmarked from scanEngine.models import * from startScan.models import * from targetApp.forms import * @@ -542,6 +545,12 @@ def list_organization(request, slug): } return render(request, 'organization/list.html', context) +def sync_organization(request, slug): + # Start the celery task + sync_h1_bookmarked.apply_async() + + return http.HttpResponseRedirect(reverse('list_organization', kwargs={'slug': slug})) + @has_permission_decorator(PERM_MODIFY_TARGETS, redirect_url=FOUR_OH_FOUR_URL) def delete_organization(request, id): From b16c112e4853b717655aa1e00a95a2a17e840b06 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 1 Sep 2024 17:47:51 +0530 Subject: [PATCH 142/211] revert back unnecessary changes --- docker-compose.yml | 4 +-- web/celery-entrypoint.sh | 1 - .../scanEngine/settings/hackerone.html | 28 +++++++------------ web/targetApp/views.py | 3 -- 4 files changed, 12 insertions(+), 24 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6eb4910e9..6f40c5bd6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: - tool_config:/root/.config - static_volume:/usr/src/app/staticfiles/ environment: - - DEBUG=1 + - DEBUG=0 - CELERY_BROKER=redis://redis:6379/0 - CELERY_BACKEND=redis://redis:6379/0 - DOMAIN_NAME=${DOMAIN_NAME} @@ -86,7 +86,7 @@ services: restart: always image: docker.pkg.github.com/yogeshojha/rengine/rengine:latest environment: - - DEBUG=1 + - DEBUG=0 - CELERY_BROKER=redis://redis:6379/0 - CELERY_BACKEND=redis://redis:6379/0 - DOMAIN_NAME=${DOMAIN_NAME} diff --git a/web/celery-entrypoint.sh b/web/celery-entrypoint.sh index 5f5dfe14d..ae964a6f0 100755 --- a/web/celery-entrypoint.sh +++ b/web/celery-entrypoint.sh @@ -188,7 +188,6 @@ watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/re watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q query_ip_history_queue -n query_ip_history_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=$loglevel -Q llm_queue -n llm_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=$loglevel -Q gpt_queue -n gpt_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q h1_sync_queue -n h1_sync_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q dorking_queue -n dorking_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q osint_discovery_queue -n osint_discovery_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q h8mail_queue -n h8mail_worker & diff --git a/web/scanEngine/templates/scanEngine/settings/hackerone.html b/web/scanEngine/templates/scanEngine/settings/hackerone.html index 95e83b223..eec049813 100644 --- a/web/scanEngine/templates/scanEngine/settings/hackerone.html +++ b/web/scanEngine/templates/scanEngine/settings/hackerone.html @@ -24,16 +24,16 @@
    - -

    - HackerOne API Setting can be used for the following: -

      -
    • Synchroinze HackerOne bookmarks with reEngine organizations
      • -
    • Automatically Reports vulnerabilities
      • -
    -

    -

    Hackerone API Settings

    +

    Hackerone Automatic Vulnerability Report Settings

    + +

    + reNgine Automatically Reports vulnerabilities to your bug bounty programs on Hackerone, if any vulnerabilities are identified. +
    A valid Hackerone API token and username is required.
    More details on how to generate your hackerone api token is provided by Hackerone Documentation @@ -58,15 +58,6 @@

    Hackerone API Settings

      Test my hackerone api key -

    Hackerone Automatic Vulnerability Report Settings

    -

    - reNgine Automatically Reports vulnerabilities to your bug bounty programs on Hackerone, if any vulnerabilities are identified. -

    -

    Report Vulnerability to hackerone when

    @@ -104,6 +95,7 @@

    Vulnerability Report Template

    {% endblock main_content %} + {% block page_level_script %} diff --git a/web/targetApp/views.py b/web/targetApp/views.py index e9dd175a1..df05e6e2e 100644 --- a/web/targetApp/views.py +++ b/web/targetApp/views.py @@ -3,10 +3,8 @@ import ipaddress import logging import validators -import pytz from datetime import timedelta -from django.utils import timezone from urllib.parse import urlparse from django import http from django.conf import settings @@ -17,7 +15,6 @@ from django.utils import timezone from django.utils.safestring import mark_safe from rolepermissions.decorators import has_permission_decorator -from django_celery_beat.models import (PeriodicTask, ClockedSchedule) from reNgine.common_func import * from reNgine.tasks import run_command, sanitize_url, sync_h1_bookmarked From 6367900a1adba02b1c7a33cae00ebd532002bc4a Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Sun, 1 Sep 2024 17:48:56 +0530 Subject: [PATCH 143/211] remove unused worker --- web/celery-entrypoint.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/web/celery-entrypoint.sh b/web/celery-entrypoint.sh index ae964a6f0..6d7968fff 100755 --- a/web/celery-entrypoint.sh +++ b/web/celery-entrypoint.sh @@ -187,7 +187,6 @@ watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/re watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q query_reverse_whois_queue -n query_reverse_whois_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q query_ip_history_queue -n query_ip_history_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=$loglevel -Q llm_queue -n llm_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=$loglevel -Q gpt_queue -n gpt_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q dorking_queue -n dorking_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q osint_discovery_queue -n osint_discovery_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q h8mail_queue -n h8mail_worker & From 381b6eab2c319388ff005b90e98eb375edd91ea3 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 2 Sep 2024 06:51:19 +0530 Subject: [PATCH 144/211] update onboarding screen --- .../templates/dashboard/onboarding.html | 126 ++++++++++-------- 1 file changed, 72 insertions(+), 54 deletions(-) diff --git a/web/dashboard/templates/dashboard/onboarding.html b/web/dashboard/templates/dashboard/onboarding.html index 742a679ac..0456d7435 100644 --- a/web/dashboard/templates/dashboard/onboarding.html +++ b/web/dashboard/templates/dashboard/onboarding.html @@ -7,87 +7,104 @@ +
    @@ -111,62 +91,40 @@

    Vulnerability Report Template

    return DOMPurify.sanitize(simplemde.markdown(plainText)); }; -function test_hackerone() { - if ($("#username").val().length == 0 || $("#api_key").val().length == 0) { - if ($("#username").val().length == 0) { - $("#username").addClass("is-invalid"); - } - if ($("#api_key").val().length == 0) { - $("#api_key").addClass("is-invalid"); +function handleFormInputs() { + const form = document.getElementById('hackerone_form'); + const checkbox = document.getElementById('send_report'); + const inputs = form.querySelectorAll('input:not(#send_report):not([type="submit"]):not([name="csrf_token"])'); + + inputs.forEach(input => { + if (input.type === 'text' || input.type === 'textarea') { + input.readOnly = !checkbox.checked; + } else if (input.type !== 'hidden') { + input.disabled = !checkbox.checked; } - } - else{ - const hackerone_api = 'testHackerone/'; - var username = $("#username").val(); - var api_key = $("#api_key").val(); - swal.queue([{ - title: 'Hackerone Configuration', - confirmButtonText: 'Test my hackerone API Key', - text: - 'This will test if your hackerone API keys are working.', - showLoaderOnConfirm: true, - preConfirm: function() { - return fetch(hackerone_api, { - method: 'POST', - headers: { - "X-CSRFToken": getCookie("csrftoken"), - "Content-Type": "application/json" - }, - body: JSON.stringify({'username': username, 'api_key': api_key}), - }, - ).then(function (response) { - return response.json(); - }) - .then(function(data) { - if (data.status == 200) { - $("#username").addClass("is-valid"); - $("#api_key").addClass("is-valid"); - $("#username").removeClass("is-invalid"); - $("#api_key").removeClass("is-invalid"); - return swal.insertQueueStep("Your hackerone Credentials are working.") - } - else{ - $("#username").addClass("is-invalid"); - $("#api_key").addClass("is-invalid"); - $("#username").removeClass("is-valid"); - $("#api_key").removeClass("is-valid"); - return swal.insertQueueStep("Oops! Your hackerone Credentials are not working, check your username and/or api_key.") - } - }) - .catch(function() { - swal.insertQueueStep({ - type: 'error', - title: 'Unable to get your public IP' - }) - }) - } - }]); -} + }); } + +document.addEventListener('DOMContentLoaded', () => { + const form = document.getElementById('hackerone_form'); + const checkbox = document.getElementById('send_report'); + + handleFormInputs(); // init state + checkbox.addEventListener('change', handleFormInputs); + + form.addEventListener('submit', (event) => { + if (!checkbox.checked) { + // If checkbox is unchecked, temporarily enable all inputs for submission + const inputs = form.querySelectorAll('input:not(#send_report):not([type="submit"]):not([name="csrf_token"])'); + inputs.forEach(input => { + input.disabled = false; + input.readOnly = false; + }); + + setTimeout(() => handleFormInputs(), 0); + } + }); +}); + {% endblock page_level_script %} diff --git a/web/static/custom/custom.js b/web/static/custom/custom.js index 617fb4adb..f04913b25 100644 --- a/web/static/custom/custom.js +++ b/web/static/custom/custom.js @@ -3402,3 +3402,60 @@ function show_scan_configuration(starting_path, out_of_scope_subdomains, exclude } +function test_hackerone() { + if ($("#username").val().length == 0 || $("#api_key").val().length == 0) { + if ($("#username").val().length == 0) { + $("#username").addClass("is-invalid"); + } + if ($("#api_key").val().length == 0) { + $("#api_key").addClass("is-invalid"); + } + } + else{ + const hackerone_api = 'testHackerone/'; + var username = $("#username").val(); + var api_key = $("#api_key").val(); + swal.queue([{ + title: 'Hackerone Configuration', + confirmButtonText: 'Test my hackerone API Key', + text: + 'This will test if your hackerone API keys are working.', + showLoaderOnConfirm: true, + preConfirm: function() { + return fetch(hackerone_api, { + method: 'POST', + headers: { + "X-CSRFToken": getCookie("csrftoken"), + "Content-Type": "application/json" + }, + body: JSON.stringify({'username': username, 'api_key': api_key}), + }, + ).then(function (response) { + return response.json(); + }) + .then(function(data) { + if (data.status == 200) { + $("#username").addClass("is-valid"); + $("#api_key").addClass("is-valid"); + $("#username").removeClass("is-invalid"); + $("#api_key").removeClass("is-invalid"); + return swal.insertQueueStep("Your hackerone Credentials are working.") + } + else{ + $("#username").addClass("is-invalid"); + $("#api_key").addClass("is-invalid"); + $("#username").removeClass("is-valid"); + $("#api_key").removeClass("is-valid"); + return swal.insertQueueStep("Oops! Your hackerone Credentials are not working, check your username and/or api_key.") + } + }) + .catch(function() { + swal.insertQueueStep({ + type: 'error', + title: 'Test Hackerone API Key', + }) + }) + } + }]); + } +} \ No newline at end of file From bc68bf14d32bc90999d6a57677737dc93dcf5e24 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 2 Sep 2024 09:12:48 +0530 Subject: [PATCH 150/211] fix condition to send report --- web/reNgine/tasks.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index bbb62342c..0d31fb5b3 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -2227,13 +2227,24 @@ def nuclei_individual_severity_module(self, cmd, severity, enable_http_crawl, sh fields, add_meta_info=False) - # Send report to hackerone - hackerone_query = Hackerone.objects.all() + """ + Send report to hackerone when + 1. send_report is True from Hackerone model in ScanEngine + 2. username and key is set in HackerOneAPIKey in Dashboard + 3. severity is not info or low + """ + hackerone_query = Hackerone.objects.filter(send_report=True) + api_key_check_query = HackerOneAPIKey.objects.filter( + Q(username__isnull=False) & Q(key__isnull=False) + ) + send_report = ( hackerone_query.exists() and + api_key_check_query.exists() and severity not in ('info', 'low') and vuln.target_domain.h1_team_handle ) + if send_report: hackerone = hackerone_query.first() if hackerone.send_critical and severity == 'critical': @@ -3315,23 +3326,23 @@ def send_hackerone_report(vulnerability_id): tpl = tpl.replace('{vulnerability_reference}', vulnerability.reference if vulnerability.reference else '') data = { - "data": { - "type": "report", - "attributes": { - "team_handle": vulnerability.target_domain.h1_team_handle, - "title": f'{vulnerability.name} found in {vulnerability.http_url}', - "vulnerability_information": tpl, - "severity_rating": severity_value, - "impact": "More information about the impact and vulnerability can be found here: \n" + vulnerability.reference if vulnerability.reference else "NA", - } + "data": { + "type": "report", + "attributes": { + "team_handle": vulnerability.target_domain.h1_team_handle, + "title": f'{vulnerability.name} found in {vulnerability.http_url}', + "vulnerability_information": tpl, + "severity_rating": severity_value, + "impact": "More information about the impact and vulnerability can be found here: \n" + vulnerability.reference if vulnerability.reference else "NA", + } } } r = requests.post( - 'https://api.hackerone.com/v1/hackers/reports', - auth=(hackerone.username, hackerone.api_key), - json=data, - headers=headers + 'https://api.hackerone.com/v1/hackers/reports', + auth=(hackerone.username, hackerone.api_key), + json=data, + headers=headers ) response = r.json() status_code = r.status_code From 7970bbee08d6c94a5e1a806f8db954a3c6341ccb Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 2 Sep 2024 09:23:58 +0530 Subject: [PATCH 151/211] fix hackerone send report task to use new api key model --- web/reNgine/tasks.py | 102 +++++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 48 deletions(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 0d31fb5b3..c7eea5119 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -2244,7 +2244,7 @@ def nuclei_individual_severity_module(self, cmd, severity, enable_http_crawl, sh severity not in ('info', 'low') and vuln.target_domain.h1_team_handle ) - + if send_report: hackerone = hackerone_query.first() if hackerone.send_critical and severity == 'critical': @@ -3304,58 +3304,64 @@ def send_hackerone_report(vulnerability_id): """ vulnerability = Vulnerability.objects.get(id=vulnerability_id) severities = {v: k for k,v in NUCLEI_SEVERITY_MAP.items()} - headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/json' + + # can only send vulnerability report if team_handle exists and send_report is True and api_key exists + hackerone = Hackerone.objects.filter(send_report=True).first() + api_key = HackerOneAPIKey.objects.filter(username__isnull=False, key__isnull=False).first() + + if not (vulnerability.target_domain.h1_team_handle and hackerone and api_key): + logger.error('Missing required data: team handle, Hackerone config, or API key.') + return {"status_code": 400, "message": "Missing required data"} + + severity_value = severities[vulnerability.severity] + tpl = hackerone.report_template or "" + + tpl_vars = { + '{vulnerability_name}': vulnerability.name, + '{vulnerable_url}': vulnerability.http_url, + '{vulnerability_severity}': severity_value, + '{vulnerability_description}': vulnerability.description or '', + '{vulnerability_extracted_results}': vulnerability.extracted_results or '', + '{vulnerability_reference}': vulnerability.reference or '', } - # can only send vulnerability report if team_handle exists - if len(vulnerability.target_domain.h1_team_handle) !=0: - hackerone_query = Hackerone.objects.all() - if hackerone_query.exists(): - hackerone = Hackerone.objects.first() - severity_value = severities[vulnerability.severity] - tpl = hackerone.report_template - - # Replace syntax of report template with actual content - tpl = tpl.replace('{vulnerability_name}', vulnerability.name) - tpl = tpl.replace('{vulnerable_url}', vulnerability.http_url) - tpl = tpl.replace('{vulnerability_severity}', severity_value) - tpl = tpl.replace('{vulnerability_description}', vulnerability.description if vulnerability.description else '') - tpl = tpl.replace('{vulnerability_extracted_results}', vulnerability.extracted_results if vulnerability.extracted_results else '') - tpl = tpl.replace('{vulnerability_reference}', vulnerability.reference if vulnerability.reference else '') - - data = { - "data": { - "type": "report", - "attributes": { - "team_handle": vulnerability.target_domain.h1_team_handle, - "title": f'{vulnerability.name} found in {vulnerability.http_url}', - "vulnerability_information": tpl, - "severity_rating": severity_value, - "impact": "More information about the impact and vulnerability can be found here: \n" + vulnerability.reference if vulnerability.reference else "NA", - } - } + # Replace syntax of report template with actual content + for key, value in tpl_vars.items(): + tpl = tpl.replace(key, value) + + data = { + "data": { + "type": "report", + "attributes": { + "team_handle": vulnerability.target_domain.h1_team_handle, + "title": f'{vulnerability.name} found in {vulnerability.http_url}', + "vulnerability_information": tpl, + "severity_rating": severity_value, + "impact": "More information about the impact and vulnerability can be found here: \n" + vulnerability.reference if vulnerability.reference else "NA", } + } + } - r = requests.post( - 'https://api.hackerone.com/v1/hackers/reports', - auth=(hackerone.username, hackerone.api_key), - json=data, - headers=headers - ) - response = r.json() - status_code = r.status_code - if status_code == 201: - vulnerability.hackerone_report_id = response['data']["id"] - vulnerability.open_status = False - vulnerability.save() - return status_code + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } - else: - logger.error('No team handle found.') - status_code = 111 - return status_code + r = requests.post( + 'https://api.hackerone.com/v1/hackers/reports', + auth=(api_key.username, api_key.key), + json=data, + headers=headers + ) + response = r.json() + status_code = r.status_code + if status_code == 201: + vulnerability.hackerone_report_id = response['data']["id"] + vulnerability.open_status = False + vulnerability.save() + return {"status_code": r.status_code, "message": "Report sent successfully"} + logger.error(f"Error sending report to HackerOne") + return {"status_code": r.status_code, "message": response} #-------------# From d2f3bd158404ff231615effb8fec9c9a85e55af1 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 2 Sep 2024 09:39:59 +0530 Subject: [PATCH 152/211] add test hackerone keys in api vault --- .../templates/scanEngine/settings/api.html | 32 ++--- web/static/custom/custom.js | 126 ++++++++++-------- 2 files changed, 88 insertions(+), 70 deletions(-) diff --git a/web/scanEngine/templates/scanEngine/settings/api.html b/web/scanEngine/templates/scanEngine/settings/api.html index 1ccdd792b..64da0ea21 100644 --- a/web/scanEngine/templates/scanEngine/settings/api.html +++ b/web/scanEngine/templates/scanEngine/settings/api.html @@ -75,27 +75,29 @@

    Hackerone Keys will be used to import targets, bookmarked programs, and submit automated vulnerability report to Hackerone. This is a bug bounty specific feature.

    -
    - - {% if hackerone_username %} {% else %} {% endif %} +
    + + {% if hackerone_username %} {% else %} {% endif %} +
    +
    + +
    + {% if hackerone_key %} {% else %} {% endif %} +
    +
    -
    - -
    - {% if hackerone_key %} {% else %} {% endif %} -
    - -
    -
    -
    -

    This is optional but recommended for bug hunters. Get your API key from Hackerone Documentation -

    +
    +

    This is optional but recommended for bug hunters. Get your API key from Hackerone Documentation

    +
    +
    + Test my hackerone keys
    - +
    diff --git a/web/static/custom/custom.js b/web/static/custom/custom.js index f04913b25..795deb82c 100644 --- a/web/static/custom/custom.js +++ b/web/static/custom/custom.js @@ -3402,60 +3402,76 @@ function show_scan_configuration(starting_path, out_of_scope_subdomains, exclude } -function test_hackerone() { - if ($("#username").val().length == 0 || $("#api_key").val().length == 0) { - if ($("#username").val().length == 0) { - $("#username").addClass("is-invalid"); - } - if ($("#api_key").val().length == 0) { - $("#api_key").addClass("is-invalid"); - } - } - else{ - const hackerone_api = 'testHackerone/'; - var username = $("#username").val(); - var api_key = $("#api_key").val(); - swal.queue([{ - title: 'Hackerone Configuration', - confirmButtonText: 'Test my hackerone API Key', - text: - 'This will test if your hackerone API keys are working.', - showLoaderOnConfirm: true, - preConfirm: function() { - return fetch(hackerone_api, { - method: 'POST', - headers: { - "X-CSRFToken": getCookie("csrftoken"), - "Content-Type": "application/json" +async function test_hackerone() { + const username = $("#username_hackerone"); + const apiKey = $("#key_hackerone"); + const fields = [username, apiKey]; + + const isValid = fields.every(field => field.val().trim().length > 0); + fields.forEach(field => { + field.toggleClass("is-invalid", field.val().trim().length === 0); + }); + + if (!isValid) return; + + try { + const result = await Swal.fire({ + title: 'HackerOne Configuration', + text: 'This will test if your HackerOne API keys are working.', + icon: 'info', + showCancelButton: true, + confirmButtonText: 'Test my HackerOne API Key', + showLoaderOnConfirm: true, + preConfirm: async () => { + try { + const response = await fetch('testHackerone/', { + method: 'POST', + headers: { + "X-CSRFToken": getCookie("csrftoken"), + "Content-Type": "application/json" + }, + body: JSON.stringify({ + username: username.val().trim(), + api_key: apiKey.val().trim() + }), + }); + + if (!response.ok) { + throw new Error('Network response was not ok'); + } + + const data = await response.json(); + return data; + } catch (error) { + Swal.showValidationMessage(`Request failed: ${error}`); + } }, - body: JSON.stringify({'username': username, 'api_key': api_key}), - }, - ).then(function (response) { - return response.json(); - }) - .then(function(data) { - if (data.status == 200) { - $("#username").addClass("is-valid"); - $("#api_key").addClass("is-valid"); - $("#username").removeClass("is-invalid"); - $("#api_key").removeClass("is-invalid"); - return swal.insertQueueStep("Your hackerone Credentials are working.") - } - else{ - $("#username").addClass("is-invalid"); - $("#api_key").addClass("is-invalid"); - $("#username").removeClass("is-valid"); - $("#api_key").removeClass("is-valid"); - return swal.insertQueueStep("Oops! Your hackerone Credentials are not working, check your username and/or api_key.") - } - }) - .catch(function() { - swal.insertQueueStep({ - type: 'error', - title: 'Test Hackerone API Key', - }) - }) - } - }]); - } + allowOutsideClick: () => !Swal.isLoading() + }); + + if (result.isConfirmed) { + const data = result.value; + const isWorking = data.status === 200; + + fields.forEach(field => { + field.toggleClass("is-valid", isWorking); + field.toggleClass("is-invalid", !isWorking); + }); + + await Swal.fire({ + title: isWorking ? 'Success' : 'Error', + text: isWorking + ? 'Your HackerOne Credentials are working.' + : 'Your HackerOne Credentials are not working. Please check your username and/or API key.', + icon: isWorking ? 'success' : 'error' + }); + } + } catch (error) { + console.error('Error:', error); + await Swal.fire({ + title: 'Error', + text: 'An unexpected error occurred while testing the HackerOne API Key.', + icon: 'error' + }); + } } \ No newline at end of file From f9ecd61fe2563ce26b314f2b40ad6fb1db1916d8 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 2 Sep 2024 20:21:33 +0530 Subject: [PATCH 153/211] create serializers for hackerone program attributes --- web/api/serializers.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/web/api/serializers.py b/web/api/serializers.py index 1acbfacf6..a01c9b909 100644 --- a/web/api/serializers.py +++ b/web/api/serializers.py @@ -10,6 +10,37 @@ from dashboard.models import InAppNotification +class HackerOneProgramAttributesSerializer(serializers.Serializer): + """ + Serializer for HackerOne Program + IMP: THIS is not a model serializer, programs will not be stored in db + due to ever changing nature of programs, rather cache will be used on these serializers + """ + handle = serializers.CharField(required=False) + name = serializers.CharField(required=False) + currency = serializers.CharField(required=False) + submission_state = serializers.CharField(required=False) + triage_active = serializers.BooleanField(allow_null=True, required=False) + state = serializers.CharField(required=False) + started_accepting_at = serializers.DateTimeField(required=False) + bookmarked = serializers.BooleanField(required=False) + allows_bounty_splitting = serializers.BooleanField(required=False) + offers_bounties = serializers.BooleanField(required=False) + open_scope = serializers.BooleanField(allow_null=True, required=False) + fast_payments = serializers.BooleanField(allow_null=True, required=False) + gold_standard_safe_harbor = serializers.BooleanField(allow_null=True, required=False) + + def to_representation(self, instance): + return {key: value for key, value in instance.items()} + + +class HackerOneProgramSerializer(serializers.Serializer): + id = serializers.CharField() + type = serializers.CharField() + attributes = HackerOneProgramAttributesSerializer() + + + class InAppNotificationSerializer(serializers.ModelSerializer): class Meta: model = InAppNotification From 789efb53558e5492247b3b02009bc00096c9a238 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 2 Sep 2024 20:21:44 +0530 Subject: [PATCH 154/211] enable caching for 30 mins --- web/reNgine/settings.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/web/reNgine/settings.py b/web/reNgine/settings.py index 841c1172c..90a5d8172 100644 --- a/web/reNgine/settings.py +++ b/web/reNgine/settings.py @@ -330,4 +330,14 @@ ''' File upload settings ''' -DATA_UPLOAD_MAX_NUMBER_FIELDS = None \ No newline at end of file +DATA_UPLOAD_MAX_NUMBER_FIELDS = None + +''' + Caching Settings +''' +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + 'TIMEOUT': 60 * 30, # 30 minutes caching will be used + } +} \ No newline at end of file From 2ea5b7e44a5213d48b2a6b2d15c3e487c36693ac Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 2 Sep 2024 20:22:33 +0530 Subject: [PATCH 155/211] add api endpoints to fetch hackerone programs, this include mainly 3 endpoints for getting bookmarked programs, programs with bounty, and all programs --- web/api/urls.py | 1 + web/api/views.py | 86 +++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/web/api/urls.py b/web/api/urls.py index 39ceb84a7..7a353275a 100644 --- a/web/api/urls.py +++ b/web/api/urls.py @@ -20,6 +20,7 @@ router.register(r'listActivityLogs', ListActivityLogsViewSet) router.register(r'listScanLogs', ListScanLogsViewSet) router.register(r'notifications', InAppNotificationManagerViewSet, basename='notification') +router.register(r'hackerone-programs', HackerOneProgramViewSet, basename='hackerone_program') urlpatterns = [ url('^', include(router.urls)), diff --git a/web/api/views.py b/web/api/views.py index 1bcd909d2..b2fdcc4a8 100644 --- a/web/api/views.py +++ b/web/api/views.py @@ -1,13 +1,11 @@ -import logging import re import socket -from ipaddress import IPv4Network - +import logging import requests import validators -from dashboard.models import * + +from ipaddress import IPv4Network from django.db.models import CharField, Count, F, Q, Value -from django.shortcuts import get_object_or_404 from django.utils import timezone from packaging import version from django.template.defaultfilters import slugify @@ -16,7 +14,9 @@ from rest_framework.views import APIView from rest_framework.status import HTTP_400_BAD_REQUEST, HTTP_204_NO_CONTENT from rest_framework.decorators import action +from django.core.exceptions import ObjectDoesNotExist +from dashboard.models import * from recon_note.models import * from reNgine.celery import app from reNgine.common_func import * @@ -28,12 +28,88 @@ from startScan.models import * from startScan.models import EndPoint from targetApp.models import * +from django.core.cache import cache from .serializers import * logger = logging.getLogger(__name__) +class HackerOneProgramViewSet(viewsets.ViewSet): + """ + This class manages the HackerOne Program model, + provides basic fetching of programs and caching + """ + CACHE_KEY = 'hackerone_programs' + CACHE_TIMEOUT = 60 * 30 # 30 minutes + + + def list(self, request): + programs = self.get_cached_programs() + serializer = HackerOneProgramSerializer(programs, many=True) + return Response(serializer.data) + + def get_api_credentials(self): + try: + api_key = HackerOneAPIKey.objects.first() + if not api_key: + raise ObjectDoesNotExist("HackerOne API credentials not found") + return api_key.username, api_key.key + except ObjectDoesNotExist: + raise Exception("HackerOne API credentials not configured") + + @action(detail=False, methods=['get']) + def bookmarked_programs(self, request): + # do not cache bookmarked programs due to the user specific nature + programs = self.fetch_programs_from_hackerone() + bookmarked = [p for p in programs if p['attributes']['bookmarked']] + serializer = HackerOneProgramSerializer(bookmarked, many=True) + return Response(serializer.data) + + @action(detail=False, methods=['get']) + def bounty_programs(self, request): + programs = self.get_cached_programs() + bounty_programs = [p for p in programs if p['attributes']['offers_bounties']] + serializer = HackerOneProgramSerializer(bounty_programs, many=True) + return Response(serializer.data) + + def get_cached_programs(self): + programs = cache.get(self.CACHE_KEY) + if programs is None: + programs = self.fetch_programs_from_hackerone() + cache.set(self.CACHE_KEY, programs, self.CACHE_TIMEOUT) + return programs + + def fetch_programs_from_hackerone(self): + url = 'https://api.hackerone.com/v1/hackers/programs' + headers = {'Accept': 'application/json'} + all_programs = [] + username, api_key = self.get_api_credentials() + + while url: + response = requests.get( + url, + headers=headers, + auth=(username, api_key) + ) + + if response.status_code != 200: + raise Exception(f"HackerOne API request failed with status code {response.status_code}") + + data = response.json() + all_programs.extend(data['data']) + + url = data['links'].get('next') + + return all_programs + + @action(detail=False, methods=['post']) + def refresh_cache(self, request): + programs = self.fetch_programs_from_hackerone() + cache.set(self.CACHE_KEY, programs, self.CACHE_TIMEOUT) + return Response({"status": "Cache refreshed successfully"}) + + class InAppNotificationManagerViewSet(viewsets.ModelViewSet): """ This class manages the notification model, provided CRUD operation on notif model From 51e307a805869688073196a4adc042e33fca7bdc Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 2 Sep 2024 20:24:39 +0530 Subject: [PATCH 156/211] remove debug code --- web/targetApp/templates/target/add.html | 45 ------------------------- 1 file changed, 45 deletions(-) diff --git a/web/targetApp/templates/target/add.html b/web/targetApp/templates/target/add.html index cca6d6a50..a37c05905 100644 --- a/web/targetApp/templates/target/add.html +++ b/web/targetApp/templates/target/add.html @@ -52,51 +52,6 @@
    -
    {% csrf_token %} From 7dbcf19864a56cf0d0638f839a2c44e05dfcda05 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 2 Sep 2024 21:09:12 +0530 Subject: [PATCH 157/211] add bountyhub --- web/templates/base/_items/top_nav.html | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/web/templates/base/_items/top_nav.html b/web/templates/base/_items/top_nav.html index 9f9c92751..bfea74d38 100644 --- a/web/templates/base/_items/top_nav.html +++ b/web/templates/base/_items/top_nav.html @@ -68,6 +68,30 @@ {% endif %}
    +
  • + + +
  • Dashboard
    • +
  • Bounty Hub
    • +
  • {{platform}} Programs
    • +{% endblock breadcrumb_title %} + +{% block main_content %} +
    +
    +
    +
    + +
    +
    +
    + + +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    +
    + +
    +
    +
    +
    + + +
    + +
    +
    + Grammarly +
    +
    Grammarly
    + @grammarly +
    +
    + +
    + Open for Submission + Public Program + Bounty $$$ + Open Scope +
    + +
    +
    My Reports: 0
    +
    My Earnings: $0.00
    +
    + +
    + +
    +
    Since Sep 2017
    +
    USD
    +
    +
    + +
    + See details +
    +
    +
    +
    + + +{% endblock main_content %} + + +{% block page_level_script %} + +{% endblock page_level_script %} diff --git a/web/dashboard/urls.py b/web/dashboard/urls.py index cec484a42..0830493f5 100644 --- a/web/dashboard/urls.py +++ b/web/dashboard/urls.py @@ -40,4 +40,8 @@ 'delete/project/', views.delete_project, name='delete_project'), + path( + '/bountyhub/list/programs', + views.list_bountyhub_programs, + name='list_bountyhub_programs'), ] diff --git a/web/dashboard/views.py b/web/dashboard/views.py index d88e16662..723f3e197 100644 --- a/web/dashboard/views.py +++ b/web/dashboard/views.py @@ -409,3 +409,13 @@ def onboarding(request): context['hackerone_username'] = HackerOneAPIKey.objects.first().username return render(request, 'dashboard/onboarding.html', context) + + + +def list_bountyhub_programs(request, slug): + context = {} + # get parameter to device which platform is being requested + platform = request.GET.get('platform') or 'hackerone' + context['platform'] = platform.capitalize() + + return render(request, 'dashboard/bountyhub_programs.html', context) \ No newline at end of file diff --git a/web/static/custom/custom.css b/web/static/custom/custom.css index 0f28c7916..485ecec08 100644 --- a/web/static/custom/custom.css +++ b/web/static/custom/custom.css @@ -586,4 +586,27 @@ mark{ .optional-text { margin-top: 4px; +} + +.bbp-bookmark-container { + transition: transform 0.3s ease; + z-index: 1; +} +.bbp-bookmark-checkbox:checked + .bbp-bookmark-label i { + color: #ffc107; + transform: scale(1.2); +} +.bbp-bookmark-label i { + transition: all 0.3s ease; + color: #6c757d; +} +.bbp-card:hover .bbp-bookmark-container { + transform: translateY(5px); +} +.bbp-badge { + transition: transform 0.3s ease; + background-color: transparent !important; +} +.bbp-badge:hover { + transform: translateY(-2px); } \ No newline at end of file From 363043fa582a841cdf1b5181f400a08d024f731e Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Tue, 3 Sep 2024 06:22:56 +0530 Subject: [PATCH 159/211] Added clickable cards, added import button on any card selection, fix card designs --- .../dashboard/bountyhub_programs.html | 55 +------- web/static/custom/bountyhub.js | 131 ++++++++++++++++++ web/static/custom/custom.css | 93 +++++++++++++ 3 files changed, 228 insertions(+), 51 deletions(-) create mode 100644 web/static/custom/bountyhub.js diff --git a/web/dashboard/templates/dashboard/bountyhub_programs.html b/web/dashboard/templates/dashboard/bountyhub_programs.html index 42fc451a4..97d42dcfb 100644 --- a/web/dashboard/templates/dashboard/bountyhub_programs.html +++ b/web/dashboard/templates/dashboard/bountyhub_programs.html @@ -60,60 +60,13 @@
    -
    -
    -
    - - -
    - -
    -
    - Grammarly -
    -
    Grammarly
    - @grammarly -
    -
    - -
    - Open for Submission - Public Program - Bounty $$$ - Open Scope -
    - -
    -
    My Reports: 0
    -
    My Earnings: $0.00
    -
    - -
    - -
    -
    Since Sep 2017
    -
    USD
    -
    -
    - -
    - See details -
    -
    -
    - - + {% endblock main_content %} {% block page_level_script %} - + {% endblock page_level_script %} diff --git a/web/static/custom/bountyhub.js b/web/static/custom/bountyhub.js new file mode 100644 index 000000000..eee34109f --- /dev/null +++ b/web/static/custom/bountyhub.js @@ -0,0 +1,131 @@ +// this js file will be used to do everything js related to the bountyhub page + +document.addEventListener('DOMContentLoaded', function() { + const api_url = '/api/hackerone-programs/'; + + Swal.fire({ + title: "Loading HackerOne Programs", + text: "Fetching the latest data...", + icon: "info", + allowOutsideClick: false, + allowEscapeKey: false, + showConfirmButton: false, + didOpen: () => { + Swal.showLoading(); + } + }); + + fetch(api_url, { + method: "GET", + credentials: "same-origin", + headers: { + "X-CSRFToken": getCookie("csrftoken"), + }, + }) + .then(response => { + if (!response.ok) { + throw new Error('Network response was not ok'); + } + return response.json(); + }) + .then(data => { + Swal.close(); + + displayPrograms(data); + }) + .catch(error => { + Swal.close(); + displayErrorMessage("An error occurred while fetching the data. Please try again later. Make sure you have hackerone api key set in your API Vault."); + console.error('Error:', error); + }); + + function displayPrograms(programs) { + const container = document.getElementById('program_cards'); + container.innerHTML = ''; + + if (!programs || programs.length === 0) { + displayErrorMessage("No programs available at the moment."); + return; + } + + programs.forEach(program => { + const { id, attributes } = program; + const card = document.createElement('div'); + card.className = 'col-md-6 col-lg-4 col-xl-3 mb-3'; + card.innerHTML = ` +
    + +
    +
    + ${attributes.name} +
    +
    ${attributes.name}  + ${attributes.bookmarked ? '' : ''} +
    + @${attributes.handle} +
    +
    + +
    + ${attributes.submission_state === 'open' ? 'Open for Submission' : 'Closed'} + ${attributes.state === 'public_mode' ? 'Public Program' : 'Private Program'} + ${attributes.offers_bounties ? 'Bounty $$$' : ''} + ${attributes.open_scope ? 'Open Scope' : ''} +
    + +
    +
    My Reports: ${attributes.number_of_reports_for_user}
    +
    My Earnings: $${attributes.bounty_earned_for_user.toFixed(2)}
    +
    + +
    + +
    +
    Since ${new Date(attributes.started_accepting_at).toLocaleDateString('en-US', { month: 'short', year: 'numeric' })}
    +
    ${attributes.currency.toUpperCase()}
    +
    + See details +
    +
    + `; + + // Initialize tooltips esp for bookmarked programs + const tooltips = document.querySelectorAll('[data-bs-toggle="tooltip"]'); + tooltips.forEach((tooltip) => { + new bootstrap.Tooltip(tooltip); + }); + container.appendChild(card); + }); + } + + function displayErrorMessage(message) { + const container = document.getElementById('program_cards'); + container.innerHTML = ` +
    + +
    + `; + } + + // below has everything to do with card selection and import button + + function toggleCardSelection(card) { + card.classList.toggle('card-selected'); + updateImportButton(); + } + + // lets create func and listeners if cards are selected + const importBtn = document.getElementById('importProgramsBtn'); + const container = document.getElementById('program_cards'); + + function updateImportButton() { + const selectedCards = container.querySelectorAll('.card-selected'); + importBtn.disabled = selectedCards.length === 0; + } + + window.toggleCardSelection = toggleCardSelection; + +}); \ No newline at end of file diff --git a/web/static/custom/custom.css b/web/static/custom/custom.css index 485ecec08..796329927 100644 --- a/web/static/custom/custom.css +++ b/web/static/custom/custom.css @@ -609,4 +609,97 @@ mark{ } .bbp-badge:hover { transform: translateY(-2px); +} + +.card-selectable { + transition: all 0.3s ease; + cursor: pointer; +} +.card-selectable:hover { + transform: translateY(-5px); + box-shadow: 0 10px 20px rgba(0,0,0,0.1); +} +.card-selected { + border: 2px solid #007bff; + background-color: rgba(0,123,255,0.05); +} + +.bbp-bookmark-label { + cursor: pointer; + transition: color 0.3s ease; +} +.bbp-bookmark-label:hover { + color: #ffc107; +} +.bbp-bookmark-checkbox:checked + .bbp-bookmark-label { + color: #ffc107; +} + + +.import-programs-btn { + position: fixed; + bottom: 30px; + left: 50%; + transform: translateX(-50%); + padding: 12px 24px; + font-size: 16px; + font-weight: bold; + color: #fff; + background-color: #007bff; + border: none; + border-radius: 50px; + cursor: pointer; + transition: all 0.3s ease; + box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1); + opacity: 0.7; + pointer-events: none; +} + +.import-programs-btn:not(:disabled) { + opacity: 1; + pointer-events: auto; + animation: enableButton 0.5s ease; +} + +.import-programs-btn:hover { + transform: translateX(-50%) translateY(-5px); + box-shadow: 0 6px 8px rgba(0, 0, 0, 0.15); +} + +.import-programs-btn:active { + transform: translateX(-50%) translateY(-2px); + box-shadow: 0 5px 7px rgba(0, 0, 0, 0.1); +} + +.import-programs-btn i { + margin-right: 8px; +} + +@keyframes enableButton { + 0% { + transform: translateX(-50%) scale(0.9); + opacity: 0.7; + } + 50% { + transform: translateX(-50%) scale(1.05); + } + 100% { + transform: translateX(-50%) scale(1); + opacity: 1; + } +} + +@keyframes disableButton { + 0% { + transform: translateX(-50%) scale(1); + opacity: 1; + } + 100% { + transform: translateX(-50%) scale(0.9); + opacity: 0.7; + } +} + +.import-programs-btn:disabled { + animation: disableButton 0.3s ease forwards; } \ No newline at end of file From dbcb5ee5cff11b4dc5c2d1d65703753362d5c8a3 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Tue, 3 Sep 2024 06:33:32 +0530 Subject: [PATCH 160/211] fix card selection when clicked on button for see details, add ripple effect when cards are selected --- web/static/custom/bountyhub.js | 43 ++++++++++++++++++++++++++-------- web/static/custom/custom.css | 34 +++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/web/static/custom/bountyhub.js b/web/static/custom/bountyhub.js index eee34109f..5372a5503 100644 --- a/web/static/custom/bountyhub.js +++ b/web/static/custom/bountyhub.js @@ -53,7 +53,7 @@ document.addEventListener('DOMContentLoaded', function() { const card = document.createElement('div'); card.className = 'col-md-6 col-lg-4 col-xl-3 mb-3'; card.innerHTML = ` -
    +
    @@ -84,7 +84,7 @@ document.addEventListener('DOMContentLoaded', function() {
    Since ${new Date(attributes.started_accepting_at).toLocaleDateString('en-US', { month: 'short', year: 'numeric' })}
    ${attributes.currency.toUpperCase()}
    - See details + See details
    `; @@ -111,21 +111,44 @@ document.addEventListener('DOMContentLoaded', function() { } // below has everything to do with card selection and import button + const container = document.getElementById('program_cards'); + const importBtn = document.getElementById('importProgramsBtn'); + + + container.addEventListener('click', function(event) { + const card = event.target.closest('.card-selectable'); + if (card) { + toggleCardSelection(event, card); + } + }); + + function toggleCardSelection(event, card) { + if (event.target.closest('#btn-see-details')) { + // If it's the "See details" button, don't toggle selection, maybe we need other actions in the future here + return; + } - function toggleCardSelection(card) { card.classList.toggle('card-selected'); updateImportButton(); } - // lets create func and listeners if cards are selected - const importBtn = document.getElementById('importProgramsBtn'); - const container = document.getElementById('program_cards'); - function updateImportButton() { const selectedCards = container.querySelectorAll('.card-selected'); - importBtn.disabled = selectedCards.length === 0; + const count = selectedCards.length; + + if (count === 0) { + importBtn.disabled = true; + importBtn.innerHTML = ' Import Programs'; + importBtn.classList.remove('button-updated'); + } else { + importBtn.disabled = false; + importBtn.innerHTML = ` Import ${count} Program${count !== 1 ? 's' : ''}`; + + // Trigger the animation + importBtn.classList.remove('button-updated'); + void importBtn.offsetWidth; // Trigger reflow + importBtn.classList.add('button-updated'); + } } - window.toggleCardSelection = toggleCardSelection; - }); \ No newline at end of file diff --git a/web/static/custom/custom.css b/web/static/custom/custom.css index 796329927..80e31ed84 100644 --- a/web/static/custom/custom.css +++ b/web/static/custom/custom.css @@ -702,4 +702,38 @@ mark{ .import-programs-btn:disabled { animation: disableButton 0.3s ease forwards; +} + +/* update on counter increase */ +@keyframes gentlePulse { + 0% { + box-shadow: 0 0 0 0 rgba(0, 123, 255, 0.7); + } + 70% { + box-shadow: 0 0 0 10px rgba(0, 123, 255, 0); + } + 100% { + box-shadow: 0 0 0 0 rgba(0, 123, 255, 0); + } +} + +@keyframes colorTransition { + 0% { + background-color: #007bff; + } + 100% { + background-color: #007bff; + } +} + +.import-programs-btn { + transition: all 0.3s ease; +} + +.import-programs-btn:not(:disabled) { + animation: gentlePulse 1.5s infinite; +} + +.import-programs-btn.button-updated { + animation: gentlePulse 1.5s infinite, colorTransition 0.5s ease; } \ No newline at end of file From 3353f7e14adc77a2fdd5206debc4e2364401191b Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Tue, 3 Sep 2024 06:56:51 +0530 Subject: [PATCH 161/211] add clear all button --- .../dashboard/bountyhub_programs.html | 9 ++++- web/static/custom/bountyhub.js | 27 +++++++++---- web/static/custom/custom.css | 39 ++++++++++++++++--- 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/web/dashboard/templates/dashboard/bountyhub_programs.html b/web/dashboard/templates/dashboard/bountyhub_programs.html index 97d42dcfb..e11074d13 100644 --- a/web/dashboard/templates/dashboard/bountyhub_programs.html +++ b/web/dashboard/templates/dashboard/bountyhub_programs.html @@ -61,9 +61,14 @@
    - + + +
    {% endblock main_content %} diff --git a/web/static/custom/bountyhub.js b/web/static/custom/bountyhub.js index 5372a5503..b244f11bc 100644 --- a/web/static/custom/bountyhub.js +++ b/web/static/custom/bountyhub.js @@ -113,7 +113,7 @@ document.addEventListener('DOMContentLoaded', function() { // below has everything to do with card selection and import button const container = document.getElementById('program_cards'); const importBtn = document.getElementById('importProgramsBtn'); - + const clearBtn = document.getElementById('clearSelectionsLink'); container.addEventListener('click', function(event) { const card = event.target.closest('.card-selectable'); @@ -139,16 +139,29 @@ document.addEventListener('DOMContentLoaded', function() { if (count === 0) { importBtn.disabled = true; importBtn.innerHTML = ' Import Programs'; - importBtn.classList.remove('button-updated'); + clearBtn.style.display = 'none'; } else { importBtn.disabled = false; importBtn.innerHTML = ` Import ${count} Program${count !== 1 ? 's' : ''}`; - - // Trigger the animation - importBtn.classList.remove('button-updated'); - void importBtn.offsetWidth; // Trigger reflow - importBtn.classList.add('button-updated'); + clearBtn.style.display = 'inline'; } } + function clearAllSelections() { + const selectedCards = container.querySelectorAll('.card-selected'); + selectedCards.forEach(card => card.classList.remove('card-selected')); + updateImportButton(); + } + + // clear btn listener + clearBtn.addEventListener('click', function(event) { + event.preventDefault(); + clearAllSelections(); + }); + + + // init state + updateImportButton(); + + }); \ No newline at end of file diff --git a/web/static/custom/custom.css b/web/static/custom/custom.css index 80e31ed84..5d0688bbd 100644 --- a/web/static/custom/custom.css +++ b/web/static/custom/custom.css @@ -635,12 +635,17 @@ mark{ color: #ffc107; } - -.import-programs-btn { +.floating-action-container { position: fixed; bottom: 30px; left: 50%; transform: translateX(-50%); + display: flex; + align-items: center; + gap: 15px; +} + +.import-programs-btn { padding: 12px 24px; font-size: 16px; font-weight: bold; @@ -658,16 +663,15 @@ mark{ .import-programs-btn:not(:disabled) { opacity: 1; pointer-events: auto; - animation: enableButton 0.5s ease; } .import-programs-btn:hover { - transform: translateX(-50%) translateY(-5px); + transform: translateY(-3px); box-shadow: 0 6px 8px rgba(0, 0, 0, 0.15); } .import-programs-btn:active { - transform: translateX(-50%) translateY(-2px); + transform: translateY(-1px); box-shadow: 0 5px 7px rgba(0, 0, 0, 0.1); } @@ -675,6 +679,31 @@ mark{ margin-right: 8px; } +.clear-selections-link { + font-size: 14px; + color: #6c757d; + text-decoration: none; + transition: all 0.3s ease; + display: flex; + align-items: center; + gap: 5px; + padding: 8px 12px; + border-radius: 20px; + background-color: rgba(255, 255, 255, 0.9); + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); +} + +.clear-selections-link:hover { + color: #495057; + background-color: #fff; + transform: translateY(-2px); + box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1); +} + +.clear-selections-link i { + font-size: 16px; +} + @keyframes enableButton { 0% { transform: translateX(-50%) scale(0.9); From 6f292cf3d335cf17545a411c2f55494cfd61f005 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Tue, 3 Sep 2024 07:06:47 +0530 Subject: [PATCH 162/211] add data attribues for filtering --- web/static/custom/bountyhub.js | 65 +++++++++++++++++----------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/web/static/custom/bountyhub.js b/web/static/custom/bountyhub.js index b244f11bc..bf7e06a8b 100644 --- a/web/static/custom/bountyhub.js +++ b/web/static/custom/bountyhub.js @@ -53,42 +53,43 @@ document.addEventListener('DOMContentLoaded', function() { const card = document.createElement('div'); card.className = 'col-md-6 col-lg-4 col-xl-3 mb-3'; card.innerHTML = ` -
    - -
    -
    - ${attributes.name} -
    -
    ${attributes.name}  - ${attributes.bookmarked ? '' : ''} -
    - @${attributes.handle} -
    -
    - -
    - ${attributes.submission_state === 'open' ? 'Open for Submission' : 'Closed'} - ${attributes.state === 'public_mode' ? 'Public Program' : 'Private Program'} - ${attributes.offers_bounties ? 'Bounty $$$' : ''} - ${attributes.open_scope ? 'Open Scope' : ''} -
    - -
    -
    My Reports: ${attributes.number_of_reports_for_user}
    -
    My Earnings: $${attributes.bounty_earned_for_user.toFixed(2)}
    -
    - -
    - -
    -
    Since ${new Date(attributes.started_accepting_at).toLocaleDateString('en-US', { month: 'short', year: 'numeric' })}
    -
    ${attributes.currency.toUpperCase()}
    -
    - See details +
    + +
    +
    + ${attributes.name} +
    +
    ${attributes.name}  + ${attributes.bookmarked ? '' : ''} +
    + @${attributes.handle}
    + +
    + ${attributes.submission_state === 'open' ? 'Open for Submission' : 'Closed'} + ${attributes.state === 'public_mode' ? 'Public Program' : 'Private Program'} + ${attributes.offers_bounties ? 'Bounty $$$' : ''} + ${attributes.open_scope ? 'Open Scope' : ''} +
    + +
    +
    My Reports: ${attributes.number_of_reports_for_user}
    +
    My Earnings: $${attributes.bounty_earned_for_user.toFixed(2)}
    +
    + +
    + +
    +
    Since ${new Date(attributes.started_accepting_at).toLocaleDateString('en-US', { month: 'short', year: 'numeric' })}
    +
    ${attributes.currency.toUpperCase()}
    +
    + See details +
    +
    `; + // Initialize tooltips esp for bookmarked programs const tooltips = document.querySelectorAll('[data-bs-toggle="tooltip"]'); tooltips.forEach((tooltip) => { From 48b72205b650dad0a6d68b8ed097c25da1d48585 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Tue, 3 Sep 2024 07:45:18 +0530 Subject: [PATCH 163/211] add loading animation and filtering --- web/static/custom/bountyhub.js | 50 +++++++++++++++++++++++++++++++++- web/static/custom/custom.css | 9 ++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/web/static/custom/bountyhub.js b/web/static/custom/bountyhub.js index bf7e06a8b..fc5c91dcb 100644 --- a/web/static/custom/bountyhub.js +++ b/web/static/custom/bountyhub.js @@ -51,7 +51,7 @@ document.addEventListener('DOMContentLoaded', function() { programs.forEach(program => { const { id, attributes } = program; const card = document.createElement('div'); - card.className = 'col-md-6 col-lg-4 col-xl-3 mb-3'; + card.className = 'col-md-6 col-lg-4 col-xl-3 mb-3 program-card-wrapper'; card.innerHTML = `
    @@ -96,6 +96,9 @@ document.addEventListener('DOMContentLoaded', function() { new bootstrap.Tooltip(tooltip); }); container.appendChild(card); + + initializeFilter(); + }); } @@ -165,4 +168,49 @@ document.addEventListener('DOMContentLoaded', function() { updateImportButton(); + // we begin filtering here + function initializeFilter() { + const filterSelect = document.querySelector('select[aria-label="Program type"]'); + const container = document.getElementById('program_cards'); + const allCards = container.querySelectorAll('.program-card-wrapper'); + + function filterCards() { + const selectedFilter = filterSelect.value; + + allCards.forEach(card => card.classList.add('filtering-hide')); + + setTimeout(() => { + allCards.forEach(cardWrapper => { + const card = cardWrapper.querySelector('.bbp-card'); + let shouldShow = false; + + switch(selectedFilter) { + case 'All programs': + shouldShow = true; + break; + case 'Bounty Eligible': + shouldShow = card.dataset.offersBounties === 'true'; + break; + case 'VDP': + shouldShow = card.dataset.offersBounties === 'false' || card.dataset.offersBounties === 'null'; + break; + case 'Private Programs': + shouldShow = card.dataset.programState === 'private_mode'; + break; + } + + if (shouldShow) { + cardWrapper.style.display = ''; + setTimeout(() => cardWrapper.classList.remove('filtering-hide'), 10); + } else { + cardWrapper.style.display = 'none'; + } + }); + }, 50); + } + + filterSelect.addEventListener('change', filterCards); + + filterCards(); // init call for filter state + } }); \ No newline at end of file diff --git a/web/static/custom/custom.css b/web/static/custom/custom.css index 5d0688bbd..ecfefffc2 100644 --- a/web/static/custom/custom.css +++ b/web/static/custom/custom.css @@ -703,6 +703,15 @@ mark{ .clear-selections-link i { font-size: 16px; } +.program-card-wrapper { + transition: opacity 0.3s ease-out, transform 0.3s ease-out; +} + +.filtering-hide { + opacity: 0; + transform: scale(0.8); +} + @keyframes enableButton { 0% { From 2b09e06d5138c6c7c5b6ea3620eed9be58dd7f79 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Tue, 3 Sep 2024 07:55:17 +0530 Subject: [PATCH 164/211] add precomputed card data and search feature --- .../dashboard/bountyhub_programs.html | 5 +- web/static/custom/bountyhub.js | 89 ++++++++++++------- 2 files changed, 56 insertions(+), 38 deletions(-) diff --git a/web/dashboard/templates/dashboard/bountyhub_programs.html b/web/dashboard/templates/dashboard/bountyhub_programs.html index e11074d13..198f6de5a 100644 --- a/web/dashboard/templates/dashboard/bountyhub_programs.html +++ b/web/dashboard/templates/dashboard/bountyhub_programs.html @@ -29,7 +29,7 @@
    - +
    @@ -49,9 +49,6 @@
    -
    - -
    diff --git a/web/static/custom/bountyhub.js b/web/static/custom/bountyhub.js index fc5c91dcb..a65e9bb82 100644 --- a/web/static/custom/bountyhub.js +++ b/web/static/custom/bountyhub.js @@ -172,45 +172,66 @@ document.addEventListener('DOMContentLoaded', function() { function initializeFilter() { const filterSelect = document.querySelector('select[aria-label="Program type"]'); const container = document.getElementById('program_cards'); - const allCards = container.querySelectorAll('.program-card-wrapper'); + const allCards = Array.from(container.querySelectorAll('.program-card-wrapper')); + const searchInput = document.querySelector('#search-program-box'); - function filterCards() { + // Pre-compute card data to avoid querying the DOM on each filter/search + const cardData = allCards.map(cardWrapper => { + const card = cardWrapper.querySelector('.bbp-card'); + return { + wrapper: cardWrapper, + name: card.querySelector('h5').textContent.toLowerCase(), + offersBounties: card.dataset.offersBounties === 'true', + isPrivate: card.dataset.programState === 'private_mode' + }; + }); + + let lastFilter = ''; + let lastSearch = ''; + + function filterAndSearchCards() { const selectedFilter = filterSelect.value; - - allCards.forEach(card => card.classList.add('filtering-hide')); - - setTimeout(() => { - allCards.forEach(cardWrapper => { - const card = cardWrapper.querySelector('.bbp-card'); - let shouldShow = false; - - switch(selectedFilter) { - case 'All programs': - shouldShow = true; - break; - case 'Bounty Eligible': - shouldShow = card.dataset.offersBounties === 'true'; - break; - case 'VDP': - shouldShow = card.dataset.offersBounties === 'false' || card.dataset.offersBounties === 'null'; - break; - case 'Private Programs': - shouldShow = card.dataset.programState === 'private_mode'; - break; - } - - if (shouldShow) { - cardWrapper.style.display = ''; - setTimeout(() => cardWrapper.classList.remove('filtering-hide'), 10); - } else { - cardWrapper.style.display = 'none'; - } + const searchTerm = searchInput.value.toLowerCase().trim(); + + if (selectedFilter === lastFilter && searchTerm === lastSearch) return; + lastFilter = selectedFilter; + lastSearch = searchTerm; + + const visibleCards = cardData.filter(({ offersBounties, isPrivate, name }) => { + let shouldShow = true; + + switch(selectedFilter) { + case 'Bounty Eligible': + shouldShow = offersBounties; + break; + case 'VDP': + shouldShow = !offersBounties; + break; + case 'Private Programs': + shouldShow = isPrivate; + break; + } + + return shouldShow && (!searchTerm || name.includes(searchTerm)); + }); + + // Batch DOM updates to avoid reflows and make the transition smoother + requestAnimationFrame(() => { + cardData.forEach(({ wrapper }) => { + wrapper.style.display = 'none'; + wrapper.classList.add('filtering-hide'); }); - }, 50); + + visibleCards.forEach(({ wrapper }) => { + wrapper.style.display = ''; + wrapper.classList.remove('filtering-hide'); + }); + }); } - filterSelect.addEventListener('change', filterCards); + filterSelect.addEventListener('change', filterAndSearchCards); + searchInput.addEventListener('input', filterAndSearchCards); - filterCards(); // init call for filter state + filterAndSearchCards(); // init call } }); \ No newline at end of file From 2204ffe94c86e32ce4453d9df27dc482d49202f2 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Tue, 3 Sep 2024 08:08:22 +0530 Subject: [PATCH 165/211] hide closed programs by default --- .../templates/dashboard/bountyhub_programs.html | 10 +++++++++- web/static/custom/bountyhub.js | 17 +++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/web/dashboard/templates/dashboard/bountyhub_programs.html b/web/dashboard/templates/dashboard/bountyhub_programs.html index 198f6de5a..cac3b6e49 100644 --- a/web/dashboard/templates/dashboard/bountyhub_programs.html +++ b/web/dashboard/templates/dashboard/bountyhub_programs.html @@ -24,7 +24,7 @@
    -
    +
    @@ -49,6 +49,14 @@
    +
    +
    + + +
    +
    diff --git a/web/static/custom/bountyhub.js b/web/static/custom/bountyhub.js index a65e9bb82..7a2dfb16d 100644 --- a/web/static/custom/bountyhub.js +++ b/web/static/custom/bountyhub.js @@ -174,6 +174,7 @@ document.addEventListener('DOMContentLoaded', function() { const container = document.getElementById('program_cards'); const allCards = Array.from(container.querySelectorAll('.program-card-wrapper')); const searchInput = document.querySelector('#search-program-box'); + const showClosedCheckbox = document.getElementById('show-closed-programs'); // Pre-compute card data to avoid querying the DOM on each filter/search const cardData = allCards.map(cardWrapper => { @@ -182,7 +183,8 @@ document.addEventListener('DOMContentLoaded', function() { wrapper: cardWrapper, name: card.querySelector('h5').textContent.toLowerCase(), offersBounties: card.dataset.offersBounties === 'true', - isPrivate: card.dataset.programState === 'private_mode' + isPrivate: card.dataset.programState === 'private_mode', + isClosed: card.querySelector('.badge').textContent.trim() !== 'Open for Submission' }; }); @@ -192,12 +194,14 @@ document.addEventListener('DOMContentLoaded', function() { function filterAndSearchCards() { const selectedFilter = filterSelect.value; const searchTerm = searchInput.value.toLowerCase().trim(); + const showClosed = showClosedCheckbox.checked; - if (selectedFilter === lastFilter && searchTerm === lastSearch) return; + if (selectedFilter === lastFilter && searchTerm === lastSearch && showClosed === lastShowClosed) return; lastFilter = selectedFilter; lastSearch = searchTerm; + lastShowClosed = showClosed; - const visibleCards = cardData.filter(({ offersBounties, isPrivate, name }) => { + const visibleCards = cardData.filter(({ offersBounties, isPrivate, name, isClosed }) => { let shouldShow = true; switch(selectedFilter) { @@ -212,7 +216,11 @@ document.addEventListener('DOMContentLoaded', function() { break; } - return shouldShow && (!searchTerm || name.includes(searchTerm)); + shouldShow = shouldShow && (!searchTerm || name.includes(searchTerm)); + + shouldShow = shouldShow && (showClosed || !isClosed); + + return shouldShow; }); // Batch DOM updates to avoid reflows and make the transition smoother @@ -231,6 +239,7 @@ document.addEventListener('DOMContentLoaded', function() { filterSelect.addEventListener('change', filterAndSearchCards); searchInput.addEventListener('input', filterAndSearchCards); + showClosedCheckbox.addEventListener('change', filterAndSearchCards); filterAndSearchCards(); // init call } From 67fa86764deb6aec3cbfbf83126dc9245334d112 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Tue, 3 Sep 2024 19:53:57 +0530 Subject: [PATCH 166/211] add new button in latest vdp from 3 months old --- web/static/custom/bountyhub.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/web/static/custom/bountyhub.js b/web/static/custom/bountyhub.js index 7a2dfb16d..ef5b8930a 100644 --- a/web/static/custom/bountyhub.js +++ b/web/static/custom/bountyhub.js @@ -38,16 +38,15 @@ document.addEventListener('DOMContentLoaded', function() { displayErrorMessage("An error occurred while fetching the data. Please try again later. Make sure you have hackerone api key set in your API Vault."); console.error('Error:', error); }); - function displayPrograms(programs) { const container = document.getElementById('program_cards'); container.innerHTML = ''; - + if (!programs || programs.length === 0) { displayErrorMessage("No programs available at the moment."); return; } - + programs.forEach(program => { const { id, attributes } = program; const card = document.createElement('div'); @@ -62,15 +61,16 @@ document.addEventListener('DOMContentLoaded', function() {
    ${attributes.name}  ${attributes.bookmarked ? '' : ''}
    - @${attributes.handle} + @${attributes.handle}
    ${attributes.submission_state === 'open' ? 'Open for Submission' : 'Closed'} ${attributes.state === 'public_mode' ? 'Public Program' : 'Private Program'} - ${attributes.offers_bounties ? 'Bounty $$$' : ''} + ${attributes.offers_bounties ? 'Bounty $$$' : 'VDP'} ${attributes.open_scope ? 'Open Scope' : ''} + ${isProgramNew(attributes.started_accepting_at) ? ' New' : ''}
    @@ -102,6 +102,13 @@ document.addEventListener('DOMContentLoaded', function() { }); } + function isProgramNew(startedAcceptingAt) { + const threeMonthsAgo = new Date(); + threeMonthsAgo.setMonth(threeMonthsAgo.getMonth() - 3); + const startedAcceptingDate = new Date(startedAcceptingAt); + return startedAcceptingDate > threeMonthsAgo; + } + function displayErrorMessage(message) { const container = document.getElementById('program_cards'); container.innerHTML = ` @@ -127,7 +134,7 @@ document.addEventListener('DOMContentLoaded', function() { }); function toggleCardSelection(event, card) { - if (event.target.closest('#btn-see-details')) { + if (event.target.closest('#btn-see-details') || event.target.closest('#handle-link')) { // If it's the "See details" button, don't toggle selection, maybe we need other actions in the future here return; } From bfcdc1008346ef0d2d0b2e0d8c1dcab3406ea856 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Tue, 3 Sep 2024 20:20:40 +0530 Subject: [PATCH 167/211] Add sorting functionality to HackerOne programs --- web/api/views.py | 17 ++- .../dashboard/bountyhub_programs.html | 10 +- web/static/custom/bountyhub.js | 133 +++++++++++++----- web/static/custom/custom.css | 3 +- 4 files changed, 122 insertions(+), 41 deletions(-) diff --git a/web/api/views.py b/web/api/views.py index b2fdcc4a8..af610f281 100644 --- a/web/api/views.py +++ b/web/api/views.py @@ -43,9 +43,24 @@ class HackerOneProgramViewSet(viewsets.ViewSet): CACHE_KEY = 'hackerone_programs' CACHE_TIMEOUT = 60 * 30 # 30 minutes - def list(self, request): + sort_by = request.query_params.get('sort_by', 'age') + sort_order = request.query_params.get('sort_order', 'desc') # Changed default to 'desc' + programs = self.get_cached_programs() + + if sort_by == 'name': + programs = sorted(programs, key=lambda x: x['attributes']['name'].lower(), + reverse=(sort_order.lower() == 'desc')) + elif sort_by == 'reports': + programs = sorted(programs, key=lambda x: x['attributes'].get('number_of_reports_for_user', 0), + reverse=(sort_order.lower() == 'desc')) + elif sort_by == 'age': + programs = sorted(programs, + key=lambda x: datetime.strptime(x['attributes'].get('started_accepting_at', '1970-01-01T00:00:00.000Z'), '%Y-%m-%dT%H:%M:%S.%fZ'), + reverse=(sort_order.lower() == 'desc') + ) + serializer = HackerOneProgramSerializer(programs, many=True) return Response(serializer.data) diff --git a/web/dashboard/templates/dashboard/bountyhub_programs.html b/web/dashboard/templates/dashboard/bountyhub_programs.html index cac3b6e49..98a820d1d 100644 --- a/web/dashboard/templates/dashboard/bountyhub_programs.html +++ b/web/dashboard/templates/dashboard/bountyhub_programs.html @@ -43,10 +43,12 @@
    diff --git a/web/static/custom/bountyhub.js b/web/static/custom/bountyhub.js index ef5b8930a..25c75ca36 100644 --- a/web/static/custom/bountyhub.js +++ b/web/static/custom/bountyhub.js @@ -1,43 +1,69 @@ // this js file will be used to do everything js related to the bountyhub page document.addEventListener('DOMContentLoaded', function() { - const api_url = '/api/hackerone-programs/'; - - Swal.fire({ - title: "Loading HackerOne Programs", - text: "Fetching the latest data...", - icon: "info", - allowOutsideClick: false, - allowEscapeKey: false, - showConfirmButton: false, - didOpen: () => { - Swal.showLoading(); + + function fetchPrograms(isSortingRequest = false){ + let api_url = '/api/hackerone-programs/'; + const sortParams = updateSortingParams(); + const queryParams = new URLSearchParams(sortParams).toString(); + + if (queryParams){ + api_url += '?' + queryParams } - }); - fetch(api_url, { - method: "GET", - credentials: "same-origin", - headers: { - "X-CSRFToken": getCookie("csrftoken"), - }, - }) - .then(response => { - if (!response.ok) { - throw new Error('Network response was not ok'); + + if (isSortingRequest) { + Swal.fire({ + title: 'Sorting...', + text: 'Please wait', + allowOutsideClick: false, + allowEscapeKey: false, + showConfirmButton: false, + willOpen: () => { + Swal.showLoading(); + }, + customClass: { + popup: 'small-swal' + } + }); + } else { + Swal.fire({ + title: "Loading HackerOne Programs", + text: "Fetching the latest data...", + icon: "info", + allowOutsideClick: false, + allowEscapeKey: false, + showConfirmButton: false, + didOpen: () => { + Swal.showLoading(); + } + }); } - return response.json(); - }) - .then(data => { - Swal.close(); - - displayPrograms(data); - }) - .catch(error => { - Swal.close(); - displayErrorMessage("An error occurred while fetching the data. Please try again later. Make sure you have hackerone api key set in your API Vault."); - console.error('Error:', error); - }); + + fetch(api_url, { + method: "GET", + credentials: "same-origin", + headers: { + "X-CSRFToken": getCookie("csrftoken"), + }, + }) + .then(response => { + if (!response.ok) { + throw new Error('Network response was not ok'); + } + return response.json(); + }) + .then(data => { + Swal.close(); + displayPrograms(data); + }) + .catch(error => { + Swal.close(); + displayErrorMessage("An error occurred while fetching the hackerone programs. Please try again later. Make sure you have hackerone api key set in your API Vault."); + console.error('Error:', error); + }); + } + function displayPrograms(programs) { const container = document.getElementById('program_cards'); container.innerHTML = ''; @@ -250,4 +276,43 @@ document.addEventListener('DOMContentLoaded', function() { filterAndSearchCards(); // init call } + + // in this function, we handle the sort select + function updateSortingParams() { + const sortSelect = document.getElementById('sort-select'); + let sortBy, sortOrder; + + if (sortSelect.value === 'Sort by' || !sortSelect.value) { + sortBy = 'age'; + sortOrder = 'desc'; + } else { + [sortBy, sortOrder] = sortSelect.value.split('-'); + } + + let apiSortBy; + + switch (sortBy) { + case 'name': + apiSortBy = 'name'; + break; + case 'reports': + apiSortBy = 'reports'; + break; + case 'posted': + apiSortBy = 'age'; + break; + default: + apiSortBy = 'age'; + } + + return { sort_by: apiSortBy, sort_order: sortOrder }; + } + + + // init call to fetch programs + const sortSelect = document.getElementById('sort-select'); + sortSelect.addEventListener('change', () => fetchPrograms(true)); + + fetchPrograms(false); + }); \ No newline at end of file diff --git a/web/static/custom/custom.css b/web/static/custom/custom.css index ecfefffc2..309f64324 100644 --- a/web/static/custom/custom.css +++ b/web/static/custom/custom.css @@ -647,8 +647,7 @@ mark{ .import-programs-btn { padding: 12px 24px; - font-size: 16px; - font-weight: bold; + font-size: 14px; color: #fff; background-color: #007bff; border: none; From 08e010cbcf5b15534762aa10d89ba30b0fca30e2 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Tue, 3 Sep 2024 20:29:25 +0530 Subject: [PATCH 168/211] debounce search, optimize query --- web/static/custom/bountyhub.js | 399 ++++++++++++++------------------- 1 file changed, 163 insertions(+), 236 deletions(-) diff --git a/web/static/custom/bountyhub.js b/web/static/custom/bountyhub.js index 25c75ca36..cefa8b5b4 100644 --- a/web/static/custom/bountyhub.js +++ b/web/static/custom/bountyhub.js @@ -1,242 +1,141 @@ -// this js file will be used to do everything js related to the bountyhub page +// all the js code for the bountyhub page document.addEventListener('DOMContentLoaded', function() { + let allPrograms = []; + const container = document.getElementById('program_cards'); + const importBtn = document.getElementById('importProgramsBtn'); + const clearBtn = document.getElementById('clearSelectionsLink'); + const filterSelect = document.querySelector('select[aria-label="Program type"]'); + const searchInput = document.querySelector('#search-program-box'); + const showClosedCheckbox = document.getElementById('show-closed-programs'); + const sortSelect = document.getElementById('sort-select'); + + // Debounce function for search input to avoid making too many requests + const debounce = (func, delay) => { + let timeoutId; + return (...args) => { + clearTimeout(timeoutId); + timeoutId = setTimeout(() => func.apply(null, args), delay); + }; + }; + + async function fetchPrograms(isSortingRequest = false) { + if (isSortingRequest) { + showLoadingIndicator("Sorting..."); + } else { + showLoadingIndicator("Loading HackerOne Programs"); + } - function fetchPrograms(isSortingRequest = false){ let api_url = '/api/hackerone-programs/'; const sortParams = updateSortingParams(); const queryParams = new URLSearchParams(sortParams).toString(); - if (queryParams){ - api_url += '?' + queryParams + if (queryParams) { + api_url += '?' + queryParams; } - - if (isSortingRequest) { - Swal.fire({ - title: 'Sorting...', - text: 'Please wait', - allowOutsideClick: false, - allowEscapeKey: false, - showConfirmButton: false, - willOpen: () => { - Swal.showLoading(); + try { + const response = await fetch(api_url, { + method: "GET", + credentials: "same-origin", + headers: { + "X-CSRFToken": getCookie("csrftoken"), }, - customClass: { - popup: 'small-swal' - } - }); - } else { - Swal.fire({ - title: "Loading HackerOne Programs", - text: "Fetching the latest data...", - icon: "info", - allowOutsideClick: false, - allowEscapeKey: false, - showConfirmButton: false, - didOpen: () => { - Swal.showLoading(); - } }); - } - - fetch(api_url, { - method: "GET", - credentials: "same-origin", - headers: { - "X-CSRFToken": getCookie("csrftoken"), - }, - }) - .then(response => { + if (!response.ok) { throw new Error('Network response was not ok'); } - return response.json(); - }) - .then(data => { - Swal.close(); + + const data = await response.json(); + allPrograms = data; displayPrograms(data); - }) - .catch(error => { - Swal.close(); + } catch (error) { displayErrorMessage("An error occurred while fetching the hackerone programs. Please try again later. Make sure you have hackerone api key set in your API Vault."); console.error('Error:', error); - }); + } finally { + hideLoadingIndicator(); + } } function displayPrograms(programs) { - const container = document.getElementById('program_cards'); - container.innerHTML = ''; + container.innerHTML = ''; // clear up the html content if (!programs || programs.length === 0) { displayErrorMessage("No programs available at the moment."); return; } + const fragment = document.createDocumentFragment(); + const template = document.createElement('template'); + programs.forEach(program => { - const { id, attributes } = program; - const card = document.createElement('div'); - card.className = 'col-md-6 col-lg-4 col-xl-3 mb-3 program-card-wrapper'; - card.innerHTML = ` -
    - -
    -
    - ${attributes.name} -
    -
    ${attributes.name}  - ${attributes.bookmarked ? '' : ''} -
    - @${attributes.handle} + const { attributes } = program; + template.innerHTML = ` +
    +
    +
    + +
    - -
    - ${attributes.submission_state === 'open' ? 'Open for Submission' : 'Closed'} - ${attributes.state === 'public_mode' ? 'Public Program' : 'Private Program'} - ${attributes.offers_bounties ? 'Bounty $$$' : 'VDP'} - ${attributes.open_scope ? 'Open Scope' : ''} - ${isProgramNew(attributes.started_accepting_at) ? ' New' : ''} -
    - -
    -
    My Reports: ${attributes.number_of_reports_for_user}
    -
    My Earnings: $${attributes.bounty_earned_for_user.toFixed(2)}
    -
    - -
    - -
    -
    Since ${new Date(attributes.started_accepting_at).toLocaleDateString('en-US', { month: 'short', year: 'numeric' })}
    -
    ${attributes.currency.toUpperCase()}
    -
    - See details -
    -
    `; - - - // Initialize tooltips esp for bookmarked programs - const tooltips = document.querySelectorAll('[data-bs-toggle="tooltip"]'); - tooltips.forEach((tooltip) => { - new bootstrap.Tooltip(tooltip); - }); - container.appendChild(card); - - initializeFilter(); - + const cardNode = template.content.firstElementChild.cloneNode(true); + cardNode.querySelector('.card-body').innerHTML = generateCardContent(attributes); + fragment.appendChild(cardNode); }); - } - function isProgramNew(startedAcceptingAt) { - const threeMonthsAgo = new Date(); - threeMonthsAgo.setMonth(threeMonthsAgo.getMonth() - 3); - const startedAcceptingDate = new Date(startedAcceptingAt); - return startedAcceptingDate > threeMonthsAgo; + container.appendChild(fragment); + initializeFilter(); } - function displayErrorMessage(message) { - const container = document.getElementById('program_cards'); - container.innerHTML = ` -
    -
    Critical Severity is found. (Default)
    + @@ -53,12 +58,17 @@ {% for organization in organizations.all %} + - + @@ -89,7 +99,6 @@ {% endblock main_content %} - {% block page_level_script %} @@ -106,7 +115,7 @@ "dom": "<'dt--top-section'<'row'<'col-12 col-sm-6 d-flex justify-content-sm-start justify-content-center mt-sm-0 mt-3'f><'col-12 col-sm-6 d-flex justify-content-sm-end justify-content-center'l>>>" + "<'table-responsive'tr>" + "<'dt--bottom-section d-sm-flex justify-content-sm-between text-center'<'dt--pages-count mb-sm-0 mb-3'i><'dt--pagination'p>>", - "lengthMenu": [5, 10, 20, 30, 50, 100, 500, 1000], + "lengthMenu": [5, 10, 20, 30, 50, 100], "pageLength": 10, "initComplete": function(settings, json) { $('[data-toggle="tooltip"]').tooltip(); @@ -115,25 +124,77 @@ multiCheck(list_organization_table); }); -function get_target_modal(organization_id){ - $('.modal-title').html(`Loading targets...`); - $('#modal_dialog').modal('show'); - $('.modal-text').empty(); - $('.modal-text').append(``); - $.getJSON(`/api/queryTargetsInOrganization/?organization_id=${organization_id}&format=json`, function(data) { - organization_name = htmlEncode(data['organization'][0]['name']); - $('#modal-loader').empty(); - $('.modal-title').html(`${data['domains'].length} targets in organization ${organization_name}`); - $('#modal-content').append(`
      `); - for (domain in data['domains']){ - domain_obj = data['domains'][domain]; - $("#target-detail-modal-ul").append(`
    • ${domain_obj['name']}
      • `); - } - }).fail(function(){ - $('#modal-loader').empty(); - $("#modal-content").append(`

      Error loading Vulnerabilities Summary

      `); - }); +function selectAllOrganizations(checkbox) { + $('.organization-checkbox').prop('checked', checkbox.checked); + toggleMultipleOrganizationButton(); } +function toggleMultipleOrganizationButton() { + var checkedCount = $('.organization-checkbox:checked').length; + if (checkedCount > 0) { + $("#delete_multiple_button").removeClass("disabled"); + } else { + $("#delete_multiple_button").addClass("disabled"); + } + + // Update the "select all" checkbox + var totalCount = $('.organization-checkbox').length; + $('#select_all_checkbox').prop('checked', checkedCount === totalCount); +} + +function deleteMultipleOrganizations() { + var checkedCount = $('.organization-checkbox:checked').length; + if (!checkedCount) { + swal({ + title: '', + text: "Oops! No organizations have been selected!", + type: 'error', + padding: '2em' + }) + } else { + swal.queue([{ + title: 'Are you sure you want to delete ' + checkedCount + ' Organizations?', + text: "This action is irreversible.", + type: 'warning', + showCancelButton: true, + confirmButtonText: 'Delete', + padding: '2em', + showLoaderOnConfirm: true, + preConfirm: function() { + var selectedOrganizations = []; + + $('.organization-checkbox:checked').each(function() { + selectedOrganizations.push($(this).val()); + }); + + var data = {'type': 'organization', 'rows': selectedOrganizations} + + return fetch('/api/action/rows/delete/', { + method: 'post', + headers: { + "X-CSRFToken": getCookie("csrftoken"), + "Content-Type": "application/json" + }, + body: JSON.stringify(data) + }).then(res => res.json()) + .then(function (response) { + if (response['status']) { + // reload page + location.reload(); + } + else { + Snackbar.show({ + text: 'Oops! Failed to delete organizations!', + pos: 'top-right', + duration: 1500, + actionTextColor: '#fff', + backgroundColor: '#e7515a', + }); + } + }); + } + }]) + } +} -{% endblock page_level_script %} +{% endblock page_level_script %} \ No newline at end of file From 205f2860580b4c60d200d44a2b9833ddf7e7496d Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Thu, 5 Sep 2024 20:42:24 +0530 Subject: [PATCH 187/211] update onboarding ui --- .../templates/dashboard/onboarding.html | 274 ++++++++---------- 1 file changed, 115 insertions(+), 159 deletions(-) diff --git a/web/dashboard/templates/dashboard/onboarding.html b/web/dashboard/templates/dashboard/onboarding.html index 55685ecf1..969ffd230 100644 --- a/web/dashboard/templates/dashboard/onboarding.html +++ b/web/dashboard/templates/dashboard/onboarding.html @@ -8,68 +8,33 @@ @@ -77,114 +42,105 @@
      +
      + +
      +       		Organization Description Total Targets
      +
      + +
      +       		{{ organization.name|capfirst }} {% if organization.description %}{{organization.description}}{% endif %} {{organization.get_domains.count}}{{organization.insert_date|naturaltime}}{{organization.insert_date|naturaltime}} -
      +
      {% if user|can:'initiate_scans_subscans' %}   Initiate Scan {% endif %} @@ -77,7 +87,7 @@   Delete Organization {% endif %}
      -
      +