From a490426dba8ab3e95cbe3066a4c4f475e3167e24 Mon Sep 17 00:00:00 2001 From: Aidan Holland Date: Mon, 8 May 2023 13:19:41 -0400 Subject: [PATCH 1/3] feat: update for certs v2 --- .gitignore | 2 + README.md | 13 ++--- censys-subdomain-finder.py | 110 +++++++++++++++++++++++++------------ cli.py | 34 ++++++------ requirements.txt | 2 +- 5 files changed, 100 insertions(+), 61 deletions(-) diff --git a/.gitignore b/.gitignore index 29f322e..3e0d7b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.pyc +.venv +venv .env diff --git a/README.md b/README.md index 60db5d8..a43d085 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Censys subdomain finder -This is a tool to enumerate subdomains using the Certificate Transparency logs stored by [Censys](https://censys.io). It should return any subdomain who has ever been issued a SSL certificate by a public CA. +This is a tool to enumerate subdomains using the Certificate Transparency logs stored in [Censys Search](https://search.censys.io). It should return any subdomain who has ever been issued a SSL certificate by a public CA. See it in action: @@ -57,8 +57,8 @@ $ python censys-subdomain-finder.py github.com ## Setup -1) Register an account (free) on https://censys.io/register -2) Browse to https://censys.io/account, and set two environment variables with your API ID and API secret: +1. Register an account (free) on +2. Browse to , and set two environment variables with your API ID and API secret: ```shell export CENSYS_API_ID=... @@ -73,13 +73,13 @@ $ python censys-subdomain-finder.py github.com Then edit the `.env` file and set the values for `CENSYS_API_ID` and `CENSYS_API_SECRET`. -3) Clone the repository: +3. Clone the repository: ```shell git clone https://github.com/christophetd/censys-subdomain-finder.git ``` -4) Install the dependencies in a virtualenv: +4. Install the dependencies in a virtualenv: ```shell cd censys-subdomain-finder @@ -124,10 +124,9 @@ optional arguments: CENSYS_API_SECRET environment variable (default: None) ``` - ## Compatibility -Should run on Python 2.7 and 3.5. +Should run on Python 3.7+. ## Notes diff --git a/censys-subdomain-finder.py b/censys-subdomain-finder.py index 8fec9e0..f4bfb6e 100644 --- a/censys-subdomain-finder.py +++ b/censys-subdomain-finder.py @@ -1,7 +1,11 @@ #!/usr/bin/env python3 -from censys.search import CensysCertificates -import censys +from censys.search import CensysCerts +from censys.common.exceptions import ( + CensysUnauthorizedException, + CensysRateLimitExceededException, + CensysException, +) from dotenv import load_dotenv import sys import cli @@ -10,50 +14,75 @@ load_dotenv() -NON_COMMERCIAL_API_LIMIT = 1000 +USER_AGENT = f"{CensysCerts.DEFAULT_USER_AGENT} (censys-subdomain-finder; +https://github.com/christophetd/censys-subdomain-finder)" + # Finds subdomains of a domain using Censys API def find_subdomains(domain, api_id, api_secret, limit_results): try: - censys_certificates = CensysCertificates(api_id=api_id, api_secret=api_secret) - certificate_query = 'parsed.names: %s' % domain + censys_certificates = CensysCerts( + api_id=api_id, api_secret=api_secret, user_agent=USER_AGENT + ) + certificate_query = "names: %s" % domain if limit_results: - certificates_search_results = censys_certificates.search(certificate_query, fields=['parsed.names'], max_records=NON_COMMERCIAL_API_LIMIT) + certificates_search_results = censys_certificates.search( + certificate_query, fields=["names"], per_page=100, pages=10 + ) else: - certificates_search_results = censys_certificates.search(certificate_query, fields=['parsed.names']) + certificates_search_results = censys_certificates.search( + certificate_query, fields=["names"] + ) # Flatten the result, and remove duplicates subdomains = [] - for search_result in certificates_search_results: - subdomains.extend(search_result['parsed.names']) + for page in certificates_search_results: + for search_result in page: + subdomains.extend(search_result["names"]) return set(subdomains) - except censys.common.exceptions.CensysUnauthorizedException: - sys.stderr.write('[-] Your Censys credentials look invalid.\n') + except CensysUnauthorizedException: + sys.stderr.write("[-] Your Censys credentials look invalid.\n") exit(1) - except censys.common.exceptions.CensysRateLimitExceededException: - sys.stderr.write('[-] Looks like you exceeded your Censys account limits rate. Exiting\n') + except CensysRateLimitExceededException: + sys.stderr.write( + "[-] Looks like you exceeded your Censys account limits rate. Exiting\n" + ) return set(subdomains) - except censys.common.exceptions.CensysException as e: + except CensysException as e: # catch the Censys Base exception, example "only 1000 first results are available" - sys.stderr.write('[-] Something bad happened, ' + repr(e)) + sys.stderr.write("[-] Something bad happened, " + repr(e)) return set(subdomains) + # Filters out uninteresting subdomains def filter_subdomains(domain, subdomains): - return [ subdomain for subdomain in subdomains if '*' not in subdomain and subdomain.endswith(domain) ] + return [ + subdomain + for subdomain in subdomains + if "*" not in subdomain and subdomain.endswith(domain) and subdomain != domain + ] + # Prints the list of found subdomains to stdout -def print_subdomains(domain, subdomains, time_ellapsed): +def print_subdomains(domain, subdomains, time_elapsed): if len(subdomains) == 0: - print('[-] Did not find any subdomain') + print("[-] Did not find any subdomain") return - print('[*] Found %d unique subdomain%s of %s in ~%s seconds\n' % (len(subdomains), 's' if len(subdomains) > 1 else '', domain, str(time_ellapsed))) + print( + "[*] Found %d unique subdomain%s of %s in ~%s seconds\n" + % ( + len(subdomains), + "s" if len(subdomains) > 1 else "", + domain, + str(time_elapsed), + ) + ) for subdomain in subdomains: - print(' - ' + subdomain) + print(" - " + subdomain) + + print("") - print('') # Saves the list of found subdomains to an output file def save_subdomains_to_file(subdomains, output_file): @@ -61,33 +90,42 @@ def save_subdomains_to_file(subdomains, output_file): return try: - with open(output_file, 'w') as f: + with open(output_file, "w") as f: for subdomain in subdomains: - f.write(subdomain + '\n') + f.write(subdomain + "\n") - print('[*] Wrote %d subdomains to %s' % (len(subdomains), os.path.abspath(output_file))) + print( + "[*] Wrote %d subdomains to %s" + % (len(subdomains), os.path.abspath(output_file)) + ) except IOError as e: - sys.stderr.write('[-] Unable to write to output file %s : %s\n' % (output_file, e)) + sys.stderr.write( + "[-] Unable to write to output file %s : %s\n" % (output_file, e) + ) + def main(domain, output_file, censys_api_id, censys_api_secret, limit_results): - print('[*] Searching Censys for subdomains of %s' % domain) + print("[*] Searching Censys for subdomains of %s" % domain) start_time = time.time() - subdomains = find_subdomains(domain, censys_api_id, censys_api_secret, limit_results) + subdomains = find_subdomains( + domain, censys_api_id, censys_api_secret, limit_results + ) subdomains = filter_subdomains(domain, subdomains) end_time = time.time() - time_ellapsed = round(end_time - start_time, 1) - print_subdomains(domain, subdomains, time_ellapsed) + time_elapsed = round(end_time - start_time, 1) + print_subdomains(domain, subdomains, time_elapsed) save_subdomains_to_file(subdomains, output_file) + if __name__ == "__main__": args = cli.parser.parse_args() censys_api_id = None censys_api_secret = None - if 'CENSYS_API_ID' in os.environ and 'CENSYS_API_SECRET' in os.environ: - censys_api_id = os.environ['CENSYS_API_ID'] - censys_api_secret = os.environ['CENSYS_API_SECRET'] + if "CENSYS_API_ID" in os.environ and "CENSYS_API_SECRET" in os.environ: + censys_api_id = os.environ["CENSYS_API_ID"] + censys_api_secret = os.environ["CENSYS_API_SECRET"] if args.censys_api_id and args.censys_api_secret: censys_api_id = args.censys_api_id @@ -95,10 +133,12 @@ def main(domain, output_file, censys_api_id, censys_api_secret, limit_results): limit_results = not args.commercial if limit_results: - print('[*] Applying non-commerical limits (' + str(NON_COMMERCIAL_API_LIMIT) + ' results at most)') + print("[*] Applying non-commercial limits (1000 results at most)") - if None in [ censys_api_id, censys_api_secret ]: - sys.stderr.write('[!] Please set your Censys API ID and secret from your environment (CENSYS_API_ID and CENSYS_API_SECRET) or from the command line.\n') + if None in [censys_api_id, censys_api_secret]: + sys.stderr.write( + "[!] Please set your Censys API ID and secret from your environment (CENSYS_API_ID and CENSYS_API_SECRET) or from the command line.\n" + ) exit(1) main(args.domain, args.output_file, censys_api_id, censys_api_secret, limit_results) diff --git a/cli.py b/cli.py index 3aa7e98..f0d5799 100644 --- a/cli.py +++ b/cli.py @@ -2,33 +2,31 @@ parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) -parser.add_argument( - 'domain', - help = 'The domain to scan' -) +parser.add_argument("domain", help="The domain to scan") parser.add_argument( - '-o', '--output', - help = 'A file to output the list of subdomains to', - dest = 'output_file' + "-o", + "--output", + help="A file to output the list of subdomains to", + dest="output_file", ) parser.add_argument( - '--censys-api-id', - help = 'Censys API ID. Can also be defined using the CENSYS_API_ID environment variable', - dest = 'censys_api_id' + "--censys-api-id", + help="Censys API ID. Can also be defined using the CENSYS_API_ID environment variable", + dest="censys_api_id", ) parser.add_argument( - '--censys-api-secret', - help = 'Censys API secret. Can also be defined using the CENSYS_API_SECRET environment variable', - dest = 'censys_api_secret' + "--censys-api-secret", + help="Censys API secret. Can also be defined using the CENSYS_API_SECRET environment variable", + dest="censys_api_secret", ) parser.add_argument( - '--commercial', - help = 'Don\'t limit search results (for commercial accounts)', - dest = 'commercial', - action='store_true', - default=False + "--commercial", + help="Don't limit search results (for commercial accounts)", + dest="commercial", + action="store_true", + default=False, ) diff --git a/requirements.txt b/requirements.txt index b1876e9..7ce0281 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -censys==2.1.2 +censys==2.2.0 python-dotenv From 7920b5ebd8c683c8f06aabf01396785f86e37562 Mon Sep 17 00:00:00 2001 From: Aidan Holland Date: Mon, 8 May 2023 17:48:12 -0400 Subject: [PATCH 2/3] fix: page limit --- censys-subdomain-finder.py | 14 +++++++++++--- requirements.txt | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/censys-subdomain-finder.py b/censys-subdomain-finder.py index f4bfb6e..cbad959 100644 --- a/censys-subdomain-finder.py +++ b/censys-subdomain-finder.py @@ -16,6 +16,10 @@ USER_AGENT = f"{CensysCerts.DEFAULT_USER_AGENT} (censys-subdomain-finder; +https://github.com/christophetd/censys-subdomain-finder)" +MAX_PER_PAGE = 100 +COMMUNITY_MAX_PAGES = 10 +MAX_PAGES = 50 + # Finds subdomains of a domain using Censys API def find_subdomains(domain, api_id, api_secret, limit_results): @@ -26,11 +30,15 @@ def find_subdomains(domain, api_id, api_secret, limit_results): certificate_query = "names: %s" % domain if limit_results: certificates_search_results = censys_certificates.search( - certificate_query, fields=["names"], per_page=100, pages=10 + certificate_query, + per_page=MAX_PER_PAGE, + pages=COMMUNITY_MAX_PAGES, ) else: certificates_search_results = censys_certificates.search( - certificate_query, fields=["names"] + certificate_query, + per_page=MAX_PER_PAGE, + pages=MAX_PAGES ) # Flatten the result, and remove duplicates @@ -133,7 +141,7 @@ def main(domain, output_file, censys_api_id, censys_api_secret, limit_results): limit_results = not args.commercial if limit_results: - print("[*] Applying non-commercial limits (1000 results at most)") + print(f"[*] Applying free plan limits ({MAX_PER_PAGE * COMMUNITY_MAX_PAGES} results at most)") if None in [censys_api_id, censys_api_secret]: sys.stderr.write( diff --git a/requirements.txt b/requirements.txt index 7ce0281..47ed02f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -censys==2.2.0 +censys==2.2.1 python-dotenv From 9e37bf6acbc6d956d817e362662705d10e3fa2c8 Mon Sep 17 00:00:00 2001 From: Aidan Holland Date: Tue, 9 May 2023 10:18:21 -0400 Subject: [PATCH 3/3] chore: for commercial plans get all results --- censys-subdomain-finder.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/censys-subdomain-finder.py b/censys-subdomain-finder.py index cbad959..44b0965 100644 --- a/censys-subdomain-finder.py +++ b/censys-subdomain-finder.py @@ -17,8 +17,7 @@ USER_AGENT = f"{CensysCerts.DEFAULT_USER_AGENT} (censys-subdomain-finder; +https://github.com/christophetd/censys-subdomain-finder)" MAX_PER_PAGE = 100 -COMMUNITY_MAX_PAGES = 10 -MAX_PAGES = 50 +COMMUNITY_PAGES = 10 # Finds subdomains of a domain using Censys API @@ -28,18 +27,14 @@ def find_subdomains(domain, api_id, api_secret, limit_results): api_id=api_id, api_secret=api_secret, user_agent=USER_AGENT ) certificate_query = "names: %s" % domain + pages = -1 # unlimited if limit_results: - certificates_search_results = censys_certificates.search( - certificate_query, - per_page=MAX_PER_PAGE, - pages=COMMUNITY_MAX_PAGES, - ) - else: - certificates_search_results = censys_certificates.search( - certificate_query, - per_page=MAX_PER_PAGE, - pages=MAX_PAGES - ) + pages = COMMUNITY_PAGES + certificates_search_results = censys_certificates.search( + certificate_query, + per_page=MAX_PER_PAGE, + pages=pages + ) # Flatten the result, and remove duplicates subdomains = [] @@ -141,7 +136,11 @@ def main(domain, output_file, censys_api_id, censys_api_secret, limit_results): limit_results = not args.commercial if limit_results: - print(f"[*] Applying free plan limits ({MAX_PER_PAGE * COMMUNITY_MAX_PAGES} results at most)") + print( + f"[*] Applying free plan limits ({MAX_PER_PAGE * COMMUNITY_PAGES} results at most)" + ) + else: + print("[*] No limits applied, getting all results") if None in [censys_api_id, censys_api_secret]: sys.stderr.write(