diff --git a/README.md b/README.md index 3d13088..0ade92b 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ If you have problems with starting installer.sh, you should try to use `dos2unix - [x] Rework Google Dorking module in separate mode - [x] Rework Google Dorks list into separate databases with different pre-configured dorks for various purposes - [x] Allow user to create their own dorks DB -- [ ] Add separate API search mode with different free APIs +- [x] Add separate API search mode with different free APIs # DPULSE mentions in social medias diff --git a/datagather_modules/data_assembler.py b/datagather_modules/data_assembler.py index ccfd49f..a8cd5fa 100644 --- a/datagather_modules/data_assembler.py +++ b/datagather_modules/data_assembler.py @@ -147,27 +147,43 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k api_scan_db = [] if used_api_flag != ['Empty']: print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: API SCANNING]\n" + Style.RESET_ALL) - if 1 in used_api_flag: + if '1' in used_api_flag: vt_cats, vt_deturls, vt_detsamples, vt_undetsamples = api_virustotal_check(short_domain) api_scan_db.append('VirusTotal') - if 2 in used_api_flag: + if '2' in used_api_flag: st_alexa, st_apex, st_hostname, st_alivesds, st_txt, a_records_list, mx_records_list, ns_records_list, soa_records_list = api_securitytrails_check(short_domain) api_scan_db.append('SecurityTrails') - print(Fore.LIGHTMAGENTA_EX + f"[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL) + if '1' not in used_api_flag: + vt_cats = vt_deturls = vt_detsamples = vt_undetsamples = 'No results because user did not selected VirusTotal API scan' + if '2' not in used_api_flag: + st_alexa = st_apex = st_hostname = st_alivesds = st_txt = a_records_list = mx_records_list = ns_records_list = soa_records_list = 'No results because user did not selected SecurityTrails API scan' + print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL) else: vt_cats = vt_deturls = vt_detsamples = vt_undetsamples = 'No results because user did not selected VirusTotal API scan' st_alexa = st_apex = st_hostname = st_alivesds = st_txt = a_records_list = mx_records_list = ns_records_list = soa_records_list = 'No results because user did not selected SecurityTrails API scan' api_scan_db.append('No') pass + #data_array = [ip, res, mails, subdomains, subdomains_amount, social_medias, subdomain_mails, sd_socials, + # subdomain_ip, issuer, subject, notBefore, notAfter, commonName, serialNumber, mx_records, + # robots_txt_result, sitemap_xml_result, sitemap_links_status, + # web_servers, cms, programming_languages, web_frameworks, analytics, javascript_frameworks, ports, + # hostnames, cpes, tags, vulns, common_socials, total_socials, ps_emails_return, + # accessible_subdomains, emails_amount, files_counter, cookies_counter, api_keys_counter, + # website_elements_counter, exposed_passwords_counter, total_links_counter, accessed_links_counter, dorking_status, dorking_results, vt_cats, + # vt_deturls, vt_detsamples, vt_undetsamples] + + cleaned_dorking = [item.strip() for item in dorking_results if item.strip()] + #print(cleaned_dorking) + data_array = [ip, res, mails, subdomains, subdomains_amount, social_medias, subdomain_mails, sd_socials, subdomain_ip, issuer, subject, notBefore, notAfter, commonName, serialNumber, mx_records, robots_txt_result, sitemap_xml_result, sitemap_links_status, web_servers, cms, programming_languages, web_frameworks, analytics, javascript_frameworks, ports, hostnames, cpes, tags, vulns, common_socials, total_socials, ps_emails_return, accessible_subdomains, emails_amount, files_counter, cookies_counter, api_keys_counter, - website_elements_counter, exposed_passwords_counter, total_links_counter, accessed_links_counter, dorking_status, dorking_results, vt_cats, - vt_deturls, vt_detsamples, vt_undetsamples] + website_elements_counter, exposed_passwords_counter, total_links_counter, accessed_links_counter, cleaned_dorking, + vt_cats, vt_deturls, vt_detsamples, vt_undetsamples, st_alexa, st_apex, st_hostname, st_alivesds, st_txt, a_records_list, mx_records_list, ns_records_list, soa_records_list] elif report_file_type == 'html': if pagesearch_flag.lower() == 'y': @@ -207,15 +223,15 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k api_scan_db = [] if used_api_flag != ['Empty']: print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: API SCANNING]\n" + Style.RESET_ALL) - if 1 in used_api_flag: + if '1' in used_api_flag: vt_cats, vt_deturls, vt_detsamples, vt_undetsamples = api_virustotal_check(short_domain) api_scan_db.append('VirusTotal') - if 2 in used_api_flag: + if '2' in used_api_flag: st_alexa, st_apex, st_hostname, st_alivesds, st_txt, a_records_list, mx_records_list, ns_records_list, soa_records_list = api_securitytrails_check(short_domain) api_scan_db.append('SecurityTrails') - if 1 not in used_api_flag: + if '1' not in used_api_flag: vt_cats = vt_deturls = vt_detsamples = vt_undetsamples = 'No results because user did not selected VirusTotal API scan' - if 2 not in used_api_flag: + if '2' not in used_api_flag: st_alexa = st_apex = st_hostname = st_alivesds = st_txt = a_records_list = mx_records_list = ns_records_list = soa_records_list = 'No results because user did not selected SecurityTrails API scan' print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL) else: diff --git a/datagather_modules/networking_processor.py b/datagather_modules/networking_processor.py index 3a886d0..1d770a7 100644 --- a/datagather_modules/networking_processor.py +++ b/datagather_modules/networking_processor.py @@ -54,7 +54,7 @@ def get_ssl_certificate(short_domain, port=443): except (ssl.CertificateError, ssl.SSLError, socket.gaierror, ConnectionRefusedError) as e: print(Fore.RED + "Error while gathering info about SSL certificate. See journal for details") logging.error(f'SSL CERTIFICATE GATHERING: ERROR. REASON: {e}') - issuer = subject = notBefore = notAfter = commonName = serialNumber = ["No information about SSL certificate was gathered"] + issuer = subject = notBefore = notAfter = commonName = serialNumber = "No information about SSL certificate was gathered" return issuer, subject, notBefore, notAfter, commonName, serialNumber def query_internetdb(ip, report_file_extension): diff --git a/dpulse.py b/dpulse.py index 0a16ba9..48c02ea 100644 --- a/dpulse.py +++ b/dpulse.py @@ -109,7 +109,6 @@ def run(): print(Fore.RED + "Entered domain is not accessible. Scan is impossible" + Style.RESET_ALL) break case_comment = input(Fore.YELLOW + "Enter case comment >> ") - print(Fore.GREEN + "Be advised that PDF report support is deprecated since v1.1.4. Instead, try to work with HTML report :)" + Style.RESET_ALL) report_filetype = input(Fore.YELLOW + "Enter report file extension [XLSX/HTML] >> ") if not report_filetype: print(Fore.RED + "\nReport filetype cannot be empty") diff --git a/poetry.lock b/poetry.lock index 8f6def7..4b233f6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -33,13 +33,13 @@ six = "*" [[package]] name = "certifi" -version = "2024.8.30" +version = "2024.12.14" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, - {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, + {file = "certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56"}, + {file = "certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"}, ] [[package]] @@ -196,13 +196,13 @@ files = [ [[package]] name = "jinja2" -version = "3.1.4" +version = "3.1.5" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, - {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, + {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, + {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, ] [package.dependencies] @@ -662,13 +662,13 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "six" -version = "1.16.0" +version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] [[package]] @@ -702,4 +702,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "6b5d2a1f3f80e9d4da489516fdda4d8ae185fd01b7e936f615b11ae6232fc722" +content-hash = "09640a069a07ce9bc68b1ac8dfa92d844663571d3a80e9829da7292479bd9d34" diff --git a/pyproject.toml b/pyproject.toml index 62d5301..b7656fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "dpulse" -version = "1.1.5" +version = "1.1.6" description = "Convenient,fast and user-friendly collector of domain information from Open-Sources" authors = ["OSINT-TECHNOLOGIES "] readme = "README.md" @@ -22,7 +22,7 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.10" -Jinja2 = "3.1.4" +Jinja2 = "^3.1.5" MarkupSafe = "2.1.5" beautifulsoup4 = "4.12.2" certifi = ">=2024.07.04" diff --git a/report_examples/html_report_example/01-robots.txt b/report_examples/html_report_example/01-robots.txt new file mode 100644 index 0000000..3738559 --- /dev/null +++ b/report_examples/html_report_example/01-robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Disallow: /missions/ +Disallow: /killing/all/humans/ diff --git a/report_examples/html_report_example/hackthissiteorg_(28-12-2024, 09h17m49s).html b/report_examples/html_report_example/hackthissiteorg_(28-12-2024, 09h17m49s).html new file mode 100644 index 0000000..b6524ec --- /dev/null +++ b/report_examples/html_report_example/hackthissiteorg_(28-12-2024, 09h17m49s).html @@ -0,0 +1,356 @@ + + + + + + +

Open Source Information Research Report

+

Data Protected

+

 

+
+
 
+

Table of contents

+

1. General scan information

+

[BASIC SCAN INFO]

+

2. WHOIS information

+

3. Social medias links

+

4. Subdomains information

+

5. DNS & SSL information

+

6. Services & frameworks information

+

7. Basic pre-pentest information

+

[DORKING SCAN INFO]

+

8. Dorking queries and results links (if was selected)

+

[PAGESEARCH SCAN INFO]

+

9. PageSearch results (if was selected)

+

10. PageSearch Sitemap Inspection results (if was selected)

+

[API SCAN INFO]

+

11. VirusTotal API scan results (if was selected)

+

12. SecurityTrails API scan results (if was selected)

+

 

+
+
 
+

GENERAL SCAN INFORMATION

+

Total subdomains found: 7

+

Total social media links found: 13

+

Status of robots.txt extraction: File "robots.txt" was extracted to text file in report folder

+

Status of sitemap.xml extraction: File "sitemap.xml" was not found

+

Status of sitemap.xml links extraction: Links from "sitemap.txt" were not parsed

+

Google Dorking status: Successfully dorked domain with WEB_DORKS dorks table

+

PageSearch conduction: Yes, without keywords search

+

Report creation time: 28-12-2024, 09:17:49

+

 

+
+
 
+

WHOIS INFORMATION

+

Domain: hackthissite.org

+

Full URL: http://hackthissite.org/

+

IP address: 137.74.187.104

+

Registrar: eNom, LLC

+

Creation date: 2003-08-10 15:01:25

+

Expiration date: 2025-08-10 15:01:25

+

Organization name: Data Protected

+

Contact e-mails: No contact e-mails were found

+

 

+
+
 
+

SOCIAL MEDIAS SEARCH RESULTS

+

FACEBOOK:

+ +

TWITTER (+ X.com):

+ + +

INSTAGRAM:

+ +

TELEGRAM:

+ +

TIKTOK:

+ +

LINKEDIN:

+ +

VKONTAKTE:

+ +

YOUTUBE:

+ +

ODNOKLASSNIKI:

+ +

WECHAT:

+ +

 

+
+
 
+

SUBDOMAINS ANALYSIS RESULTS

+

Found subdomains:

+ +

Subdomains IP addresses:

+ +

Subdomains e-mails:

+ +

 

+
+
 
+

DNS & SSL INFORMATION

+

(DNS) Name servers: c.ns.buddyns.com, f.ns.buddyns.com, g.ns.buddyns.com, h.ns.buddyns.com, j.ns.buddyns.com

+

(DNS) MX addresses: None

+

(SSL) Issuer: No information about SSL certificate was gathered

+

(SSL) Subject: No information about SSL certificate was gathered

+

(SSL) Not before: No information about SSL certificate was gathered

+

(SSL) Not after: No information about SSL certificate was gathered

+

(SSL) Certificate name: No information about SSL certificate was gathered

+

(SSL) Certificate serial number: No information about SSL certificate was gathered

+

 

+
+
 
+

SERVICES & FRAMEWORKS INFORMATION

+

Web servers:

+ +

CMS:

+ +

Used programming languages:

+ +

Used web frameworks:

+ +

Analytics service:

+ +

Used JavaScript frameworks:

+ +

Tags:

+ +

Common Platform Enumeration:

+ +

 

+
+
 
+

BASIC PRE-PENTEST INFORMATION

+

Open ports:

+ +

Hostnames:

+ +

Potential vulnerabilities:

+ +

 

+
+
 
+

DORKING SCAN INFO

+ 
QUERY #1: site:hackthissite.org intext:"index of"
+=> NO RESULT FOUND
+
+QUERY #2: site:hackthissite.org inurl:admin
+=> NO RESULT FOUND
+
+QUERY #3: site:hackthissite.org inurl:login
+=> NO RESULT FOUND
+
+QUERY #4: site:hackthissite.org inurl:dashboard
+=> NO RESULT FOUND
+
+QUERY #5: site:hackthissite.org inurl:wp-content
+=> NO RESULT FOUND
+
+QUERY #6: site:hackthissite.org inurl:backup
+=> NO RESULT FOUND
+
+QUERY #7: site:hackthissite.org inurl:old
+=> NO RESULT FOUND
+
+QUERY #8: site:hackthissite.org inurl:temp
+=> NO RESULT FOUND
+
+QUERY #9: site:hackthissite.org inurl:upload
+=> NO RESULT FOUND
+
+QUERY #10: site:hackthissite.org inurl:download
+=> NO RESULT FOUND
+
+QUERY #11: site:hackthissite.org inurl:config
+=> NO RESULT FOUND
+
+QUERY #12: site:hackthissite.org inurl:setup
+=> NO RESULT FOUND
+
+QUERY #13: site:hackthissite.org inurl:install
+=> NO RESULT FOUND
+
+QUERY #14: site:hackthissite.org inurl:database
+=> NO RESULT FOUND
+
+QUERY #15: site:hackthissite.org inurl:log
+=> NO RESULT FOUND
+
+QUERY #16: site:hackthissite.org inurl:debug
+=> NO RESULT FOUND
+
+QUERY #17: site:hackthissite.org inurl:api
+=> NO RESULT FOUND
+
+QUERY #18: site:hackthissite.org inurl:secret
+=> NO RESULT FOUND
+
+QUERY #19: site:hackthissite.org inurl:private
+=> NO RESULT FOUND
+
+QUERY #20: site:hackthissite.org inurl:secure
+=> NO RESULT FOUND
+
+QUERY #21: site:hackthissite.org inurl:password
+=> NO RESULT FOUND
+
+QUERY #22: site:hackthissite.org inurl:auth
+=> NO RESULT FOUND
+
+QUERY #23: site:hackthissite.org inurl:token
+=> NO RESULT FOUND
+
+QUERY #24: site:hackthissite.org inurl:session
+=> NO RESULT FOUND
+
+QUERY #25: site:hackthissite.org inurl:panel
+=> NO RESULT FOUND
+
+
+

 

+
+
 
+

PAGESEARCH RESULTS

+

 

+

Amount of accessible subdomains: 4

+

Amount of email addresses: 0

+

Amount of found documents: 1

+

Amount of found cookies: 4

+

Amount of found API key: 0

+

Amount of WEB elements found: 0

+

Amount of exposed passwords found: 0

+

 

+
+
 
+

PAGESEARCH SITEMAP INSPECTION RESULTS

+

 

+

Total links amount: No results because PageSearch does not gather these categories

+

Amount of accessed links: No results because PageSearch does not gather these categories

+

 

+
+

 

+

VIRUSTOTAL API SCAN RESULTS

+

 

+

Categories: No results because user did not selected VirusTotal API scan

+

Detected URLs: No results because user did not selected VirusTotal API scan

+

Detected samples: No results because user did not selected VirusTotal API scan

+

Undetected samples: No results because user did not selected VirusTotal API scan

+

 

+
+

 

+

SECURITY TRAILS API SCAN RESULTS

+

 

+

Alexa rank: No results because user did not selected SecurityTrails API scan

+

Apex domain: No results because user did not selected SecurityTrails API scan

+

Hostname: No results because user did not selected SecurityTrails API scan

+

A records:

+ +

MX records:

+ +

NS records:

+ +

SOA records:

+ +

TXT records values:

+ +

Subdomains list

+ +

 

+
+

 

+

Created using DPULSE software by OSINT-TECHNOLOGIES

+

Visit our web-pages:

+ diff --git a/report_examples/html_report_example/ps_documents/extracted_About%20the%20HackThisSite%20Mirror.txt b/report_examples/html_report_example/ps_documents/extracted_About%20the%20HackThisSite%20Mirror.txt new file mode 100644 index 0000000..cea00f1 --- /dev/null +++ b/report_examples/html_report_example/ps_documents/extracted_About%20the%20HackThisSite%20Mirror.txt @@ -0,0 +1,28 @@ +-=- What is the HackThisSite Mirror? -=- + +HackThisSite may from time to time mirror things we think are deserving of our +bandwidth and maintenance, simply because of a belief in the cause or a general +liking of the data and its producers. + + +-=- Will you mirror my data? -=- + +You are more than welcome to request mirroring by emailing us at +admin - at - hackthissite - dot - org, but we will NOT guarantee anything! +Our bandwidth and time is precious, and just because you think your data is +well-deserving of our attention, does not mean we will think the same. + + +-=- What do you generally mirror? -=- + +HackThisSite will mirror projects we are involved with, or have a vested +interest in. For example, we are affiliated with Hackbloc, which produces +the HackThisZine periodical. Due to this, we mirror their publications. +Other such examples could someday include FreeBSD, nginx, Asterisk, UnrealIRC, +among others, since these are all projects whose benefits we reap. + + +-=- What are your mirror specifications? -=- + +mirror.hackthissite.org is hosted on a dedicated server in Europe with +redundant drives and a 1 Gbps connection. diff --git a/report_examples/xlsx_report_example/01-robots.txt b/report_examples/xlsx_report_example/01-robots.txt new file mode 100644 index 0000000..3738559 --- /dev/null +++ b/report_examples/xlsx_report_example/01-robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Disallow: /missions/ +Disallow: /killing/all/humans/ diff --git a/report_examples/xlsx_report_example/hackthissiteorg_(28-12-2024, 09h33m02s).csv b/report_examples/xlsx_report_example/hackthissiteorg_(28-12-2024, 09h33m02s).csv new file mode 100644 index 0000000..987fe78 Binary files /dev/null and b/report_examples/xlsx_report_example/hackthissiteorg_(28-12-2024, 09h33m02s).csv differ diff --git a/report_examples/xlsx_report_example/ps_documents/extracted_About%20the%20HackThisSite%20Mirror.txt b/report_examples/xlsx_report_example/ps_documents/extracted_About%20the%20HackThisSite%20Mirror.txt new file mode 100644 index 0000000..cea00f1 --- /dev/null +++ b/report_examples/xlsx_report_example/ps_documents/extracted_About%20the%20HackThisSite%20Mirror.txt @@ -0,0 +1,28 @@ +-=- What is the HackThisSite Mirror? -=- + +HackThisSite may from time to time mirror things we think are deserving of our +bandwidth and maintenance, simply because of a belief in the cause or a general +liking of the data and its producers. + + +-=- Will you mirror my data? -=- + +You are more than welcome to request mirroring by emailing us at +admin - at - hackthissite - dot - org, but we will NOT guarantee anything! +Our bandwidth and time is precious, and just because you think your data is +well-deserving of our attention, does not mean we will think the same. + + +-=- What do you generally mirror? -=- + +HackThisSite will mirror projects we are involved with, or have a vested +interest in. For example, we are affiliated with Hackbloc, which produces +the HackThisZine periodical. Due to this, we mirror their publications. +Other such examples could someday include FreeBSD, nginx, Asterisk, UnrealIRC, +among others, since these are all projects whose benefits we reap. + + +-=- What are your mirror specifications? -=- + +mirror.hackthissite.org is hosted on a dedicated server in Europe with +redundant drives and a 1 Gbps connection. diff --git a/reporting_modules/html_report_creation.py b/reporting_modules/html_report_creation.py index 82bfd71..3eb01ee 100644 --- a/reporting_modules/html_report_creation.py +++ b/reporting_modules/html_report_creation.py @@ -92,7 +92,7 @@ def report_assembling(short_domain, url, case_comment, data_array, report_info_a api_scan_db = report_info_array[7] used_api_flag = report_info_array[8] - if 2 in used_api_flag: + if '2' in used_api_flag: st_a_combined = [] if len(a_records_list) > 0: if len(a_records_list) == 1: diff --git a/reporting_modules/xlsx_report_creation.py b/reporting_modules/xlsx_report_creation.py index 1e8895d..334e56c 100644 --- a/reporting_modules/xlsx_report_creation.py +++ b/reporting_modules/xlsx_report_creation.py @@ -13,6 +13,7 @@ from openpyxl.styles import Font from colorama import Fore, Style import sqlite3 + from urllib.parse import unquote except ImportError as e: print(Fore.RED + "Import error appeared. Reason: {}".format(e) + Style.RESET_ALL) sys.exit() @@ -59,16 +60,65 @@ def create_report(short_domain, url, case_comment, data_array, report_info_array exposed_passwords_counter = data_array[39] total_links_counter = data_array[40] accessed_links_counter = data_array[41] - #keywords_messages_list = data_array[42] - dorking_results = data_array[43] + cleaned_dorking = data_array[42] + vt_cats = data_array[43] + vt_deturls = data_array[44] + vt_detsamples = data_array[45] + vt_undetsamples = data_array[46] + st_alexa = data_array[47] + st_apex = data_array[48] + st_hostname = data_array[49] + st_alivesds = data_array[50] + st_txt = data_array[51] + a_records_list = data_array[52] + mx_records_list = data_array[53] + ns_records_list = data_array[54] + soa_records_list = data_array[55] + casename = report_info_array[0] db_casename = report_info_array[1] db_creation_date = report_info_array[2] report_folder = report_info_array[3] report_ctime = report_info_array[6] api_scan_db = report_info_array[7] + used_api_flag = report_info_array[8] os.makedirs(report_folder, exist_ok=True) + if '2' in used_api_flag: + st_a_combined = [] + if len(a_records_list) > 0: + if len(a_records_list) == 1: + record = a_records_list[0] + st_a_combined = [f"IPv4 address: {record.get('ip', '')}, owned by {record.get('organization', '')}"] + else: + st_a_combined = [f"IPv4 address: {record.get('ip', '')}, owned by {record.get('organization', '')}" for record in a_records_list] + + st_mx_combined = [] + if len(mx_records_list) > 0: + if len(mx_records_list) == 1: + record = mx_records_list[0] + st_mx_combined = [f"Hostname {record.get('mx_hostname', '')} with priority={record.get('mx_priority', '')}, owned by {record.get('mx_organization', '')}"] + else: + st_mx_combined = [f"Hostname {record.get('mx_hostname', '')} with priority={record.get('mx_priority', '')}, owned by {record.get('mx_organization', '')}" for record in mx_records_list] + + st_ns_combined = [] + if len(ns_records_list) > 0: + if len(ns_records_list) == 1: + record = ns_records_list[0] + st_ns_combined = [f"Nameserver: {record.get('ns_nameserver', '')}, owned by {record.get('ns_organization', '')}"] + else: + st_ns_combined = [f"Nameserver: {record.get('ns_nameserver', '')}, owned by {record.get('ns_organization', '')}" for record in ns_records_list] + + st_soa_combined = [] + if len(soa_records_list) > 0: + if len(soa_records_list) == 1: + record = soa_records_list[0] + st_soa_combined = [f"Email: {record.get('soa_email', '')}, TTL={record.get('soa_ttl', '')}"] + else: + st_soa_combined = [f"Email: {record.get('soa_email', '')}, TTL={record.get('soa_ttl', '')}" for record in soa_records_list] + else: + st_soa_combined = st_ns_combined = st_mx_combined = st_a_combined = st_txt = st_alivesds = ['No results because user did not selected SecurityTrails API scan'] + if len(ps_emails_return) > 0: subdomain_mails += ps_emails_return subdomain_mails = list(set(subdomain_mails)) @@ -80,6 +130,16 @@ def create_report(short_domain, url, case_comment, data_array, report_info_array for email in subdomain_mails: new_emails = email.split(', ') subdomain_mails_cleaned.extend(new_emails) + else: + subdomain_mails = list(set(subdomain_mails)) + subdomain_mails_cleaned = [] + substrings = ['m=Base64', 'Ë','Á','Æ','Å','Ä','Ò','Á','ó','ð','É','ë','â'] + for substring in substrings: + if any(substring in s for s in subdomain_mails): + subdomain_mails.remove(next(s for s in subdomain_mails if substring in s)) + for email in subdomain_mails: + new_emails = email.split(', ') + subdomain_mails_cleaned.extend(new_emails) wb = openpyxl.Workbook() sheet_names = [ @@ -87,11 +147,14 @@ def create_report(short_domain, url, case_comment, data_array, report_info_array "WHOIS", "SOCIAL MEDIAS", "SUBDOMAINS", - "DNS SCAN", - "SSL CERTIFICATE", - "INTERNETDB SEARCH", - "WEBSITE TECHNOLOGIES", - "DORKING RESULTS" + "DNS & SSL", + "WEB INFO", + "PRE-PENTEST INFO", + "DORKING RESULTS", + "PAGESEARCH", + "PAGESEARCH (SI)", + "VIRUSTOTAL API", + "SECURITYTRAILS API" ] sheet = wb.active sheet.title = sheet_names[0] @@ -100,60 +163,28 @@ def create_report(short_domain, url, case_comment, data_array, report_info_array bold_font = Font(bold=True) ws = wb['GENERAL INFO'] - if pagesearch_keyword == 'n': - for col in ['1', '2', '3', '4', '5', '6', '7', '8', '9']: - cell = f"A{col}" - ws[cell].font = bold_font - elif pagesearch_keyword == 'y': - for col in ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16']: - cell = f"A{col}" - ws[cell].font = bold_font - elif pagesearch_keyword == 'si': - for col in ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11']: - cell = f"A{col}" - ws[cell].font = bold_font - ws.column_dimensions['A'].width = 45 + for col in ['1', '2', '3', '4', '5', '6', '7', '8', '9']: + cell = f"A{col}" + ws[cell].font = bold_font + ws.column_dimensions['A'].width = 60 ws.column_dimensions['B'].width = 60 - ws['A1'] = 'SUBDOMAINS FOUND' - ws['A2'] = 'SOCIAL MEDIAS FOUND' - ws['A3'] = 'ROBOTS EXTRACTED?' - ws['A4'] = 'SITEMAP.XML EXTRACTED?' - ws['A5'] = 'SITEMAP.XML LINKS EXTRACTED?' - ws['A6'] = 'DORKING STATUS' - ws['A7'] = 'PAGESEARCH STATUS' - ws['A8'] = 'REPORT CREATION TIME' - if pagesearch_keyword == 'y': - ws['A9'] = 'TOTAL SUBDOMAINS CHECKED' - ws['A10'] = 'AMOUNT OF ACCESSIBLE SUBDOMAINS' - ws['A11'] = 'AMOUNT OF ADDITIONAL EMAILS FOUND' - ws['A12'] = 'AMOUNT OF EXTRACTED FILES' - ws['A13'] = 'FOUND COOKIES WITH VALUES' - ws['A14'] = 'FOUND API VALUES' - ws['A15'] = 'FOUND DIFFERENT WEB PAGE ELEMENTS' - ws['A16'] = 'FOUND EXPOSED PASSWORDS' - ws['B9'] = subdomains_amount - ws['B10'] = accessible_subdomains - ws['B11'] = emails_amount - ws['B12'] = files_counter - ws['B13'] = cookies_counter - ws['B14'] = api_keys_counter - ws['B15'] = website_elements_counter - ws['B16'] = exposed_passwords_counter - elif pagesearch_keyword == 'si': - ws['A9'] = 'TOTAL LINKS CHECKED' - ws['A10'] = 'AMOUNT OF ACCESSIBLE LINKS' - ws['A11'] = 'AMOUNT OF ADDITIONAL EMAILS FOUND' - ws['B9'] = total_links_counter - ws['B10'] = accessed_links_counter - ws['B11'] = emails_amount - ws['B1'] = subdomains_amount - ws['B2'] = total_socials - ws['B3'] = robots_txt_result - ws['B4'] = sitemap_xml_result - ws['B5'] = sitemap_links_status - ws['B6'] = dorking_status - ws['B7'] = pagesearch_ui_mark - ws['B8'] = report_ctime + ws['A1'] = 'TARGET DOMAIN' + ws['A2'] = 'TOTAL SUBDOMAINS FOUND' + ws['A3'] = 'TOTAL SOCIAL MEDIAS LINKS FOUND' + ws['A4'] = 'STATUS OF ROBOTS.TXT EXTRACTION' + ws['A5'] = 'STATUS OF SITEMAP.XML EXTRACTION' + ws['A6'] = 'STATUS OF SITEMAP.XML LINKS EXTRACTION' + ws['A7'] = 'GOOGLE DORKING STATUS' + ws['A8'] = 'PAGESEARCH CONDUCTION' + ws['A9'] = 'REPORT CREATION TIME' + ws['B1'] = short_domain + ws['B2'] = subdomains_amount + ws['B3'] = total_socials + ws['B4'] = robots_txt_result + ws['B5'] = sitemap_xml_result + ws['B6'] = sitemap_links_status + ws['B8'] = pagesearch_ui_mark + ws['B9'] = report_ctime ws = wb['WHOIS'] for col in ['1', '2', '3', '4', '5', '6', '7', '8']: @@ -226,7 +257,7 @@ def create_report(short_domain, url, case_comment, data_array, report_info_array ws[f"J{i + 2}"] = wc_links[i] ws = wb['SUBDOMAINS'] - ws['A1'] = 'FOUNDED SUBDOMAINS' + ws['A1'] = 'FOUND SUBDOMAINS' ws['B1'] = 'SUBDOMAIN IP ADDRESSES (NOT CORRELATED)' ws['C1'] = 'SUBDOMAIN EMAILS (NOT CORRELATED)' for col in ['A', 'B', 'C']: @@ -245,78 +276,158 @@ def create_report(short_domain, url, case_comment, data_array, report_info_array logging.error(f'ERROR WHEN WRITING INFORMATION IN XLSX REPORT. REASON: {e}') pass - ws = wb['DNS SCAN'] - ws['A1'] = 'NAME SERVERS' - ws['A2'] = 'MX ADDRESSES' - for col in ['1', '2']: + ws = wb['DNS & SSL'] + for col in ['1', '2', '3', '4', '5', '6', '7', '8']: cell = f"A{col}" ws[cell].font = bold_font - ws.column_dimensions['A'].width = 45 + ws.column_dimensions['A'].width = 60 ws.column_dimensions['B'].width = 60 + ws['A1'] = '(DNS) NAME SERVERS' + ws['A2'] = '(DNS) MX ADDRESSES' + ws['A3'] = '(SSL) ISSUER' + ws['A4'] = '(SSL) SUBJECT' + ws['A5'] = '(SSL) NOT BEFORE' + ws['A6'] = '(SSL) NOT AFTER' + ws['A7'] = '(SSL) CERTIFICATE NAME' + ws['A8'] = '(SSL) CERTIFICATE SERIAL NUMBER' ws['B1'] = ', '.join(res['name_servers']) ws['B2'] = mx_records + ws['B3'] = issuer + ws['B4'] = subject + ws['B5'] = notBefore + ws['B6'] = notAfter + ws['B7'] = commonName + ws['B8'] = serialNumber - ws = wb['SSL CERTIFICATE'] - ws['A1'] = 'ISSUER' - ws['A2'] = 'SUBJECT' - ws['A3'] = 'NOT BEFORE' - ws['A4'] = 'NOT AFTER' - ws['A5'] = 'CERTIFICATE NAME' - ws['A6'] = 'CERTIFICATE SERIAL NUMBER' - for col in ['1', '2', '3', '4', '5', '6']: + ws = wb['WEB INFO'] + for col in ['1', '2', '3', '4', '5', '6', '7', '8']: cell = f"A{col}" ws[cell].font = bold_font - ws.column_dimensions['A'].width = 45 + ws.column_dimensions['A'].width = 35 ws.column_dimensions['B'].width = 60 - ws['B1'] = issuer - ws['B2'] = subject - ws['B3'] = notBefore - ws['B4'] = notAfter - ws['B5'] = commonName - ws['B6'] = serialNumber + ws['A1'] = 'WEB SERVERS' + ws['A2'] = 'CMS' + ws['A3'] = 'USED PROGRAMMING LANGUAGES' + ws['A4'] = 'USED WEB FRAMEWORKS' + ws['A5'] = 'ANALYTICS SERVICE' + ws['A6'] = 'USED JAVASCRIPT FRAMEWORKS' + ws['A7'] = 'TAGS' + ws['A8'] = 'CPEs' + ws['B1'] = ', '.join(web_servers) + ws['B2'] = ', '.join(cms) + ws['B3'] = ', '.join(programming_languages) + ws['B4'] = ', '.join(web_frameworks) + ws['B5'] = ', '.join(analytics) + ws['B6'] = ', '.join(javascript_frameworks) + ws['B7'] = ', '.join(str(item) for item in tags) + ws['B8'] = ', '.join(str(item) for item in cpes) - ws = wb['INTERNETDB SEARCH'] + ws = wb['PRE-PENTEST INFO'] ws['A1'] = 'OPEN PORTS' ws['A2'] = 'HOSTNAMES' - ws['A3'] = 'TAGS' - ws['A4'] = 'CPEs' - ws['I1'] = 'POTENTIAL VULNERABILITIES' - for col in ['1', '2', '3', '4']: + ws['F1'] = 'POTENTIAL VULNERABILITIES' + for col in ['1', '2']: cell = f"A{col}" ws[cell].font = bold_font - ws['I1'].font = bold_font + ws['F1'].font = bold_font ws.column_dimensions['A'].width = 45 ws.column_dimensions['B'].width = 60 ws['B1'] = ', '.join(str(item) for item in ports) ws['B2'] = ', '.join(str(item) for item in hostnames) - ws['B3'] = ', '.join(str(item) for item in tags) - ws['B4'] = ', '.join(str(item) for item in cpes) for i in range(len(vulns)): - ws[f"I{i + 2}"] = str(vulns[i]) + ws[f"F{i + 2}"] = str(vulns[i]) - ws = wb['WEBSITE TECHNOLOGIES'] - ws['A1'] = 'WEB SERVERS' - ws['A2'] = 'CMS' - ws['A3'] = 'USED PROGRAMMING LANGUAGES' - ws['A4'] = 'USED WEB FRAMEWORKS' - ws['A5'] = 'ANALYTICS SERVICE' - ws['A6'] = 'USED JAVASCRIPT FRAMEWORKS' - for col in ['1', '2', '3', '4', '5', '6']: + ws = wb['DORKING RESULTS'] + ws.column_dimensions['A'].width = 80 + for i, item in enumerate(cleaned_dorking, start=2): + ws[f"A{i}"] = str(item) + + ws = wb['PAGESEARCH'] + for col in ['1', '2', '3', '4', '5', '6', '7']: cell = f"A{col}" ws[cell].font = bold_font ws.column_dimensions['A'].width = 45 ws.column_dimensions['B'].width = 60 - ws['B1'] = ', '.join(web_servers) - ws['B2'] = ', '.join(cms) - ws['B3'] = ', '.join(programming_languages) - ws['B4'] = ', '.join(web_frameworks) - ws['B5'] = ', '.join(analytics) - ws['B6'] = ', '.join(javascript_frameworks) + ws['A1'] = 'ACCESSIBLE SUBDOMAINS' + ws['A2'] = 'ADDITIONAL EMAILS FOUND' + ws['A3'] = 'FOUND DOCUMENTS' + ws['A4'] = 'FOUND COOKIES' + ws['A5'] = 'FOUND API KEYS' + ws['A6'] = 'WEB ELEMENTS FOUND' + ws['A7'] = 'FOUND EXPOSED PASSWORDS' + ws['B1'] = accessible_subdomains + ws['B2'] = emails_amount + ws['B3'] = files_counter + ws['B4'] = cookies_counter + ws['B5'] = api_keys_counter + ws['B6'] = website_elements_counter + ws['B7'] = exposed_passwords_counter + + ws = wb['PAGESEARCH (SI)'] + for col in ['1', '2']: + cell = f"A{col}" + ws[cell].font = bold_font + ws.column_dimensions['A'].width = 45 + ws.column_dimensions['B'].width = 60 + ws['A1'] = 'TOTAL LINKS AMOUNT' + ws['A2'] = 'AMOUNT OF ACCESSED LINKS' + ws['B1'] = total_links_counter + ws['B2'] = accessed_links_counter + + ws = wb['VIRUSTOTAL API'] + for col in ['1', '2', '3', '4']: + cell = f"A{col}" + ws[cell].font = bold_font + ws.column_dimensions['A'].width = 45 + ws.column_dimensions['B'].width = 60 + ws['A1'] = 'CATEGORIES' + ws['A2'] = 'DETECTED URLS' + ws['A3'] = 'DETECTED SAMPLES' + ws['A4'] = 'UNDETECTED SAMPLES' + ws['B1'] = vt_cats + ws['B2'] = vt_deturls + ws['B3'] = vt_detsamples + ws['B4'] = vt_undetsamples + + ws = wb['SECURITYTRAILS API'] + for col in ['1', '2', '3']: + cell = f"A{col}" + ws[cell].font = bold_font + ws.column_dimensions['A'].width = 18 + ws.column_dimensions['B'].width = 60 + ws.column_dimensions['E'].width = 70 + ws.column_dimensions['F'].width = 70 + ws.column_dimensions['G'].width = 70 + ws.column_dimensions['H'].width = 70 + ws.column_dimensions['I'].width = 70 + ws.column_dimensions['J'].width = 70 + ws['A1'] = 'ALEXA RANK' + ws['A2'] = 'APEX DOMAIN' + ws['A3'] = 'HOSTNAME' + ws['E1'] = 'A RECORDS' + ws['F1'] = 'MX RECORDS' + ws['G1'] = 'NS RECORDS' + ws['H1'] = 'SOA RECORDS' + ws['I1'] = 'TXT RECORDS' + ws['J1'] = 'SUBDOMAINS LIST' + ws['E1'].font = ws['F1'].font = ws['G1'].font = ws['H1'].font = ws['I1'].font = ws['J1'].font = bold_font + ws['B1'] = st_alexa + ws['B2'] = st_apex + ws['B3'] = st_hostname + + for i in range(len(st_a_combined)): + ws[f"E{i + 2}"] = str(st_a_combined[i]) + for i in range(len(st_mx_combined)): + ws[f"F{i + 2}"] = str(st_mx_combined[i]) + for i in range(len(st_ns_combined)): + ws[f"G{i + 2}"] = str(st_ns_combined[i]) + for i in range(len(st_soa_combined)): + ws[f"H{i + 2}"] = str(st_soa_combined[i]) + for i in range(len(st_txt)): + ws[f"I{i + 2}"] = str(st_txt[i]) + for i in range(len(st_alivesds)): + ws[f"J{i + 2}"] = str(st_alivesds[i]) - ws = wb['DORKING RESULTS'] - ws.column_dimensions['A'].width = 80 - for i in range(len(dorking_results)): - ws[f"A{i + 1}"] = str(dorking_results[i]) report_file = report_folder + "//" + casename wb.save(report_file) diff --git a/requirements.txt b/requirements.txt index 8a6b94d..4d6f65e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Jinja2==3.1.4 +Jinja2>=3.1.5 MarkupSafe==2.1.5 beautifulsoup4==4.12.2 certifi>=2024.07.04 diff --git a/service/cli_init.py b/service/cli_init.py index e7ed8c2..ea1c436 100644 --- a/service/cli_init.py +++ b/service/cli_init.py @@ -20,7 +20,7 @@ def welcome_menu(self): fig = Figlet(font=wm_font) print('\n') self.console.print(fig.renderText('DPULSE'), style=preview_style) - print(Fore.MAGENTA + Style.BRIGHT + '[DPULSE-CLI] - [v1.1.5 stable] - [OSINT-TECHNOLOGIES]\n' + Style.RESET_ALL) + print(Fore.MAGENTA + Style.BRIGHT + '[DPULSE-CLI] - [v1.1.6 stable] - [OSINT-TECHNOLOGIES]\n' + Style.RESET_ALL) print(Fore.MAGENTA + Style.BRIGHT + 'Visit our pages:\nhttps://github.com/OSINT-TECHNOLOGIES\nhttps://pypi.org/project/dpulse/' + Style.RESET_ALL) def print_main_menu(self):