gitea-mirror/OSINT-Cheat-sheet
1
0
Fork 0
mirror of https://github.com/Jieyab89/OSINT-Cheat-sheet.git synced 2026-07-28 14:47:03 -07:00
Code Issues Packages Projects Releases Wiki Activity

update darkweb onion scrapper links logen int

Browse Source
This commit is contained in:
Jieyab89
2026-07-04 16:07:17 +07:00
parent be93375b94
commit 8dd2625413
1 changed files with 35 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Script/Darkweb-Scrapping/onion-site-directory-scrapper.py
+35 -6
View File
@@ -72,7 +72,7 @@ def tor_session():
"http": f"socks5h://127.0.0.1:{port}",
"https": f"socks5h://127.0.0.1:{port}",
}
s.headers["User-Agent"] = "Mozilla/5.0 (Tor; OnionDirectoryIndexer)" # u can custom the user agent here
s.headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36" # u can custom the user agent here
print(f"[INFO] Trying Tor port {port}...")
s.get("http://check.torproject.org", timeout=10)
print(f"[+] Tor connected via port {port}")
@@ -150,11 +150,26 @@ def extract_onions(html, source):
})
print(f" [DEBUG] [{source}] Format2 OK - {title[:40]} => {url}")
# FORMAT 3: Deepdark-CTI
md_matches = re.findall(r'\[([^\]]+)\]\((https?://[^\s\)]+\.onion[^\s\)]*)\)', html)
print(f" [DEBUG] [{source}] Format3 markdown matches: {len(md_matches)}")
for title, raw_url in md_matches:
url = normalize_onion(raw_url)
if not url or url in seen:
continue
seen.add(url)
results.append({
"title": title.strip()[:120],
"link": url,
"source": source
})
print(f" [DEBUG] [{source}] Format3 OK - {title.strip()[:40]} => {url}")
print(f" [DEBUG] [{source}] Total extracted: {len(results)} onions")
return results
# FORMAT 3 ..... SOON
# ==================================================================
# COLLECTORS JIEYAB LOGEN.INT DARKWEB TRANSOFRM AND DATA SOURCE
# ==================================================================
@@ -196,6 +211,16 @@ COLLECTORS = {
"deeplink": "http://deepxfmkjlils3vd7bru5rrxy3x3lchjgmv3wsgycjfynktmuwgm64qd.onion",
"breaking-bad-forums": "http://bbzzzsvqcrqtki6umym6itiixfhni37ybtt7mkbjyxn2pgllzxf2qgyd.onion",
# LOGEN.INT GITHUB DATA SOURCES
"deepdarkcti-others": "https://raw.githubusercontent.com/fastfire/deepdarkCTI/refs/heads/main/others.md",
"deepdarkcti-ransomware": "https://raw.githubusercontent.com/fastfire/deepdarkCTI/refs/heads/main/ransomware_gang.md",
"deepdarkcti-rat": "https://raw.githubusercontent.com/fastfire/deepdarkCTI/refs/heads/main/rat.md",
"deepdarkcti-search-engines": "https://raw.githubusercontent.com/fastfire/deepdarkCTI/refs/heads/main/search_engines.md",
"deepdarkcti-counterfeit": "https://raw.githubusercontent.com/fastfire/deepdarkCTI/refs/heads/main/counterfeit_goods.md",
"deepdarkcti-forum": "https://raw.githubusercontent.com/fastfire/deepdarkCTI/refs/heads/main/forum.md",
"deepdarkcti-maas": "https://raw.githubusercontent.com/fastfire/deepdarkCTI/refs/heads/main/maas.md",
"deepdarkcti-markets": "https://raw.githubusercontent.com/fastfire/deepdarkCTI/refs/heads/main/markets.md",
# Clearnet trackers (onion link lists)
"darkwebdaily.net": "https://darkwebdaily.net/",
"daunt.link": "https://daunt.link/",
@@ -259,6 +284,7 @@ def main():
continue
if domain in index_map:
# sudah ada, cuma update timestamp, gak perlu write ulang ke json
index_map[domain]["updated_date"] = now()
else:
item["stored_date"] = now()
@@ -266,6 +292,10 @@ def main():
index_map[domain] = item
added += 1
# LANGSUNG SAVE begitu ketemu domain baru & unik
save_index(list(index_map.values()))
print(f" [SAVE] +1 new unique domain saved => {domain} (total {len(index_map)})")
except requests.exceptions.Timeout:
print(f"[!] TIMEOUT [{source}]")
except requests.exceptions.ConnectionError as e:
@@ -274,11 +304,10 @@ def main():
print(f"[!] ERROR [{source}]: {type(e).__name__}: {e}")
if added > 0:
save_index(list(index_map.values()))
print(f"\n[+] Finished: +{added} new domains (total {len(index_map)})")
print(f"\n[+] Finished: +{added} new domains this cycle (total {len(index_map)})")
added = 0
else:
print("\n[INFO] No new domains found")
print("\n[INFO] No new domains found")
print(f"[INFO] Sleeping {INTERVAL//60} minutes...")
for _ in range(INTERVAL):