From 718ae066a6cb6edddf7bf8f2ba5e76dc17703b1e Mon Sep 17 00:00:00 2001 From: Jieyab89 Date: Tue, 16 Jun 2026 20:21:25 +0700 Subject: [PATCH] fix darkweb scrapper also add local transform before audit 2 June --- README.md | 5 +- .../onion-site-directory-scrapper.py | 276 +++++++++++------- 2 files changed, 179 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index efc5479..958c1be 100644 --- a/README.md +++ b/README.md @@ -3663,8 +3663,9 @@ Browser Sandbox # Darkweb Intelligence +- [DeepLink Onion Directory (ONION SITES)](http://deepxfmkjlils3vd7bru5rrxy3x3lchjgmv3wsgycjfynktmuwgm64qd.onion/) +- [Wiki Pages (ONION SITES)](http://uquroyobsaquslaunwkz6bmc3wutpzvwe7mv62xeq64645a57bugnsyd.onion/) - [ail-framework](https://github.com/ail-project/ail-framework) -- [breaking bad forums (ONION SITE)](http://bbzzzsvqcrqtki6umym6itiixfhni37ybtt7mkbjyxn2pgllzxf2qgyd.onion/) - [onion scan](https://onionscan.org/) - [onionite](https://github.com/lukechilds/onionite?tab=readme-ov-file) - [WhiteIntel](https://whiteintel.io/) @@ -3762,11 +3763,13 @@ Browser Sandbox - [Lowernet dir (ONION SITES)](http://loweruw2vupx3izp4hkmaotner6gjo4ooz6d4kdzqjbfkxqa2nlvjcyd.onion/) - [deepweeblink lists (ONION SITES)](http://darkwev6xtagl7742tqu24v2j4namr5ocfsfpha74a5nh4bwyp27a3ad.onion/) - [hidden links 2 (ONION SITES)](http://hiddenl3nnlkhu2lz3exwpuiflna2in5uxetcjlflaunva6ls6nyzyad.onion/) +- [grams (ONION SITES)](http://grams64rarzrk7rzdaz2fpb7lehcyi7zrrf5kd6w2uoamp7jw2aq6vyd.onion/) # Darkweb Forums Onion and surface site forums +- [breaking bad forums (ONION SITE)](http://bbzzzsvqcrqtki6umym6itiixfhni37ybtt7mkbjyxn2pgllzxf2qgyd.onion/) - [BHF FORUMS](http://bhf2b5nb3lb2kxpaoyqz7cuk2dkgej5n2refuffxzyhldwt4d7de4zqd.onion/) - [CRYPTBB](http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion) - [altenens surface web](https://altenens.is/threads/darkweb-2025-onion-links-new.2619050/) diff --git a/Script/Darkweb-Scrapping/onion-site-directory-scrapper.py b/Script/Darkweb-Scrapping/onion-site-directory-scrapper.py index ad788c3..7083abf 100644 --- a/Script/Darkweb-Scrapping/onion-site-directory-scrapper.py +++ b/Script/Darkweb-Scrapping/onion-site-directory-scrapper.py @@ -3,16 +3,16 @@ import time import json import requests import signal +import re from bs4 import BeautifulSoup from urllib.parse import urlparse from pathlib import Path -from datetime import datetime +from datetime import datetime, timezone BASE_DIR = Path(__file__).resolve().parent +OUTPUT_FILE = (BASE_DIR / "output.json").resolve() # U can custom your relative path! -OUTPUT_FILE = (BASE_DIR / "output.json").resolve() # Conf w your path! - -INTERVAL = 60 * 60 # 1 hour for scrapper and daemon +INTERVAL = 60 * 60 # U can custom the interval RUNNING = True # ========================= @@ -22,14 +22,14 @@ def shutdown(signum, frame): global RUNNING RUNNING = False -signal.signal(signal.SIGINT, shutdown) +# Only intercept SIGTERM. Let SIGINT raise KeyboardInterrupt for responsive Ctrl+C handling. signal.signal(signal.SIGTERM, shutdown) # ========================= # HELPERS # ========================= def now(): - return datetime.now(UTC).isoformat() + return datetime.now(timezone.utc).isoformat() # FIX 1: UTC → timezone.utc def onion_domain(url): try: @@ -61,111 +61,156 @@ def save_index(data): encoding="utf-8" ) -# ========================= -# TOR SESSION AND CONF -# ========================= +# ================================== +# TOR SESSION and CONFIG Global +# ================================== def tor_session(): for port in (9150, 9050): try: s = requests.Session() s.proxies = { - # Windows - # "http": f"socks5h://127.0.0.1:{port}", - # "https": f"socks5h://127.0.0.1:{port}", - # Linux - "http": f"socks4://127.0.0.1:{port}", - "https": f"socks4://127.0.0.1:{port}", + "http": f"socks5h://127.0.0.1:{port}", + "https": f"socks5h://127.0.0.1:{port}", } - s.headers["User-Agent"] = "Mozilla/5.0 (Tor; OnionDirectoryIndexer)" + s.headers["User-Agent"] = "Mozilla/5.0 (Tor; OnionDirectoryIndexer)" # u can custom the user agent here + print(f"[INFO] Trying Tor port {port}...") s.get("http://check.torproject.org", timeout=10) + print(f"[+] Tor connected via port {port}") return s - except: + except Exception as e: + print(f"[!] Port {port} failed: {e}") continue - raise RuntimeError("Tor not available") + raise RuntimeError( + "Tor is not available on port 9150 or 9050. " + "Linux OS: Settings in /etc/proxychains4.conf and /etc/tor/torrc\n" + "Windows OS: Start Tor Browser or Tor Expert Bundle." + ) + # raise RuntimeError("Tor is not available on port 9150 or 9050. Settings in /etc/proxychains4.con and /etc/tor/torrc\n" "Windows OS start Tor Browser or Tor Expert Bundle") # ========================= -# EXTRACTOR +# EXTRACTOR GLOBAL # ========================= def extract_onions(html, source): soup = BeautifulSoup(html, "html.parser") results = [] + seen = set() - for a in soup.find_all("a", href=True): - href = a["href"] - if ".onion" not in href: + # FORMAT 1: dark.fail + service_items = soup.find_all("div", class_="service-item") + print(f" [DEBUG] [{source}] Format1 found: {len(service_items)} service-item") + + for service in service_items: + title_tag = service.find("a", class_="service-name") + link_tag = service.find("a", class_="link") + + if not title_tag or not link_tag: continue - url = normalize_onion(href) - if not url: + raw_url = link_tag.get("href", "") + if ".onion" not in raw_url: continue + url = normalize_onion(raw_url) + if not url or url in seen: + continue + + seen.add(url) results.append({ - "title": a.get_text(strip=True)[:120], + "title": title_tag.get_text(strip=True)[:120], "link": url, "source": source }) + print(f" [DEBUG] [{source}] Format1 OK - {title_tag.get_text(strip=True)[:40]} => {url}") + # FORMAT 2: TorLinks style + all_anchors = soup.find_all("a", href=True) + print(f" [DEBUG] [{source}] Format2 total anchors: {len(all_anchors)}") + + for a in all_anchors: + href = a["href"] + title = a.get_text(strip=True) + + match = re.search(r'\((https?://[^)]+\.onion[^)]*)\)\s*$', href) + if match: + raw_url = match.group(1) + elif ".onion" in href: + raw_url = href + else: + continue + + url = normalize_onion(raw_url) + if not url or url in seen: + continue + + seen.add(url) + results.append({ + "title": title[:120], + "link": url, + "source": source + }) + print(f" [DEBUG] [{source}] Format2 OK - {title[:40]} => {url}") + + print(f" [DEBUG] [{source}] Total extracted: {len(results)} onions") return results -# ========================= -# SAFE DIRECTORY SOURCES -# ========================= + # FORMAT 3 ..... SOON + +# ================================================================== +# COLLECTORS JIEYAB LOGEN.INT DARKWEB TRANSOFRM AND DATA SOURCE +# ================================================================== COLLECTORS = { + # NOTES WILL UPDATE AS CAN WITH MY RESEARCH FOR DATA SOURCE "CHECK AT JIYEAB89 MAIN REPO FOR DARKWEB" + # Clearnet directories (WAJIB) + "dark.fail": "https://dark.fail/", + "ahmia": "https://ahmia.fi/", + "onionlinks": "https://onionlinks.com/", + "tordex-clearnet": "https://www.tordex.app/", - # ====================================== - # ADDED THE ONION LIST DIRECTORY SITE - # ====================================== - - # === CORE DARKWEB DIRECTORIES (WAJIB) === - "dark.fail": "https://dark.fail/", - "ahmia": "https://ahmia.fi/", - "onionlinks": "https://onionlinks.com/", - "tordex": "https://www.tordex.app/", - - # Onion / Tor - "torch66": "http://tor66sewebgixwhcqfnp5inzp5x5uohhdy3kvtnyfxc2e5mxiuh34iid.onion/", - "deepweblinks": "http://darkwev6xtagl7742tqu24v2j4namr5ocfsfpha74a5nh4bwyp27a3ad.onion/", + # Onion — searchengines & directories (onion link lists) + "torch66": "http://tor66sewebgixwhcqfnp5inzp5x5uohhdy3kvtnyfxc2e5mxiuh34iid.onion/", + "deepweblinks": "http://darkwev6xtagl7742tqu24v2j4namr5ocfsfpha74a5nh4bwyp27a3ad.onion/", "onionland-search": "http://3bbad7fauom4d6sgppalyqddsqbf5u5p56b5k5uk2zxsy3d6ey2jobad.onion/", - "torlinks": "http://torlinksge6enmcyyuxjpjkoouw4oorgdgeo7ftnq3zodj7g2zxi3kyd.onion/", - "deep-search": "http://search7tdrcvri22rieiwgi5g46qnwsesvnubqav2xakhezv4hjzkkad.onion/", - "excavator": "http://2fd6cemt4gmccflhm6imvdfvli3nf7zn6rfrwpsy7uhxrgbypvwf5fad.onion/", - "darksearch": "http://darkzqtmbdeauwq5mzcmgeeuhet42fhfjj4p5wbak3ofx2yqgecoeqyd.onion/", - "torwatch": "http://xq5hcm32m7ipdqt2ydqj6cc7lpj3lw3iwqnxiak2juynysoevjmancad.onion/", - "duckduckgo": "https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/", - "danex": "http://danexio627wiswvlpt6ejyhpxl5gla5nt2tgvgm2apj2ofrgm44vbeyd.onion/", - "grams": "http://grams64rarzrk7rzdaz2fpb7lehcyi7zrrf5kd6w2uoamp7jw2aq6vyd.onion/", - "torgle": "http://iy3544gmoeclh5de6gez2256v6pjh4omhpqdh2wpeeppjtvqmjhkfwad.onion/", - "tordex": "http://tordexu73joywapk2txdr54jed4imqledpcvcuf75qsas2gwdgksvnyd.onion/", - "freshonion": "http://freshonifyfe4rmuh6qwpsexfhdrww7wnt5qmkoertwxmcuvm4woo4ad.onion/", - "Logen.int": "http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/", - "Logen.int": "http://3fzh7yuupdfyjhwt3ugzqqof6ulbcl27ecev33knxe3u7goi3vfn2qqd.onion", - "Logen.int": "http://uquroyobsaquslaunwkz6bmc3wutpzvwe7mv62xeq64645a57bugnsyd.onion", - "Logen.int": "http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/cgi-bin/omega/omega", - "Hidden Wiki blog": "http://darkwebp7lyr44rpgqdtevalty2pk5oqmc6m2cnicnix7itelt3lp3id.onion/", - "Onionx": "http://onionxpwovutq7bfv54ees3sfhcj32euy4bd555vvc4aufemxhkzvoyd.onion", - "Darksearch": "http://darkzqtmbdeauwq5mzcmgeeuhet42fhfjj4p5wbak3ofx2yqgecoeqyd.onion", - "OnionFind": "http://ofinde3b67voi7xiq3qflof2mwriwngicd7glwvf3bclgdgcjfozlzqd.onion/", - "SearchXNG": "http://searx3aolosaf3urwnhpynlhuokqsgz47si4pzz5hvb7uuzyjncl2tid.onion/", - "4GET": "http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion", - "DeepLink": "http://deepxfmkjlils3vd7bru5rrxy3x3lchjgmv3wsgycjfynktmuwgm64qd.onion", - "breaking bad forums" : "http://bbzzzsvqcrqtki6umym6itiixfhni37ybtt7mkbjyxn2pgllzxf2qgyd.onion", + "torlinks": "http://torlinksge6enmcyyuxjpjkoouw4oorgdgeo7ftnq3zodj7g2zxi3kyd.onion/", + "deep-search": "http://search7tdrcvri22rieiwgi5g46qnwsesvnubqav2xakhezv4hjzkkad.onion/", + "excavator": "http://2fd6cemt4gmccflhm6imvdfvli3nf7zn6rfrwpsy7uhxrgbypvwf5fad.onion/", + "darksearch": "http://darkzqtmbdeauwq5mzcmgeeuhet42fhfjj4p5wbak3ofx2yqgecoeqyd.onion/", + "torwatch": "http://xq5hcm32m7ipdqt2ydqj6cc7lpj3lw3iwqnxiak2juynysoevjmancad.onion/", + "duckduckgo-onion": "https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/", + "danex": "http://danexio627wiswvlpt6ejyhpxl5gla5nt2tgvgm2apj2ofrgm44vbeyd.onion/", + "grams": "http://grams64rarzrk7rzdaz2fpb7lehcyi7zrrf5kd6w2uoamp7jw2aq6vyd.onion/", + "torgle": "http://iy3544gmoeclh5de6gez2256v6pjh4omhpqdh2wpeeppjtvqmjhkfwad.onion/", + "tordex-onion": "http://tordexu73joywapk2txdr54jed4imqledpcvcuf75qsas2gwdgksvnyd.onion/", + "freshonion": "http://freshonifyfe4rmuh6qwpsexfhdrww7wnt5qmkoertwxmcuvm4woo4ad.onion/", - # Clearnet (darkweb directory / tracker) + # FIX: LOGEN.INT NAME TRANSFORM LOGEN.INT DATA SOURCE (WAJIB) + "logen-1": "http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/", + "logen-2": "http://3fzh7yuupdfyjhwt3ugzqqof6ulbcl27ecev33knxe3u7goi3vfn2qqd.onion", + "logen-3": "http://uquroyobsaquslaunwkz6bmc3wutpzvwe7mv62xeq64645a57bugnsyd.onion", + "logen-4": "http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/cgi-bin/omega/omega", + + "hidden-wiki-blog": "http://darkwebp7lyr44rpgqdtevalty2pk5oqmc6m2cnicnix7itelt3lp3id.onion/", + "onionx": "http://onionxpwovutq7bfv54ees3sfhcj32euy4bd555vvc4aufemxhkzvoyd.onion", + "onionfind": "http://ofinde3b67voi7xiq3qflof2mwriwngicd7glwvf3bclgdgcjfozlzqd.onion/", + "searxng": "http://searx3aolosaf3urwnhpynlhuokqsgz47si4pzz5hvb7uuzyjncl2tid.onion/", + "4get": "http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion", + "deeplink": "http://deepxfmkjlils3vd7bru5rrxy3x3lchjgmv3wsgycjfynktmuwgm64qd.onion", + "breaking-bad-forums": "http://bbzzzsvqcrqtki6umym6itiixfhni37ybtt7mkbjyxn2pgllzxf2qgyd.onion", + + # Clearnet trackers (onion link lists) "darkwebdaily.net": "https://darkwebdaily.net/", - "daut.link": "https://daunt.link/", - "tor taxi": "https://tor.taxi/", - "darkwebdaily": "https://darkwebdaily.live/", - "immuniweb": "https://www.immuniweb.com/darkweb/", - "ransomwatch": "https://ransomwatch.telemetry.ltd/", - "watchguard": "https://www.watchguard.com/wgrd-security-hub/ransomware-tracker", - "onionlive": "https://onion.live/", - "torlink": "https://tor.link/", - "onionland": "https://onionland.io/", - "dargle": "https://www.dargle.net/domains", - "torry": "https://www.torry.io/", - "onionlandse": "https://onionlandsearchengine.net/", - "torsearch": "https://torsearch.com/", - "onionlinkhub": "https://www.onionlinkhub.com/", + "daunt.link": "https://daunt.link/", + "tor-taxi": "https://tor.taxi/", + "darkwebdaily": "https://darkwebdaily.live/", + "immuniweb": "https://www.immuniweb.com/darkweb/", + "ransomwatch": "https://ransomwatch.telemetry.ltd/", + "onionlive": "https://onion.live/", + "torlink": "https://tor.link/", + "onionland": "https://onionland.io/", + "dargle": "https://www.dargle.net/domains", + "torry": "https://www.torry.io/", + "onionlandse": "https://onionlandsearchengine.net/", + "torsearch": "https://torsearch.com/", + "onionlinkhub": "https://www.onionlinkhub.com/", } # ========================= @@ -173,23 +218,41 @@ COLLECTORS = { # ========================= def main(): print("[*] Onion directory daemon started") + index_map = {} - while RUNNING: - try: - session = tor_session() - index = load_index() + try: + while RUNNING: + try: + session = tor_session() + except RuntimeError as e: + print(f"[!] {e}") + print(f"[!] Please check your TOR network....................") + for _ in range(60): + if not RUNNING: + break + time.sleep(1) + continue + index = load_index() index_map = { onion_domain(i["link"]): i for i in index if onion_domain(i.get("link")) } - added = 0 for source, url in COLLECTORS.items(): + if not RUNNING: + break try: - r = session.get(url, timeout=45) + print(f"\n[INFO] Fetching [{source}] => {url}") + start = time.time() + + r = session.get(url, timeout=45) + elapsed = round(time.time() - start, 2) + + print(f"[+] Response [{source}] status={r.status_code} size={len(r.text)} chars time={elapsed}s") + for item in extract_onions(r.text, source): domain = onion_domain(item["link"]) if not domain: @@ -198,29 +261,40 @@ def main(): if domain in index_map: index_map[domain]["updated_date"] = now() else: - item["stored_date"] = now() - item["updated_date"] = item["stored_date"] - index_map[domain] = item + item["stored_date"] = now() + item["updated_date"] = item["stored_date"] + index_map[domain] = item added += 1 - except: - continue + + except requests.exceptions.Timeout: + print(f"[!] TIMEOUT [{source}]") + except requests.exceptions.ConnectionError as e: + print(f"[!] CONNECTION ERROR [{source}]: {e}") + except Exception as e: + print(f"[!] ERROR [{source}]: {type(e).__name__}: {e}") if added > 0: save_index(list(index_map.values())) - print(f"[+] Added {added} new domains (total {len(index_map)})") + print(f"\n[+] Finished: +{added} new domains (total {len(index_map)})") + added = 0 else: - print("[=] No new domains found") + print("\n[INFO] No new domains found") - except Exception as e: - print("[!] Error:", e) + print(f"[INFO] Sleeping {INTERVAL//60} minutes...") + for _ in range(INTERVAL): + if not RUNNING: + break + time.sleep(1) - # Sleep with graceful stop - for _ in range(INTERVAL): - if not RUNNING: - break - time.sleep(1) + except KeyboardInterrupt: + print("\n[!] Process interrupted by user (Ctrl+C). Saving progress...") + if index_map: + save_index(list(index_map.values())) + print(f"[+] Index saved to {OUTPUT_FILE} (total {len(index_map)} domains)") + else: + print("[-] Nothing to save.") - print("[*] Onion directory daemon stopped cleanly") + print("[*] Daemon stopped cleanly") if __name__ == "__main__": - main() + main() \ No newline at end of file