gitea-mirror/OSINT-Cheat-sheet
1
0
Fork 0
mirror of https://github.com/Jieyab89/OSINT-Cheat-sheet.git synced 2026-07-28 14:47:03 -07:00
Code Issues Packages Projects Releases Wiki Activity

fix darkweb scrapper also add local transform before audit 2 June

Browse Source
This commit is contained in:
Jieyab89
2026-06-16 20:21:25 +07:00
parent b9fcfe5932
commit 718ae066a6
2 changed files with 179 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+4 -1
View File
@@ -3663,8 +3663,9 @@ Browser Sandbox
# Darkweb Intelligence # Darkweb Intelligence
- [DeepLink Onion Directory (ONION SITES)](http://deepxfmkjlils3vd7bru5rrxy3x3lchjgmv3wsgycjfynktmuwgm64qd.onion/)
- [Wiki Pages (ONION SITES)](http://uquroyobsaquslaunwkz6bmc3wutpzvwe7mv62xeq64645a57bugnsyd.onion/)
- [ail-framework](https://github.com/ail-project/ail-framework) - [ail-framework](https://github.com/ail-project/ail-framework)
- [breaking bad forums (ONION SITE)](http://bbzzzsvqcrqtki6umym6itiixfhni37ybtt7mkbjyxn2pgllzxf2qgyd.onion/)
- [onion scan](https://onionscan.org/) - [onion scan](https://onionscan.org/)
- [onionite](https://github.com/lukechilds/onionite?tab=readme-ov-file) - [onionite](https://github.com/lukechilds/onionite?tab=readme-ov-file)
- [WhiteIntel](https://whiteintel.io/) - [WhiteIntel](https://whiteintel.io/)
@@ -3762,11 +3763,13 @@ Browser Sandbox
- [Lowernet dir (ONION SITES)](http://loweruw2vupx3izp4hkmaotner6gjo4ooz6d4kdzqjbfkxqa2nlvjcyd.onion/) - [Lowernet dir (ONION SITES)](http://loweruw2vupx3izp4hkmaotner6gjo4ooz6d4kdzqjbfkxqa2nlvjcyd.onion/)
- [deepweeblink lists (ONION SITES)](http://darkwev6xtagl7742tqu24v2j4namr5ocfsfpha74a5nh4bwyp27a3ad.onion/) - [deepweeblink lists (ONION SITES)](http://darkwev6xtagl7742tqu24v2j4namr5ocfsfpha74a5nh4bwyp27a3ad.onion/)
- [hidden links 2 (ONION SITES)](http://hiddenl3nnlkhu2lz3exwpuiflna2in5uxetcjlflaunva6ls6nyzyad.onion/) - [hidden links 2 (ONION SITES)](http://hiddenl3nnlkhu2lz3exwpuiflna2in5uxetcjlflaunva6ls6nyzyad.onion/)
- [grams (ONION SITES)](http://grams64rarzrk7rzdaz2fpb7lehcyi7zrrf5kd6w2uoamp7jw2aq6vyd.onion/)
# Darkweb Forums # Darkweb Forums
Onion and surface site forums Onion and surface site forums
- [breaking bad forums (ONION SITE)](http://bbzzzsvqcrqtki6umym6itiixfhni37ybtt7mkbjyxn2pgllzxf2qgyd.onion/)
- [BHF FORUMS](http://bhf2b5nb3lb2kxpaoyqz7cuk2dkgej5n2refuffxzyhldwt4d7de4zqd.onion/) - [BHF FORUMS](http://bhf2b5nb3lb2kxpaoyqz7cuk2dkgej5n2refuffxzyhldwt4d7de4zqd.onion/)
- [CRYPTBB](http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion) - [CRYPTBB](http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion)
- [altenens surface web](https://altenens.is/threads/darkweb-2025-onion-links-new.2619050/) - [altenens surface web](https://altenens.is/threads/darkweb-2025-onion-links-new.2619050/)
Script/Darkweb-Scrapping/onion-site-directory-scrapper.py
+174 -100
View File
@@ -3,16 +3,16 @@ import time
import json import json
import requests import requests
import signal import signal
import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from urllib.parse import urlparse from urllib.parse import urlparse
from pathlib import Path from pathlib import Path
from datetime import datetime from datetime import datetime, timezone
BASE_DIR = Path(__file__).resolve().parent BASE_DIR = Path(__file__).resolve().parent
OUTPUT_FILE = (BASE_DIR / "output.json").resolve() # U can custom your relative path!
OUTPUT_FILE = (BASE_DIR / "output.json").resolve() # Conf w your path! INTERVAL = 60 * 60 # U can custom the interval
INTERVAL = 60 * 60 # 1 hour for scrapper and daemon
RUNNING = True RUNNING = True
# ========================= # =========================
@@ -22,14 +22,14 @@ def shutdown(signum, frame):
global RUNNING global RUNNING
RUNNING = False RUNNING = False
signal.signal(signal.SIGINT, shutdown) # Only intercept SIGTERM. Let SIGINT raise KeyboardInterrupt for responsive Ctrl+C handling.
signal.signal(signal.SIGTERM, shutdown) signal.signal(signal.SIGTERM, shutdown)
# ========================= # =========================
# HELPERS # HELPERS
# ========================= # =========================
def now(): def now():
return datetime.now(UTC).isoformat() return datetime.now(timezone.utc).isoformat() # FIX 1: UTC → timezone.utc
def onion_domain(url): def onion_domain(url):
try: try:
@@ -61,111 +61,156 @@ def save_index(data):
encoding="utf-8" encoding="utf-8"
) )
# ========================= # ==================================
# TOR SESSION AND CONF # TOR SESSION and CONFIG Global
# ========================= # ==================================
def tor_session(): def tor_session():
for port in (9150, 9050): for port in (9150, 9050):
try: try:
s = requests.Session() s = requests.Session()
s.proxies = { s.proxies = {
# Windows "http": f"socks5h://127.0.0.1:{port}",
# "http": f"socks5h://127.0.0.1:{port}", "https": f"socks5h://127.0.0.1:{port}",
# "https": f"socks5h://127.0.0.1:{port}",
# Linux
"http": f"socks4://127.0.0.1:{port}",
"https": f"socks4://127.0.0.1:{port}",
} }
s.headers["User-Agent"] = "Mozilla/5.0 (Tor; OnionDirectoryIndexer)" s.headers["User-Agent"] = "Mozilla/5.0 (Tor; OnionDirectoryIndexer)" # u can custom the user agent here
print(f"[INFO] Trying Tor port {port}...")
s.get("http://check.torproject.org", timeout=10) s.get("http://check.torproject.org", timeout=10)
print(f"[+] Tor connected via port {port}")
return s return s
except: except Exception as e:
print(f"[!] Port {port} failed: {e}")
continue continue
raise RuntimeError("Tor not available") raise RuntimeError(
"Tor is not available on port 9150 or 9050. "
"Linux OS: Settings in /etc/proxychains4.conf and /etc/tor/torrc\n"
"Windows OS: Start Tor Browser or Tor Expert Bundle."
)
# raise RuntimeError("Tor is not available on port 9150 or 9050. Settings in /etc/proxychains4.con and /etc/tor/torrc\n" "Windows OS start Tor Browser or Tor Expert Bundle")
# ========================= # =========================
# EXTRACTOR # EXTRACTOR GLOBAL
# ========================= # =========================
def extract_onions(html, source): def extract_onions(html, source):
soup = BeautifulSoup(html, "html.parser") soup = BeautifulSoup(html, "html.parser")
results = [] results = []
seen = set()
for a in soup.find_all("a", href=True): # FORMAT 1: dark.fail
href = a["href"] service_items = soup.find_all("div", class_="service-item")
if ".onion" not in href: print(f" [DEBUG] [{source}] Format1 found: {len(service_items)} service-item")
for service in service_items:
title_tag = service.find("a", class_="service-name")
link_tag = service.find("a", class_="link")
if not title_tag or not link_tag:
continue continue
url = normalize_onion(href) raw_url = link_tag.get("href", "")
if not url: if ".onion" not in raw_url:
continue continue
url = normalize_onion(raw_url)
if not url or url in seen:
continue
seen.add(url)
results.append({ results.append({
"title": a.get_text(strip=True)[:120], "title": title_tag.get_text(strip=True)[:120],
"link": url, "link": url,
"source": source "source": source
}) })
print(f" [DEBUG] [{source}] Format1 OK - {title_tag.get_text(strip=True)[:40]} => {url}")
# FORMAT 2: TorLinks style
all_anchors = soup.find_all("a", href=True)
print(f" [DEBUG] [{source}] Format2 total anchors: {len(all_anchors)}")
for a in all_anchors:
href = a["href"]
title = a.get_text(strip=True)
match = re.search(r'\((https?://[^)]+\.onion[^)]*)\)\s*$', href)
if match:
raw_url = match.group(1)
elif ".onion" in href:
raw_url = href
else:
continue
url = normalize_onion(raw_url)
if not url or url in seen:
continue
seen.add(url)
results.append({
"title": title[:120],
"link": url,
"source": source
})
print(f" [DEBUG] [{source}] Format2 OK - {title[:40]} => {url}")
print(f" [DEBUG] [{source}] Total extracted: {len(results)} onions")
return results return results
# ========================= # FORMAT 3 ..... SOON
# SAFE DIRECTORY SOURCES
# ========================= # ==================================================================
# COLLECTORS JIEYAB LOGEN.INT DARKWEB TRANSOFRM AND DATA SOURCE
# ==================================================================
COLLECTORS = { COLLECTORS = {
# NOTES WILL UPDATE AS CAN WITH MY RESEARCH FOR DATA SOURCE "CHECK AT JIYEAB89 MAIN REPO FOR DARKWEB"
# Clearnet directories (WAJIB)
"dark.fail": "https://dark.fail/",
"ahmia": "https://ahmia.fi/",
"onionlinks": "https://onionlinks.com/",
"tordex-clearnet": "https://www.tordex.app/",
# ====================================== # Onion — searchengines & directories (onion link lists)
# ADDED THE ONION LIST DIRECTORY SITE "torch66": "http://tor66sewebgixwhcqfnp5inzp5x5uohhdy3kvtnyfxc2e5mxiuh34iid.onion/",
# ====================================== "deepweblinks": "http://darkwev6xtagl7742tqu24v2j4namr5ocfsfpha74a5nh4bwyp27a3ad.onion/",
# === CORE DARKWEB DIRECTORIES (WAJIB) ===
"dark.fail": "https://dark.fail/",
"ahmia": "https://ahmia.fi/",
"onionlinks": "https://onionlinks.com/",
"tordex": "https://www.tordex.app/",
# Onion / Tor
"torch66": "http://tor66sewebgixwhcqfnp5inzp5x5uohhdy3kvtnyfxc2e5mxiuh34iid.onion/",
"deepweblinks": "http://darkwev6xtagl7742tqu24v2j4namr5ocfsfpha74a5nh4bwyp27a3ad.onion/",
"onionland-search": "http://3bbad7fauom4d6sgppalyqddsqbf5u5p56b5k5uk2zxsy3d6ey2jobad.onion/", "onionland-search": "http://3bbad7fauom4d6sgppalyqddsqbf5u5p56b5k5uk2zxsy3d6ey2jobad.onion/",
"torlinks": "http://torlinksge6enmcyyuxjpjkoouw4oorgdgeo7ftnq3zodj7g2zxi3kyd.onion/", "torlinks": "http://torlinksge6enmcyyuxjpjkoouw4oorgdgeo7ftnq3zodj7g2zxi3kyd.onion/",
"deep-search": "http://search7tdrcvri22rieiwgi5g46qnwsesvnubqav2xakhezv4hjzkkad.onion/", "deep-search": "http://search7tdrcvri22rieiwgi5g46qnwsesvnubqav2xakhezv4hjzkkad.onion/",
"excavator": "http://2fd6cemt4gmccflhm6imvdfvli3nf7zn6rfrwpsy7uhxrgbypvwf5fad.onion/", "excavator": "http://2fd6cemt4gmccflhm6imvdfvli3nf7zn6rfrwpsy7uhxrgbypvwf5fad.onion/",
"darksearch": "http://darkzqtmbdeauwq5mzcmgeeuhet42fhfjj4p5wbak3ofx2yqgecoeqyd.onion/", "darksearch": "http://darkzqtmbdeauwq5mzcmgeeuhet42fhfjj4p5wbak3ofx2yqgecoeqyd.onion/",
"torwatch": "http://xq5hcm32m7ipdqt2ydqj6cc7lpj3lw3iwqnxiak2juynysoevjmancad.onion/", "torwatch": "http://xq5hcm32m7ipdqt2ydqj6cc7lpj3lw3iwqnxiak2juynysoevjmancad.onion/",
"duckduckgo": "https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/", "duckduckgo-onion": "https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/",
"danex": "http://danexio627wiswvlpt6ejyhpxl5gla5nt2tgvgm2apj2ofrgm44vbeyd.onion/", "danex": "http://danexio627wiswvlpt6ejyhpxl5gla5nt2tgvgm2apj2ofrgm44vbeyd.onion/",
"grams": "http://grams64rarzrk7rzdaz2fpb7lehcyi7zrrf5kd6w2uoamp7jw2aq6vyd.onion/", "grams": "http://grams64rarzrk7rzdaz2fpb7lehcyi7zrrf5kd6w2uoamp7jw2aq6vyd.onion/",
"torgle": "http://iy3544gmoeclh5de6gez2256v6pjh4omhpqdh2wpeeppjtvqmjhkfwad.onion/", "torgle": "http://iy3544gmoeclh5de6gez2256v6pjh4omhpqdh2wpeeppjtvqmjhkfwad.onion/",
"tordex": "http://tordexu73joywapk2txdr54jed4imqledpcvcuf75qsas2gwdgksvnyd.onion/", "tordex-onion": "http://tordexu73joywapk2txdr54jed4imqledpcvcuf75qsas2gwdgksvnyd.onion/",
"freshonion": "http://freshonifyfe4rmuh6qwpsexfhdrww7wnt5qmkoertwxmcuvm4woo4ad.onion/", "freshonion": "http://freshonifyfe4rmuh6qwpsexfhdrww7wnt5qmkoertwxmcuvm4woo4ad.onion/",
"Logen.int": "http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/",
"Logen.int": "http://3fzh7yuupdfyjhwt3ugzqqof6ulbcl27ecev33knxe3u7goi3vfn2qqd.onion",
"Logen.int": "http://uquroyobsaquslaunwkz6bmc3wutpzvwe7mv62xeq64645a57bugnsyd.onion",
"Logen.int": "http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/cgi-bin/omega/omega",
"Hidden Wiki blog": "http://darkwebp7lyr44rpgqdtevalty2pk5oqmc6m2cnicnix7itelt3lp3id.onion/",
"Onionx": "http://onionxpwovutq7bfv54ees3sfhcj32euy4bd555vvc4aufemxhkzvoyd.onion",
"Darksearch": "http://darkzqtmbdeauwq5mzcmgeeuhet42fhfjj4p5wbak3ofx2yqgecoeqyd.onion",
"OnionFind": "http://ofinde3b67voi7xiq3qflof2mwriwngicd7glwvf3bclgdgcjfozlzqd.onion/",
"SearchXNG": "http://searx3aolosaf3urwnhpynlhuokqsgz47si4pzz5hvb7uuzyjncl2tid.onion/",
"4GET": "http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion",
"DeepLink": "http://deepxfmkjlils3vd7bru5rrxy3x3lchjgmv3wsgycjfynktmuwgm64qd.onion",
"breaking bad forums" : "http://bbzzzsvqcrqtki6umym6itiixfhni37ybtt7mkbjyxn2pgllzxf2qgyd.onion",
# Clearnet (darkweb directory / tracker) # FIX: LOGEN.INT NAME TRANSFORM LOGEN.INT DATA SOURCE (WAJIB)
"logen-1": "http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/",
"logen-2": "http://3fzh7yuupdfyjhwt3ugzqqof6ulbcl27ecev33knxe3u7goi3vfn2qqd.onion",
"logen-3": "http://uquroyobsaquslaunwkz6bmc3wutpzvwe7mv62xeq64645a57bugnsyd.onion",
"logen-4": "http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/cgi-bin/omega/omega",
"hidden-wiki-blog": "http://darkwebp7lyr44rpgqdtevalty2pk5oqmc6m2cnicnix7itelt3lp3id.onion/",
"onionx": "http://onionxpwovutq7bfv54ees3sfhcj32euy4bd555vvc4aufemxhkzvoyd.onion",
"onionfind": "http://ofinde3b67voi7xiq3qflof2mwriwngicd7glwvf3bclgdgcjfozlzqd.onion/",
"searxng": "http://searx3aolosaf3urwnhpynlhuokqsgz47si4pzz5hvb7uuzyjncl2tid.onion/",
"4get": "http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion",
"deeplink": "http://deepxfmkjlils3vd7bru5rrxy3x3lchjgmv3wsgycjfynktmuwgm64qd.onion",
"breaking-bad-forums": "http://bbzzzsvqcrqtki6umym6itiixfhni37ybtt7mkbjyxn2pgllzxf2qgyd.onion",
# Clearnet trackers (onion link lists)
"darkwebdaily.net": "https://darkwebdaily.net/", "darkwebdaily.net": "https://darkwebdaily.net/",
"daut.link": "https://daunt.link/", "daunt.link": "https://daunt.link/",
"tor taxi": "https://tor.taxi/", "tor-taxi": "https://tor.taxi/",
"darkwebdaily": "https://darkwebdaily.live/", "darkwebdaily": "https://darkwebdaily.live/",
"immuniweb": "https://www.immuniweb.com/darkweb/", "immuniweb": "https://www.immuniweb.com/darkweb/",
"ransomwatch": "https://ransomwatch.telemetry.ltd/", "ransomwatch": "https://ransomwatch.telemetry.ltd/",
"watchguard": "https://www.watchguard.com/wgrd-security-hub/ransomware-tracker", "onionlive": "https://onion.live/",
"onionlive": "https://onion.live/", "torlink": "https://tor.link/",
"torlink": "https://tor.link/", "onionland": "https://onionland.io/",
"onionland": "https://onionland.io/", "dargle": "https://www.dargle.net/domains",
"dargle": "https://www.dargle.net/domains", "torry": "https://www.torry.io/",
"torry": "https://www.torry.io/", "onionlandse": "https://onionlandsearchengine.net/",
"onionlandse": "https://onionlandsearchengine.net/", "torsearch": "https://torsearch.com/",
"torsearch": "https://torsearch.com/", "onionlinkhub": "https://www.onionlinkhub.com/",
"onionlinkhub": "https://www.onionlinkhub.com/",
} }
# ========================= # =========================
@@ -173,23 +218,41 @@ COLLECTORS = {
# ========================= # =========================
def main(): def main():
print("[*] Onion directory daemon started") print("[*] Onion directory daemon started")
index_map = {}
while RUNNING: try:
try: while RUNNING:
session = tor_session() try:
index = load_index() session = tor_session()
except RuntimeError as e:
print(f"[!] {e}")
print(f"[!] Please check your TOR network....................")
for _ in range(60):
if not RUNNING:
break
time.sleep(1)
continue
index = load_index()
index_map = { index_map = {
onion_domain(i["link"]): i onion_domain(i["link"]): i
for i in index for i in index
if onion_domain(i.get("link")) if onion_domain(i.get("link"))
} }
added = 0 added = 0
for source, url in COLLECTORS.items(): for source, url in COLLECTORS.items():
if not RUNNING:
break
try: try:
r = session.get(url, timeout=45) print(f"\n[INFO] Fetching [{source}] => {url}")
start = time.time()
r = session.get(url, timeout=45)
elapsed = round(time.time() - start, 2)
print(f"[+] Response [{source}] status={r.status_code} size={len(r.text)} chars time={elapsed}s")
for item in extract_onions(r.text, source): for item in extract_onions(r.text, source):
domain = onion_domain(item["link"]) domain = onion_domain(item["link"])
if not domain: if not domain:
@@ -198,29 +261,40 @@ def main():
if domain in index_map: if domain in index_map:
index_map[domain]["updated_date"] = now() index_map[domain]["updated_date"] = now()
else: else:
item["stored_date"] = now() item["stored_date"] = now()
item["updated_date"] = item["stored_date"] item["updated_date"] = item["stored_date"]
index_map[domain] = item index_map[domain] = item
added += 1 added += 1
except:
continue except requests.exceptions.Timeout:
print(f"[!] TIMEOUT [{source}]")
except requests.exceptions.ConnectionError as e:
print(f"[!] CONNECTION ERROR [{source}]: {e}")
except Exception as e:
print(f"[!] ERROR [{source}]: {type(e).__name__}: {e}")
if added > 0: if added > 0:
save_index(list(index_map.values())) save_index(list(index_map.values()))
print(f"[+] Added {added} new domains (total {len(index_map)})") print(f"\n[+] Finished: +{added} new domains (total {len(index_map)})")
added = 0
else: else:
print("[=] No new domains found") print("\n[INFO] No new domains found")
except Exception as e: print(f"[INFO] Sleeping {INTERVAL//60} minutes...")
print("[!] Error:", e) for _ in range(INTERVAL):
if not RUNNING:
break
time.sleep(1)
# Sleep with graceful stop except KeyboardInterrupt:
for _ in range(INTERVAL): print("\n[!] Process interrupted by user (Ctrl+C). Saving progress...")
if not RUNNING: if index_map:
break save_index(list(index_map.values()))
time.sleep(1) print(f"[+] Index saved to {OUTPUT_FILE} (total {len(index_map)} domains)")
else:
print("[-] Nothing to save.")
print("[*] Onion directory daemon stopped cleanly") print("[*] Daemon stopped cleanly")
if __name__ == "__main__": if __name__ == "__main__":
main() main()