diff --git a/Web-Based/gen-node/node-generate.py b/Web-Based/gen-node/node-generate.py index 76a17c0..add912a 100644 --- a/Web-Based/gen-node/node-generate.py +++ b/Web-Based/gen-node/node-generate.py @@ -2,6 +2,7 @@ import requests import json import os from bs4 import BeautifulSoup +import time # Arr URLs urls = { @@ -10,6 +11,14 @@ urls = { "articles": "https://raw.githubusercontent.com/Jieyab89/OSINT-Cheat-sheet/main/awesome-article.md" } +headers = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/138.0 Safari/537.36" + ) +} + json_file = os.path.join(os.pardir, "osint_data.json") if os.path.exists(json_file): @@ -26,7 +35,12 @@ new_categories = {} current_category = None def parse_markdown_md(url): - response = requests.get(url) + time.sleep(6) + response = requests.get(url, headers=headers, timeout=30) + # response = requests.get(url) + time.sleep(6) + response = requests.get(url, headers=headers, timeout=30) + if response.status_code != 200: print(f"[-] Failed to fetch: {url}") return {} @@ -52,7 +66,14 @@ def parse_markdown_md(url): def parse_github_wiki(url): wiki_items = [] - response = requests.get(url) + + time.sleep(6) + response = requests.get(url, headers=headers, timeout=30) + wiki_items = [] + # response = requests.get(url) + time.sleep(6) + response = requests.get(url, headers=headers, timeout=30) + if response.status_code != 200: print(f"[-] Failed to fetch: {url}") return {} diff --git a/Web-Based/osint_data.json b/Web-Based/osint_data.json index b703732..6c2e140 100644 --- a/Web-Based/osint_data.json +++ b/Web-Based/osint_data.json @@ -21890,6 +21890,54 @@ { "name": "Enumerate and Attack Surface w Github", "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/Enumerate-and-Attack-Surface-w-Github" + }, + { + "name": "2 Notes Jieyab Wiki and Gitbook", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/2-Notes-Jieyab-Wiki-and-Gitbook" + }, + { + "name": "All About Darkweb", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/All-About-Darkweb" + }, + { + "name": "All About Narcotics or Drugs", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/All-About-Narcotics-or-Drugs" + }, + { + "name": "All About Penetration Testing", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/All-About-Penetration-Testing" + }, + { + "name": "All About Scammer", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/All-About-Scammer" + }, + { + "name": "Android Mobile Application for OSINT", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/Android-Mobile-Application-for-OSINT" + }, + { + "name": "Basic Programming Language", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/Basic-Programming-Language" + }, + { + "name": "Levaraging AI Agent (LLM) for OSINT", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/Levaraging-AI-Agent-(LLM)-for-OSINT" + }, + { + "name": "OSINT for Blockchain", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/OSINT-for-Blockchain" + }, + { + "name": "OSINT for Journalist and Activist", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/OSINT-for-Journalist-and-Activist" + }, + { + "name": "OSINT for Military", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/OSINT-for-Military" + }, + { + "name": "What is Social Engineering", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/What-is-Social-Engineering" } ] }