diff --git a/Web-Based/robots.txt b/Web-Based/robots.txt new file mode 100644 index 0000000..299c98a --- /dev/null +++ b/Web-Based/robots.txt @@ -0,0 +1,43 @@ +User-agent: * +Allow: / + +# Optional crawl delay +Crawl-delay: 5 + +# Sitemap +Sitemap: https://jieyab89.github.io/OSINT-Cheat-sheet/sitemap.xml +Sitemap: https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/sitemap-pages.xml + +# Block unnecessary paths +Disallow: /Script + +# AI / LLM discovery +Allow: /llms.txt + +# Block archive crawlers +User-agent: ia_archiver +Disallow: / + +User-agent: archive.org_bot +Disallow: / + +User-agent: archivebot +Disallow: / + +User-agent: wayback +Disallow: / + +User-agent: waybackbot +Disallow: / + +User-agent: archive-is +Disallow: / + +User-agent: archive.today_bot +Disallow: / + +User-agent: archive.ph +Disallow: / + +User-agent: saveweb2zip +Disallow: / \ No newline at end of file diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..299c98a --- /dev/null +++ b/robots.txt @@ -0,0 +1,43 @@ +User-agent: * +Allow: / + +# Optional crawl delay +Crawl-delay: 5 + +# Sitemap +Sitemap: https://jieyab89.github.io/OSINT-Cheat-sheet/sitemap.xml +Sitemap: https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/sitemap-pages.xml + +# Block unnecessary paths +Disallow: /Script + +# AI / LLM discovery +Allow: /llms.txt + +# Block archive crawlers +User-agent: ia_archiver +Disallow: / + +User-agent: archive.org_bot +Disallow: / + +User-agent: archivebot +Disallow: / + +User-agent: wayback +Disallow: / + +User-agent: waybackbot +Disallow: / + +User-agent: archive-is +Disallow: / + +User-agent: archive.today_bot +Disallow: / + +User-agent: archive.ph +Disallow: / + +User-agent: saveweb2zip +Disallow: / \ No newline at end of file diff --git a/sitemap.xml b/sitemap.xml new file mode 100644 index 0000000..3510c14 --- /dev/null +++ b/sitemap.xml @@ -0,0 +1,290 @@ + + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips + 1 + 2026-01-20T13:13:22.847Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-resources/osint-resources + 0.84 + 2026-02-14T20:29:27.177Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-resources/osint-resources/osint-cheat-sheet + 0.7 + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/homr + 0.84 + 2025-06-15T18:10:02.227Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/intelligence-base-and-knowledge-base + 0.84 + 2026-05-26T18:35:14.490Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/all-about-doxing + 0.84 + 2026-05-12T17:16:27.661Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/all-about-osint-thing-in-osint + 0.84 + 2026-05-07T18:21:45.666Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/all-about-sigint-signal-intelligence + 0.84 + 2026-05-05T08:01:34.257Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/intelligence-cell-phone + 0.84 + 2026-04-14T15:44:37.789Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/geospatial-intelligence + 0.84 + 2026-05-05T08:01:34.270Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/how-to-collection-data-in-osint + 0.84 + 2025-06-15T16:53:01.127Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/how-to-verify-content-in-osint + 0.84 + 2025-06-15T16:53:59.103Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/image-intelligence-vidio-intelligence-analysis + 0.84 + 2025-06-15T16:56:47.277Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/osint-for-academic-literature-and-sciene + 0.84 + 2025-06-15T16:59:30.097Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/how-to-email-lookup-in-osint + 0.84 + 2026-05-26T18:28:28.096Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/osint-for-historical + 0.84 + 2025-06-15T17:04:57.841Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/osint-for-journalism + 0.84 + 2025-06-15T17:51:24.336Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/osint-opsec-operation-security-things-opsec-for-osint + 0.84 + 2025-06-15T17:52:42.557Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/osint-for-war-analysis-and-geopolitics + 0.84 + 2025-06-29T15:49:13.226Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/risk-and-general-in-osint + 0.84 + 2025-06-15T17:57:23.420Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/where-data-from-in-osint + 0.84 + 2025-06-15T17:58:51.662Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/inside-about-data-breach-in-osint + 0.84 + 2025-06-15T18:03:14.090Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/osint-tips-investigation-and-analysis + 0.84 + 2025-06-15T18:04:30.309Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/challenge-in-socmint-osint-socmint-tips + 0.84 + 2025-06-15T18:05:39.941Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/osint-facebook-socmint-facebook-social-media + 0.84 + 2025-06-15T18:07:40.789Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/what-is-bias-in-analysis + 0.84 + 2025-06-15T18:12:31.310Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/what-is-data-broker + 0.84 + 2026-05-10T16:19:38.433Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/what-is-metadata + 0.84 + 2025-06-15T18:16:02.040Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/what-is-propaganda + 0.84 + 2025-06-15T18:17:13.250Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/what-is-sock-puppets-account + 0.84 + 2025-06-15T18:19:58.499Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/who-was-used-osint + 0.84 + 2025-06-15T18:19:58.499Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/what-is-data-enrichment-and-pivoting + 0.84 + 2026-01-20T13:27:48.781Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/all-about-stealer-log-and-cyber-threat + 0.84 + 2026-05-04T18:47:30.547Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/all-about-penetration-testing + 0.84 + 2026-05-26T16:27:28.499Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/all-about-device-farm + 0.84 + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/intelligence-base/all-about-scammer + 0.84 + 2026-05-17T15:30:27.117Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/scripting-for-osint/home + 0.84 + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/scripting-for-osint/basic-programming-language + 0.84 + 2026-05-12T17:04:18.835Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/scripting-for-osint/all-about-scrapping-art-of-scrapping + 0.84 + 2026-01-20T13:44:44.450Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/scripting-for-osint/openapi + 0.84 + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/scripting-for-osint/android-mobile-application-for-osint + 0.84 + 2026-05-06T15:58:58.123Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/scripting-for-osint/ios-mobile-application-for-osint + 0.84 + 2026-02-01T15:45:03.762Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/home + 0.84 + 2025-06-25T06:50:58.974Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/how-to-spot-a-deepfake + 0.84 + 2026-05-26T16:44:47.212Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/guide-satellite-imagery + 0.84 + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/what-is-social-engineering + 0.84 + 2025-06-25T07:16:41.149Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/sexual-abuse-how-to-revenge + 0.84 + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/osint-for-military + 0.84 + 2025-07-20T18:39:50.013Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/osint-for-journalist-and-activist + 0.84 + 2025-06-29T16:53:06.487Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/osint-for-blockchain-and-transaction + 0.84 + 2025-06-27T17:16:13.418Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/osint-and-socmint-tips + 0.84 + 2026-04-18T10:12:21.725Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/all-about-humint + 0.84 + 2026-05-04T17:29:28.050Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/levaraging-ai-agent-llm-for-osint + 0.84 + 2025-07-20T19:09:59.468Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/osint-for-medic-and-analsyis-forensic-dead-body + 0.84 + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/protect-your-fingerprint-and-sock-account + 0.84 + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/red-team-hid-hardware-pentesting + 0.84 + 2025-08-18T15:57:48.060Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/red-team-and-wiretapping-intel-osint + 0.84 + 2025-08-09T19:00:08.365Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/attacking-ai-and-ml-with-prompt-injection + 0.84 + 2025-08-18T16:09:12.611Z + + + https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/all-about-darkweb-tips-darkweb-osint-assessments + 0.84 + 2026-05-19T16:00:14.239Z + + \ No newline at end of file