edit the script and add readme

Script/CNN-News-Scrapping-and-Archive/CNN-scarpper.py

@@ -2,17 +2,41 @@ import asyncio
 import json
 import os
 import re
+import signal
+from datetime import datetime, UTC
+
 import requests
 from bs4 import BeautifulSoup
-from urllib.parse import urljoin, urlparse
+from urllib.parse import urljoin
 from playwright.async_api import async_playwright
 
-SITEMAP_URL = "https://www.cnnindonesia.com/nasional/sitemap_web.xml"
+SITEMAP_URLS = [
+    "https://www.cnnindonesia.com/nasional/sitemap_web.xml",
+    "https://www.cnnindonesia.com/internasional/sitemap_web.xml",
+]
 
 HEADERS = {
     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240"
 }
 
+OUTPUT_DIR = "output"
+RESULTS_FILE = "cnn_results.json"
+STATE_FILE = "scraped_urls.json"
+
+ARTICLE_DELAY = 3       # seconds, delay between articles so we don't hammer the server
+CYCLE_SLEEP = 300       # seconds, pause between scraping cycles (5 minutes)
+REQUEST_TIMEOUT = 60000 # ms, playwright goto timeout
+
+# =========================
+# GRACEFUL SHUTDOWN
+# =========================
+shutdown_event = asyncio.Event()
+
+
+def handle_shutdown(sig, frame):
+    print("\n[STOP] Shutdown signal received, finishing current task...")
+    shutdown_event.set()
+
 
 # =========================
 # UTILS
@@ -27,18 +51,61 @@ def download_file(url, path):
         if r.status_code == 200:
             with open(path, "wb") as f:
                 f.write(r.content)
-    except:
+    except Exception:
         pass
 
 
-# =========================
+def load_json_safe(path, default):
-# GET SITEMAP
+    if os.path.exists(path):
-# =========================
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                return json.load(f)
+        except Exception:
+            return default
+    return default
+
+
+def load_scraped_urls():
+    data = load_json_safe(STATE_FILE, [])
+    return set(data)
+
+
+def save_scraped_urls(urls_set):
+    with open(STATE_FILE, "w", encoding="utf-8") as f:
+        json.dump(list(urls_set), f, ensure_ascii=False, indent=2)
+
+
+def append_result(result):
+    results = load_json_safe(RESULTS_FILE, [])
+    results.append(result)
+    with open(RESULTS_FILE, "w", encoding="utf-8") as f:
+        json.dump(results, f, ensure_ascii=False, indent=4)
+
+
+# =========================================================
+# GET SITEMAPS (unlimited, multiple sources)
+# =========================================================
 def get_urls():
-    res = requests.get(SITEMAP_URL, headers=HEADERS)
+    all_urls = []
-    soup = BeautifulSoup(res.text, "xml")
+    for sitemap_url in SITEMAP_URLS:
-    urls = [loc.text for loc in soup.find_all("loc")]
+        try:
-    return urls[:5]
+            res = requests.get(sitemap_url, headers=HEADERS, timeout=15)
+            res.raise_for_status()
+            soup = BeautifulSoup(res.text, "xml")
+            urls = [loc.text.strip() for loc in soup.find_all("loc") if loc.text.strip()]
+            print(f"[SITEMAP] {sitemap_url} -> {len(urls)} URLs")
+            all_urls.extend(urls)
+        except Exception as e:
+            print(f"[ERROR] Failed to fetch sitemap {sitemap_url}: {e}")
+
+    # dedupe while preserving order
+    seen = set()
+    deduped = []
+    for u in all_urls:
+        if u not in seen:
+            seen.add(u)
+            deduped.append(u)
+    return deduped
 
 
 # =========================
@@ -46,54 +113,40 @@ def get_urls():
 # =========================
 async def scrape_article(page, url):
     try:
-        await page.goto(url, timeout=60000)
+        await page.goto(url, timeout=REQUEST_TIMEOUT)
-        await page.wait_for_selector("h1")
+        await page.wait_for_selector("h1", timeout=15000)
-
         html = await page.content()
         soup = BeautifulSoup(html, "html.parser")
 
-        title = soup.select_one("h1").get_text(strip=True)
+        h1 = soup.select_one("h1")
+        if not h1:
+            print(f"[SKIP] No title found: {url}")
+            return None
+        title = h1.get_text(strip=True)
         slug = safe_filename(title)
-
+        base_dir = f"{OUTPUT_DIR}/{slug}"
-        base_dir = f"output/{slug}"
         img_dir = f"{base_dir}/images"
-
         os.makedirs(img_dir, exist_ok=True)
 
-        # =========================
         # DOWNLOAD IMAGES
-        # =========================
         images = soup.find_all("img")
-
         for i, img in enumerate(images):
             src = img.get("src")
-
             if not src:
                 continue
-
             full_url = urljoin(url, src)
-
-            # filter hanya gambar
             if not any(ext in full_url.lower() for ext in [".jpg", ".jpeg", ".png"]):
                 continue
-
             filename = f"img_{i}.jpg"
             filepath = os.path.join(img_dir, filename)
-
             download_file(full_url, filepath)
-
-            # rewrite path di HTML
             img["src"] = f"images/{filename}"
 
-        # =========================
         # SAVE HTML
-        # =========================
         with open(f"{base_dir}/index.html", "w", encoding="utf-8") as f:
             f.write(str(soup))
 
-        # =========================
         # EXTRACT TEXT
-        # =========================
         paragraphs = soup.select("div.detail-text p")
         content = "\n".join([p.get_text(strip=True) for p in paragraphs])
 
@@ -107,42 +160,85 @@ async def scrape_article(page, url):
             "title": title,
             "date": date,
             "content": content,
-            "saved_path": base_dir
+            "saved_path": base_dir,
+            "scraped_at": datetime.now(UTC).isoformat(),
         }
-
     except Exception as e:
-        print(f"Error: {url} -> {e}")
+        print(f"[ERROR] {url} -> {e}")
         return None
 
 
 # =========================
-# MAIN
+# SINGLE SCRAPE CYCLE
+# =========================
+async def run_cycle(browser, scraped_urls):
+    all_urls = get_urls()
+    new_urls = [u for u in all_urls if u not in scraped_urls]
+
+    if not new_urls:
+        print("[INFO] No new articles this cycle.")
+        return
+
+    print(f"[INFO] {len(new_urls)} new articles found out of {len(all_urls)} total sitemap URLs.")
+    page = await browser.new_page()
+
+    try:
+        for url in new_urls:
+            if shutdown_event.is_set():
+                print("[STOP] Stopping mid-cycle due to shutdown.")
+                break
+
+            print("[SCRAPE]", url)
+            data = await scrape_article(page, url)
+            if data:
+                append_result(data)
+                scraped_urls.add(url)
+                save_scraped_urls(scraped_urls)
+            else:
+                # mark as seen anyway so we don't keep retrying it every cycle
+                scraped_urls.add(url)
+                save_scraped_urls(scraped_urls)
+
+            # delay between articles, interruptible by shutdown
+            try:
+                await asyncio.wait_for(shutdown_event.wait(), timeout=ARTICLE_DELAY)
+            except asyncio.TimeoutError:
+                pass
+    finally:
+        await page.close()
+
+
+# =========================
+# MAIN DAEMON LOOP
 # =========================
 async def main():
-    urls = get_urls()
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
-    results = []
+    scraped_urls = load_scraped_urls()
+    print(f"[INIT] {len(scraped_urls)} URLs already recorded from previous runs.")
 
     async with async_playwright() as p:
         browser = await p.chromium.launch(headless=True)
-        page = await browser.new_page()
+        try:
+            while not shutdown_event.is_set():
+                print(f"\n[CYCLE] Starting scrape cycle - {datetime.now(UTC).isoformat()}")
+                await run_cycle(browser, scraped_urls)
 
-        for url in urls:
+                if shutdown_event.is_set():
-            print("[SCRAPE]", url)
+                    break
 
-            data = await scrape_article(page, url)
+                print(f"[SLEEP] Sleeping {CYCLE_SLEEP}s before next cycle...")
+                try:
+                    await asyncio.wait_for(shutdown_event.wait(), timeout=CYCLE_SLEEP)
+                except asyncio.TimeoutError:
+                    pass
+        finally:
+            print("[CLEANUP] Closing browser...")
+            await browser.close()
 
-            if data:
+    print("[DONE] Daemon stopped cleanly.")
-                results.append(data)
-
-            await asyncio.sleep(2)
-
-        await browser.close()
-
-    with open("cnn_results.json", "w", encoding="utf-8") as f:
-        json.dump(results, f, ensure_ascii=False, indent=4)
-
-    print("[DONE] Archive + JSON selesai")
 
 
 if __name__ == "__main__":
+    signal.signal(signal.SIGINT, handle_shutdown)
+    signal.signal(signal.SIGTERM, handle_shutdown)
     asyncio.run(main())

Script/CNN-News-Scrapping-and-Archive/Readme.md

@@ -0,0 +1,9 @@
+# Tutorial 
+
+```install playwright
+playwright install chromium
+```
+
+You can visit my Gitbook 
+
+https://jieyab89-osint.gitbook.io/jieyab89-osint-cheat-sheet-wiki-tips/osint-tool-resouces-usage/all-about-darkweb-tips-darkweb-osint-assessments 

Script/CNN-News-Scrapping-and-Archive/index.html

@@ -2,57 +2,149 @@
 <html lang="id">
 <head>
     <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>CNN News Scrapper</title>
     <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
+    <style>
+        .card-title { word-break: break-word; }
+        #search-input::placeholder { color: #adb5bd; }
+        .stat-badge { font-size: 0.85rem; }
+    </style>
 </head>
 <body class="bg-dark text-light">
 
 <div class="container mt-5">
-    <h2 class="mb-4">CNN News Scrapper</h2>
+    <div class="d-flex justify-content-between align-items-center flex-wrap gap-2 mb-3">
+        <h2 class="mb-0">CNN News Scrapper</h2>
+        <span id="article-count" class="badge bg-secondary stat-badge"></span>
+    </div>
+
+    <div class="mb-4">
+        <input type="text" id="search-input" class="form-control bg-secondary text-light border-0"
+               placeholder="Search the article">
+    </div>
+
+    <div id="loading" class="text-center text-secondary py-5">
+        <div class="spinner-border text-light" role="status"></div>
+        <p class="mt-2">Loading..............</p>
+    </div>
+
     <div id="news-container" class="row"></div>
+    <div id="empty-state" class="alert alert-secondary d-none">Article not found :(</div>
 </div>
 
 <script>
-async function loadData() {
+let allArticles = [];
-    try {
-        const res = await fetch('./cnn_results.json');
-        const data = await res.json();
 
-        const container = document.getElementById('news-container');
+function escapeHtml(str) {
+    if (str === null || str === undefined) return "";
+    return String(str)
+        .replace(/&/g, "&amp;")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;")
+        .replace(/"/g, "&quot;")
+        .replace(/'/g, "&#039;");
+}
 
-        data.forEach(article => {
+function normalizePath(path) {
-            const col = document.createElement('div');
+    if (!path) return "";
-            col.className = "col-md-6 mb-4";
+    return path.replace(/\\/g, "/");
+}
 
-            col.innerHTML = `
+function renderArticles(articles) {
-                <div class="card bg-secondary text-light h-100">
+    const container = document.getElementById('news-container');
-                    <div class="card-body d-flex flex-column">
+    const emptyState = document.getElementById('empty-state');
-                        <h5 class="card-title">${article.title}</h5>
+    container.innerHTML = "";
-                        <p class="text-warning">${article.date}</p>
-                        <p class="card-text flex-grow-1">
-                            ${article.content.substring(0, 200)}...
-                        </p>
 
-                        <div class="d-flex gap-2 mt-3">
+    if (articles.length === 0) {
-                            <a href="${article.saved_path}/index.html" target="_blank" class="btn btn-success btn-sm">
+        emptyState.classList.remove('d-none');
-                                Open the Archive 
+        return;
-                            </a>
+    }
+    emptyState.classList.add('d-none');
 
-                            <a href="${article.url}" target="_blank" class="btn btn-primary btn-sm">
+    articles.forEach(article => {
-                                Original Source
+        const title = escapeHtml(article.title || "(No title article)");
-                            </a>
+        const date = escapeHtml(article.date || "");
-                        </div>
+        const rawContent = article.content || "";
+        const preview = escapeHtml(rawContent.substring(0, 200)) + (rawContent.length > 200 ? "..." : "");
+        const archivePath = normalizePath(article.saved_path) + "/index.html";
+        const archiveHref = encodeURI(archivePath);
+        const originalHref = encodeURI(article.url || "#");
+
+        const col = document.createElement('div');
+        col.className = "col-md-6 mb-4";
+
+        col.innerHTML = `
+            <div class="card bg-secondary text-light h-100">
+                <div class="card-body d-flex flex-column">
+                    <h5 class="card-title">${title}</h5>
+                    <p class="text-warning">${date || "Date not available"}</p>
+                    <p class="card-text flex-grow-1">${preview || "There is no content.........."}</p>
+
+                    <div class="d-flex gap-2 mt-3">
+                        <a href="${archiveHref}" target="_blank" rel="noopener" class="btn btn-success btn-sm">
+                            Open the Archive
+                        </a>
+                        <a href="${originalHref}" target="_blank" rel="noopener" class="btn btn-primary btn-sm">
+                            Original Source
+                        </a>
                     </div>
                 </div>
-            `;
+            </div>
+        `;
 
-            container.appendChild(col);
+        container.appendChild(col);
+    });
+}
+
+function updateCount(shown, total) {
+    const badge = document.getElementById('article-count');
+    badge.textContent = shown === total
+        ? `${total} article`
+        : `${shown} from ${total} article`;
+}
+
+function filterArticles(query) {
+    const q = query.trim().toLowerCase();
+    if (!q) {
+        renderArticles(allArticles);
+        updateCount(allArticles.length, allArticles.length);
+        return;
+    }
+    const filtered = allArticles.filter(a =>
+        (a.title || "").toLowerCase().includes(q) ||
+        (a.content || "").toLowerCase().includes(q)
+    );
+    renderArticles(filtered);
+    updateCount(filtered.length, allArticles.length);
+}
+
+async function loadData() {
+    const loading = document.getElementById('loading');
+    try {
+        const res = await fetch('./cnn_results.json');
+        if (!res.ok) {
+            throw new Error(`HTTP ${res.status}`);
+        }
+        const data = await res.json();
+
+        allArticles = Array.isArray(data) ? data : [];
+        allArticles = [...allArticles].reverse();
+
+        loading.classList.add('d-none');
+        renderArticles(allArticles);
+        updateCount(allArticles.length, allArticles.length);
+
+        document.getElementById('search-input').addEventListener('input', (e) => {
+            filterArticles(e.target.value);
         });
 
     } catch (err) {
+        loading.classList.add('d-none');
         document.getElementById('news-container').innerHTML = `
             <div class="alert alert-danger">
-                Failed to load data in json (python -m http.server)
+                Failed to load cnn_results.json. Makesure the local server run
+                (example: <code>python -m http.server</code>).
             </div>
         `;
         console.error(err);