added new script

Script/CNN-News-Scrapping-and-Archive/CNN-scarpper.py

@@ -0,0 +1,148 @@
+import asyncio
+import json
+import os
+import re
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+from playwright.async_api import async_playwright
+
+SITEMAP_URL = "https://www.cnnindonesia.com/nasional/sitemap_web.xml"
+
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240"
+}
+
+
+# =========================
+# UTILS
+# =========================
+def safe_filename(text):
+    return re.sub(r'[^a-zA-Z0-9]', '_', text)[:100]
+
+
+def download_file(url, path):
+    try:
+        r = requests.get(url, headers=HEADERS, timeout=10)
+        if r.status_code == 200:
+            with open(path, "wb") as f:
+                f.write(r.content)
+    except:
+        pass
+
+
+# =========================
+# GET SITEMAP
+# =========================
+def get_urls():
+    res = requests.get(SITEMAP_URL, headers=HEADERS)
+    soup = BeautifulSoup(res.text, "xml")
+    urls = [loc.text for loc in soup.find_all("loc")]
+    return urls[:5]
+
+
+# =========================
+# SCRAPE + ARCHIVE
+# =========================
+async def scrape_article(page, url):
+    try:
+        await page.goto(url, timeout=60000)
+        await page.wait_for_selector("h1")
+
+        html = await page.content()
+        soup = BeautifulSoup(html, "html.parser")
+
+        title = soup.select_one("h1").get_text(strip=True)
+        slug = safe_filename(title)
+
+        base_dir = f"output/{slug}"
+        img_dir = f"{base_dir}/images"
+
+        os.makedirs(img_dir, exist_ok=True)
+
+        # =========================
+        # DOWNLOAD IMAGES
+        # =========================
+        images = soup.find_all("img")
+
+        for i, img in enumerate(images):
+            src = img.get("src")
+
+            if not src:
+                continue
+
+            full_url = urljoin(url, src)
+
+            # filter hanya gambar
+            if not any(ext in full_url.lower() for ext in [".jpg", ".jpeg", ".png"]):
+                continue
+
+            filename = f"img_{i}.jpg"
+            filepath = os.path.join(img_dir, filename)
+
+            download_file(full_url, filepath)
+
+            # rewrite path di HTML
+            img["src"] = f"images/{filename}"
+
+        # =========================
+        # SAVE HTML
+        # =========================
+        with open(f"{base_dir}/index.html", "w", encoding="utf-8") as f:
+            f.write(str(soup))
+
+        # =========================
+        # EXTRACT TEXT
+        # =========================
+        paragraphs = soup.select("div.detail-text p")
+        content = "\n".join([p.get_text(strip=True) for p in paragraphs])
+
+        date = ""
+        date_el = soup.select_one("div.text-cnn_grey")
+        if date_el:
+            date = date_el.get_text(strip=True)
+
+        return {
+            "url": url,
+            "title": title,
+            "date": date,
+            "content": content,
+            "saved_path": base_dir
+        }
+
+    except Exception as e:
+        print(f"Error: {url} -> {e}")
+        return None
+
+
+# =========================
+# MAIN
+# =========================
+async def main():
+    urls = get_urls()
+    results = []
+
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True)
+        page = await browser.new_page()
+
+        for url in urls:
+            print("[SCRAPE]", url)
+
+            data = await scrape_article(page, url)
+
+            if data:
+                results.append(data)
+
+            await asyncio.sleep(2)
+
+        await browser.close()
+
+    with open("cnn_results.json", "w", encoding="utf-8") as f:
+        json.dump(results, f, ensure_ascii=False, indent=4)
+
+    print("[DONE] Archive + JSON selesai")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

Script/CNN-News-Scrapping-and-Archive/index.html

@@ -0,0 +1,66 @@
+<!DOCTYPE html>
+<html lang="id">
+<head>
+    <meta charset="UTF-8">
+    <title>CNN News Scrapper</title>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
+</head>
+<body class="bg-dark text-light">
+
+<div class="container mt-5">
+    <h2 class="mb-4">CNN News Scrapper</h2>
+    <div id="news-container" class="row"></div>
+</div>
+
+<script>
+async function loadData() {
+    try {
+        const res = await fetch('./cnn_results.json');
+        const data = await res.json();
+
+        const container = document.getElementById('news-container');
+
+        data.forEach(article => {
+            const col = document.createElement('div');
+            col.className = "col-md-6 mb-4";
+
+            col.innerHTML = `
+                <div class="card bg-secondary text-light h-100">
+                    <div class="card-body d-flex flex-column">
+                        <h5 class="card-title">${article.title}</h5>
+                        <p class="text-warning">${article.date}</p>
+                        <p class="card-text flex-grow-1">
+                            ${article.content.substring(0, 200)}...
+                        </p>
+
+                        <div class="d-flex gap-2 mt-3">
+                            <a href="${article.saved_path}/index.html" target="_blank" class="btn btn-success btn-sm">
+                                Open the Archive 
+                            </a>
+
+                            <a href="${article.url}" target="_blank" class="btn btn-primary btn-sm">
+                                Original Source
+                            </a>
+                        </div>
+                    </div>
+                </div>
+            `;
+
+            container.appendChild(col);
+        });
+
+    } catch (err) {
+        document.getElementById('news-container').innerHTML = `
+            <div class="alert alert-danger">
+                Failed to load data in json (python -m http.server)
+            </div>
+        `;
+        console.error(err);
+    }
+}
+
+loadData();
+</script>
+
+</body>
+</html>

Script/CNN-News-Scrapping-and-Archive/requirements.txt

@@ -0,0 +1,4 @@
+requests>=2.32.0
+beautifulsoup4>=4.13.0
+playwright>=1.54.0
+lxml>=6.0.0