Day 11：召喚別的名單變成我的

17th鐵人賽

Finley

2025-09-25 21:44:03

12 瀏覽

分享至

[鐵人賽] Day 11：威脅情資進防火牆——用 Ollama 生出爬蟲，再用 Python 一鍵產生 FortiGate 地址物件

流程概觀

情資平台/社群黑名單 (IPs / CIDRs)
            │  (HTTP/HTTPS 下載爬取)
            ▼
        Python 清洗去重
     (正規化、CIDR→子網掩碼)
            │
            ├─ fortigate_blacklist_cli.txt   ← 一鍵貼上 FortiGate CLI
            └─ fortigate_blacklist.csv       ← 給你備查/手動匯入/稽核

使用方法（三步驟）

把你要抓的黑名單網址放到文字檔（例如 sources.txt），每行一個 URL。
執行腳本（Windows/Linux 都可）
產出的 fortigate_blacklist_cli.txt 內容整段貼到 FortiGate（或透過 SSH/腳本套用）；或用 CSV 做審閱/備查。

單檔腳本（Crawler + 轉 FortiGate）

說明：

預設從 sources.txt 讀 URL；也可用 --urls 直接給多個網址。

產出 fortigate_blacklist_cli.txt 與 fortigate_blacklist.csv。

只處理 IPv4（含 /CIDR）；孤立 IP 會自動視為 /32。

FQDN 黑名單延伸在文末附。

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
ti2fortigate.py
抓取威脅情資 IP/CIDR 黑名單 → 轉成 FortiGate 可匯入的地址物件與群組
作者：你今天的 D11
用法：
  python ti2fortigate.py --sources sources.txt \
                         --group-name BL_IP_ALL \
                         --name-prefix BL_IP_ \
                         --vdom root \
                         --max-group-members 5000
或直接給多個網址：
  python ti2fortigate.py --urls https://example.com/ips.txt https://foo/bar.list
"""

import argparse
import csv
import ipaddress
import re
import sys
from typing import Iterable, List, Set, Tuple

# 嘗試使用 requests；若缺少則退回 urllib
try:
    import requests
    def fetch_text(url: str, timeout: int = 20) -> str:
        r = requests.get(url, timeout=timeout)
        r.raise_for_status()
        return r.text
except Exception:
    import urllib.request
    def fetch_text(url: str, timeout: int = 20) -> str:
        with urllib.request.urlopen(url, timeout=timeout) as resp:
            return resp.read().decode("utf-8", errors="ignore")

IP_TOKEN_RE = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}(?:/\d{1,2})?\b")

def extract_ipv4_cidrs(text: str) -> Set[str]:
    """從文字中抓 IPv4 token；校驗後只留下合格的 IP 或 CIDR。"""
    out: Set[str] = set()
    for token in IP_TOKEN_RE.findall(text):
        try:
            if "/" in token:
                # 驗證 CIDR
                net = ipaddress.IPv4Network(token, strict=False)
                out.add(str(net))
            else:
                # 單一 IP 視為 /32
                ipaddress.IPv4Address(token)
                out.add(str(ipaddress.IPv4Network(f"{token}/32")))
        except Exception:
            continue
    return out

def cidr_to_ip_mask(cidr: str) -> Tuple[str, str]:
    """CIDR → (network_ip, netmask_str) for FortiGate 'set subnet'"""
    net = ipaddress.IPv4Network(cidr, strict=False)
    return (str(net.network_address), str(net.netmask))

def load_sources_from_file(path: str) -> List[str]:
    urls: List[str] = []
    with open(path, "r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            urls.append(line)
    return urls

def chunked(seq: List[str], size: int) -> Iterable[List[str]]:
    for i in range(0, len(seq), size):
        yield seq[i:i+size]

def build_fgt_cli(
    cidrs: List[str],
    group_name: str = "BL_IP_ALL",
    name_prefix: str = "BL_IP_",
    vdom: str = None,
    max_group_members: int = 2000,
) -> str:
    """產生 FortiGate CLI 設定（地址物件 + 群組）。"""
    lines: List[str] = []
    if vdom:
        lines += [f"config vdom", f"edit {vdom}"]

    # 建立地址物件
    lines.append("config firewall address")
    for idx, cidr in enumerate(cidrs, 1):
        obj_name = f"{name_prefix}{idx}"
        ip, mask = cidr_to_ip_mask(cidr)
        lines += [
            f"    edit {obj_name}",
            f"        set type ipmask",
            f"        set subnet {ip} {mask}",
            f"        set comment \"TI import {cidr}\"",
            f"    next",
        ]
    lines.append("end")

    # 建立/更新群組，若成員過多則分批多個群組
    if cidrs:
        obj_names = [f"{name_prefix}{i}" for i in range(1, len(cidrs)+1)]
        group_chunks = list(chunked(obj_names, max_group_members))
        for gi, members in enumerate(group_chunks, 1):
            gname = group_name if len(group_chunks) == 1 else f"{group_name}_{gi}"
            # FortiGate: config firewall addrgrp; edit NAME; set member a b c; next
            lines.append("config firewall addrgrp")
            lines += [f"    edit {gname}"]
            # 避免超長行，分批 set member
            for ch in chunked(members, 256):
                joined = " ".join(ch)
                lines += [f"        set member {joined}"]
            lines += ["    next", "end"]

    if vdom:
        lines.append("end")  # 對應 config vdom

    return "\n".join(lines) + "\n"

def write_cli(path: str, cli: str) -> None:
    with open(path, "w", encoding="utf-8") as f:
        f.write(cli)

def write_csv(path: str, cidrs: List[str], name_prefix: str) -> None:
    """友善備查：Name,Type,Network,Netmask,Comment"""
    with open(path, "w", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow(["Name", "Type", "Network", "Netmask", "Comment"])
        for idx, cidr in enumerate(cidrs, 1):
            ip, mask = cidr_to_ip_mask(cidr)
            writer.writerow([f"{name_prefix}{idx}", "ipmask", ip, mask, f"TI import {cidr}"])

def main():
    ap = argparse.ArgumentParser(description="Threat Intel → FortiGate Address Objects")
    g = ap.add_mutually_exclusive_group(required=True)
    g.add_argument("--sources", help="文字檔，內含多個黑名單 URL（每行一個）")
    g.add_argument("--urls", nargs="+", help="直接給多個黑名單 URL")
    ap.add_argument("--group-name", default="BL_IP_ALL", help="地址群組名稱")
    ap.add_argument("--name-prefix", default="BL_IP_", help="地址物件命名前綴")
    ap.add_argument("--vdom", default=None, help="VDOM 名稱（預設不使用）")
    ap.add_argument("--max-group-members", type=int, default=2000, help="群組最大成員數（超過會自動切成多群組）")
    ap.add_argument("--cli-out", default="fortigate_blacklist_cli.txt", help="輸出 FortiGate CLI 檔名")
    ap.add_argument("--csv-out", default="fortigate_blacklist.csv", help="輸出 CSV 檔名")
    ap.add_argument("--print-stats", action="store_true", help="印出統計訊息")
    args = ap.parse_args()

    urls = args.urls or load_sources_from_file(args.sources)
    if not urls:
        print("❌ 沒有任何 URL", file=sys.stderr)
        sys.exit(2)

    all_cidrs: Set[str] = set()
    for url in urls:
        try:
            text = fetch_text(url)
            got = extract_ipv4_cidrs(text)
            all_cidrs |= got
            if args.print_stats:
                print(f"✔ 來源 {url}：擷取 {len(got)} 筆（累計 {len(all_cidrs)}）")
        except Exception as e:
            print(f"⚠ 無法取得 {url}: {e}", file=sys.stderr)

    cidrs = sorted(all_cidrs, key=lambda s: (ipaddress.IPv4Network(s, strict=False).network_address.packed,
                                             ipaddress.IPv4Network(s, strict=False).prefixlen))

    cli = build_fgt_cli(
        cidrs=cidrs,
        group_name=args.group_name,
        name_prefix=args.name_prefix,
        vdom=args.vdom,
        max_group_members=args.max_group_members,
    )
    write_cli(args.cli_out, cli)
    write_csv(args.csv_out, cidrs, args.name_prefix)

    if args.print_stats:
        print(f"🧾 產出 CLI：{args.cli_out}")
        print(f"🧾 產出 CSV：{args.csv_out}")
        print(f"📦 地址物件：{len(cidrs)}；群組：{args.group_name}（或自動切片）")

if __name__ == "__main__":
    main()

快速上手（實例）

建一個 sources.txt：

https://example.org/blacklist-ips.txt
https://another-source/foo/cidr.list

執行

python ti2fortigate.py --sources sources.txt --group-name BL_IP_ALL --name-prefix BL_IP_ --vdom root --print-stats

把 fortigate_blacklist_cli.txt 整段貼進 FortiGate（SSH/Console）

config vdom
edit root
config firewall address
    edit BL_IP_1
        set type ipmask
        set subnet 1.2.3.0 255.255.255.0
        set comment "TI import 1.2.3.0/24"
    next
    ...
end
config firewall addrgrp
    edit BL_IP_ALL
        set member BL_IP_1 BL_IP_2 BL_IP_3 ...
    next
end
end

之後在你的安全政策（policy）裡，把 Source/Destination Address 用到 BL_IP_ALL 即可。