#!/usr/bin/env python3
"""Fetch the PROHARDVER! "Home server / hazi szerver epitese" topic pages, politely.

Read-only research (forum-research-2026-10-12). Stdlib only.
- One request at a time, at least DELAY seconds apart (robots.txt asks for 1 s; we use 5).
- Cookie jar + redirects: the first request answers 302 -> ?_tc=1 (a cookie check).
- Each page is cached as raw HTML outside git; a cached page is never fetched again.
- STOPS on 403, 429, a captcha or a login wall (spec section 9.2). No retry around a block.

Usage:
  python3 fetch.py FIRST_POST LAST_POST     # fetch every 100-post page covering the range
Pages: hsz_<a>-<b>.html with a = 1 + 100*k, b = a + 99.
"""
import http.cookiejar
import os
import sys
import time
import urllib.error
import urllib.request

BASE = "https://prohardver.hu/tema/hazi_szerver_epitese/"
CACHE = os.path.expanduser("~/research-cache/prohardver-hazi-szerver")
PAGES = os.path.join(CACHE, "pages")
LOG = os.path.join(CACHE, "fetch.log")
DELAY = 5.0
UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36")
BLOCK_MARKERS = ("captcha", "recaptcha", "hcaptcha")


def log(msg):
    line = time.strftime("%Y-%m-%dT%H:%M:%S%z") + " " + msg
    print(line, flush=True)
    with open(LOG, "a", encoding="utf-8") as f:
        f.write(line + "\n")


def page_name(first):
    return "hsz_%d-%d.html" % (first, first + 99)


def count_posts(html):
    # A rough count for the log only; parse.py does the real work.
    return html.count('data-id="')


def main():
    if len(sys.argv) != 3:
        sys.exit(__doc__)
    lo, hi = int(sys.argv[1]), int(sys.argv[2])
    os.makedirs(PAGES, exist_ok=True)
    jar = http.cookiejar.CookieJar()
    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar))
    opener.addheaders = [("User-Agent", UA),
                         ("Accept", "text/html,application/xhtml+xml"),
                         ("Accept-Language", "hu-HU,hu;q=0.9,en;q=0.5")]
    first = ((lo - 1) // 100) * 100 + 1
    last_req = 0.0
    while first <= hi:
        name = page_name(first)
        path = os.path.join(PAGES, name)
        if os.path.exists(path):
            log("CACHED %s" % name)
            first += 100
            continue
        wait = DELAY - (time.time() - last_req)
        if wait > 0:
            time.sleep(wait)
        last_req = time.time()
        url = BASE + name
        try:
            resp = opener.open(url, timeout=60)
            status, body, final = resp.status, resp.read(), resp.geturl()
        except urllib.error.HTTPError as e:
            log("HTTP %d %s -- STOP" % (e.code, url) if e.code in (403, 429)
                else "HTTP %d %s -- STOP (unexpected)" % (e.code, url))
            sys.exit(2)
        except Exception as e:  # network error: stop, do not hammer
            log("ERROR %s %r -- STOP" % (url, e))
            sys.exit(3)
        html = body.decode("utf-8", errors="replace")
        low = html.lower()
        if any(m in low for m in BLOCK_MARKERS) or "/belepes" in final:
            log("BLOCK? %s final=%s -- STOP" % (url, final))
            with open(path + ".blocked", "wb") as f:
                f.write(body)
            sys.exit(4)
        with open(path, "wb") as f:
            f.write(body)
        log("OK %d %s bytes=%d posts~%d final=%s" % (status, url, len(body), count_posts(html), final))
        first += 100


if __name__ == "__main__":
    main()