admin
52b6599d97
gates / gates (push) Successful in 6m36s
Read-only. fetch.py (polite, 5 s apart, 34 pages, all 200), parse.py (own text only; four parser defects found by the controls and fixed), build.py + spotcheck.py (stdlib). tags.csv holds post numbers and tags only - no username, author id, salt or post text in git. FINDINGS.md (2,908 posts, 195 people, 82 beginner threads in 7 clusters), FELHOM.md (catalog gap, hardware fit, 15 website questions, objections, ten ranked recommendations), METHOD.md (controls, spot-check: role and hardware below 85 % after two rounds, the table-driving splits 92.5 % and 97.3 %), REPORT.md. R-990: operator rules on the recommendations. R-991: no buyer-readable hardware floor. Claude-Session: https://claude.ai/code/session_012qRErfCoiTkvDK9N5XHbzb
96 lines
3.4 KiB
Python
96 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Fetch the PROHARDVER! "Home server / hazi szerver epitese" topic pages, politely.
|
|
|
|
Read-only research (forum-research-2026-10-12). Stdlib only.
|
|
- One request at a time, at least DELAY seconds apart (robots.txt asks for 1 s; we use 5).
|
|
- Cookie jar + redirects: the first request answers 302 -> ?_tc=1 (a cookie check).
|
|
- Each page is cached as raw HTML outside git; a cached page is never fetched again.
|
|
- STOPS on 403, 429, a captcha or a login wall (spec section 9.2). No retry around a block.
|
|
|
|
Usage:
|
|
python3 fetch.py FIRST_POST LAST_POST # fetch every 100-post page covering the range
|
|
Pages: hsz_<a>-<b>.html with a = 1 + 100*k, b = a + 99.
|
|
"""
|
|
import http.cookiejar
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
BASE = "https://prohardver.hu/tema/hazi_szerver_epitese/"
|
|
CACHE = os.path.expanduser("~/research-cache/prohardver-hazi-szerver")
|
|
PAGES = os.path.join(CACHE, "pages")
|
|
LOG = os.path.join(CACHE, "fetch.log")
|
|
DELAY = 5.0
|
|
UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36")
|
|
BLOCK_MARKERS = ("captcha", "recaptcha", "hcaptcha")
|
|
|
|
|
|
def log(msg):
|
|
line = time.strftime("%Y-%m-%dT%H:%M:%S%z") + " " + msg
|
|
print(line, flush=True)
|
|
with open(LOG, "a", encoding="utf-8") as f:
|
|
f.write(line + "\n")
|
|
|
|
|
|
def page_name(first):
|
|
return "hsz_%d-%d.html" % (first, first + 99)
|
|
|
|
|
|
def count_posts(html):
|
|
# A rough count for the log only; parse.py does the real work.
|
|
return html.count('data-id="')
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) != 3:
|
|
sys.exit(__doc__)
|
|
lo, hi = int(sys.argv[1]), int(sys.argv[2])
|
|
os.makedirs(PAGES, exist_ok=True)
|
|
jar = http.cookiejar.CookieJar()
|
|
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar))
|
|
opener.addheaders = [("User-Agent", UA),
|
|
("Accept", "text/html,application/xhtml+xml"),
|
|
("Accept-Language", "hu-HU,hu;q=0.9,en;q=0.5")]
|
|
first = ((lo - 1) // 100) * 100 + 1
|
|
last_req = 0.0
|
|
while first <= hi:
|
|
name = page_name(first)
|
|
path = os.path.join(PAGES, name)
|
|
if os.path.exists(path):
|
|
log("CACHED %s" % name)
|
|
first += 100
|
|
continue
|
|
wait = DELAY - (time.time() - last_req)
|
|
if wait > 0:
|
|
time.sleep(wait)
|
|
last_req = time.time()
|
|
url = BASE + name
|
|
try:
|
|
resp = opener.open(url, timeout=60)
|
|
status, body, final = resp.status, resp.read(), resp.geturl()
|
|
except urllib.error.HTTPError as e:
|
|
log("HTTP %d %s -- STOP" % (e.code, url) if e.code in (403, 429)
|
|
else "HTTP %d %s -- STOP (unexpected)" % (e.code, url))
|
|
sys.exit(2)
|
|
except Exception as e: # network error: stop, do not hammer
|
|
log("ERROR %s %r -- STOP" % (url, e))
|
|
sys.exit(3)
|
|
html = body.decode("utf-8", errors="replace")
|
|
low = html.lower()
|
|
if any(m in low for m in BLOCK_MARKERS) or "/belepes" in final:
|
|
log("BLOCK? %s final=%s -- STOP" % (url, final))
|
|
with open(path + ".blocked", "wb") as f:
|
|
f.write(body)
|
|
sys.exit(4)
|
|
with open(path, "wb") as f:
|
|
f.write(body)
|
|
log("OK %d %s bytes=%d posts~%d final=%s" % (status, url, len(body), count_posts(html), final))
|
|
first += 100
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|