wonder fi works
This commit is contained in:
+103
-115
@@ -367,106 +367,42 @@ data:
|
|||||||
widgets:
|
widgets:
|
||||||
# Weather Widget (Időkép)
|
# Weather Widget (Időkép)
|
||||||
- type: custom-api
|
- type: custom-api
|
||||||
title: Időkép – Budapest VIII.
|
title: "Időkép – Budapest VIII."
|
||||||
url: http://idokep-proxy.glance-system.svc.cluster.local:8000/api/idokep?place=Budapest%20VIII.%20ker
|
url: "http://idokep-scraper.glance-system.svc.cluster.local:8000/api"
|
||||||
cache: 15m
|
|
||||||
template: |
|
template: |
|
||||||
<style>
|
{{ $loc := .JSON.String "location.name" }}
|
||||||
.idokep-wrap { display:grid; gap:12px; }
|
{{ $cur := .JSON "current" }}
|
||||||
|
{{ $daily := .JSON.Array "daily" }}
|
||||||
.idokep-top { display:flex; align-items:center; justify-content:space-between; gap:12px; }
|
|
||||||
.idokep-now { display:flex; align-items:center; gap:12px; }
|
|
||||||
.idokep-now img { width:44px; height:44px; }
|
|
||||||
.idokep-temp { font-size:44px; font-weight:700; line-height:1; }
|
|
||||||
.idokep-cond { opacity:.85; font-size:13px; margin-top:2px; }
|
|
||||||
.idokep-place { opacity:.75; font-size:12px; text-align:right; }
|
|
||||||
|
|
||||||
.idokep-hourly {
|
|
||||||
display:grid;
|
|
||||||
grid-template-columns: repeat(6, 1fr);
|
|
||||||
gap:8px;
|
|
||||||
}
|
|
||||||
.idokep-h {
|
|
||||||
background: rgba(255,255,255,0.04);
|
|
||||||
border-radius: 12px;
|
|
||||||
padding: 8px;
|
|
||||||
display:flex;
|
|
||||||
flex-direction:column;
|
|
||||||
align-items:center;
|
|
||||||
justify-content:center;
|
|
||||||
gap:6px;
|
|
||||||
}
|
|
||||||
.idokep-h .t { font-size:12px; opacity:.85; }
|
|
||||||
.idokep-h img { width:22px; height:22px; opacity:.95; }
|
|
||||||
.idokep-h .v { font-weight:600; }
|
|
||||||
.idokep-h .p { font-size:11px; opacity:.70; }
|
|
||||||
|
|
||||||
.idokep-daily { display:grid; gap:10px; margin-top:2px; }
|
|
||||||
.idokep-d {
|
|
||||||
display:grid;
|
|
||||||
grid-template-columns: 42px 22px 1fr auto;
|
|
||||||
align-items:center;
|
|
||||||
gap:10px;
|
|
||||||
}
|
|
||||||
.idokep-d .dow { font-size:12px; opacity:.9; }
|
|
||||||
.idokep-d img { width:18px; height:18px; opacity:.95; }
|
|
||||||
|
|
||||||
.idokep-bar {
|
|
||||||
height: 12px;
|
|
||||||
border-radius: 999px;
|
|
||||||
background: rgba(255,255,255,0.08);
|
|
||||||
position: relative;
|
|
||||||
overflow: hidden;
|
|
||||||
}
|
|
||||||
.idokep-bar > span{
|
|
||||||
position:absolute;
|
|
||||||
top:0; bottom:0;
|
|
||||||
border-radius: 999px;
|
|
||||||
background: linear-gradient(90deg, #7dd3fc, #a7f3d0, #fde68a, #fdba74, #fca5a5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.idokep-mm { font-size:11px; opacity:.7; margin-left:8px; white-space:nowrap; }
|
|
||||||
.idokep-foot { font-size:11px; opacity:.65; margin-top:2px; }
|
|
||||||
.idokep-foot a { opacity:.8; }
|
|
||||||
.idokep-err { opacity:.85; font-size:13px; }
|
|
||||||
.idokep-muted { opacity:.7; font-size:12px; padding: 4px 0; }
|
|
||||||
</style>
|
|
||||||
|
|
||||||
{{ if .JSON.Exists "error" }}
|
|
||||||
<div class="idokep-err">
|
|
||||||
{{ .JSON.String "error" }}<br>
|
|
||||||
Place: <b>{{ .JSON.String "place" }}</b>
|
|
||||||
</div>
|
|
||||||
{{ else }}
|
|
||||||
<div class="idokep-wrap">
|
|
||||||
<div class="idokep-top">
|
|
||||||
<div class="idokep-now">
|
|
||||||
{{ if .JSON.Exists "current.icon_url" }}<img src="{{ .JSON.String "current.icon_url" }}" alt="">{{ end }}
|
|
||||||
<div>
|
|
||||||
{{ if .JSON.Exists "current.temp_c" }}
|
|
||||||
<div class="idokep-temp">{{ .JSON.Float "current.temp_c" | toInt }}°C</div>
|
|
||||||
{{ else }}
|
|
||||||
<div class="idokep-temp">—</div>
|
|
||||||
{{ end }}
|
|
||||||
<div class="idokep-cond">{{ .JSON.String "current.condition_hu" }}</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div class="idokep-place">{{ .JSON.String "place" }}</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{{ $hourly := .JSON.Array "hourly" }}
|
{{ $hourly := .JSON.Array "hourly" }}
|
||||||
|
|
||||||
|
<div class="idokep">
|
||||||
|
<div class="idokep-top">
|
||||||
|
<div class="idokep-top-left">
|
||||||
|
{{ $icon := $cur.String "icon_url" }}
|
||||||
|
{{ if $icon }}
|
||||||
|
<img class="idokep-icon" src="{{ $icon }}" alt="" />
|
||||||
|
{{ end }}
|
||||||
|
<div class="idokep-temp">
|
||||||
|
{{ printf "%.0f" ($cur.Float "temp_c") }}°C
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="idokep-top-right">
|
||||||
|
<div class="idokep-loc">{{ $loc }}</div>
|
||||||
|
<div class="idokep-src">Forrás: <a href="{{ .JSON.String "source.url" }}" target="_blank" rel="noreferrer">Időkép</a></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
{{ if gt (len $hourly) 0 }}
|
{{ if gt (len $hourly) 0 }}
|
||||||
<div class="idokep-hourly">
|
<div class="idokep-hourly">
|
||||||
{{ range $hourly }}
|
{{ range $i, $h := $hourly }}
|
||||||
<div class="idokep-h">
|
<div class="idokep-hour">
|
||||||
<div class="t">{{ .String "hour" }}</div>
|
<div class="idokep-hour-time">{{ $h.String "time" }}</div>
|
||||||
{{ if .Exists "icon_url" }}<img src="{{ .String "icon_url" }}" alt="">{{ end }}
|
{{ $hicon := $h.String "icon_url" }}
|
||||||
{{ if .Exists "temp_c" }}<div class="v">{{ .Float "temp_c" | toInt }}°</div>{{ else }}<div class="v">—</div>{{ end }}
|
{{ if $hicon }}
|
||||||
{{ if .Exists "precip_pct" }}
|
<img class="idokep-hour-icon" src="{{ $hicon }}" alt="" />
|
||||||
<div class="p">{{ .Float "precip_pct" | toInt }}%</div>
|
|
||||||
{{ else }}
|
|
||||||
<div class="p"> </div>
|
|
||||||
{{ end }}
|
{{ end }}
|
||||||
|
<div class="idokep-hour-temp">{{ printf "%.0f" ($h.Float "temp_c") }}°</div>
|
||||||
</div>
|
</div>
|
||||||
{{ end }}
|
{{ end }}
|
||||||
</div>
|
</div>
|
||||||
@@ -474,32 +410,46 @@ data:
|
|||||||
<div class="idokep-muted">No hourly data returned by scraper yet.</div>
|
<div class="idokep-muted">No hourly data returned by scraper yet.</div>
|
||||||
{{ end }}
|
{{ end }}
|
||||||
|
|
||||||
|
{{ if gt (len $daily) 0 }}
|
||||||
|
{{/* compute global min/max across shown days for bar scaling */}}
|
||||||
|
{{ $gmin := 9999.0 }}
|
||||||
|
{{ $gmax := -9999.0 }}
|
||||||
|
{{ range $daily }}
|
||||||
|
{{ $tmin := .Float "tmin_c" }}
|
||||||
|
{{ $tmax := .Float "tmax_c" }}
|
||||||
|
{{ if lt $tmin $gmin }}{{ $gmin = $tmin }}{{ end }}
|
||||||
|
{{ if gt $tmax $gmax }}{{ $gmax = $tmax }}{{ end }}
|
||||||
|
{{ end }}
|
||||||
|
{{ $span := sub $gmax $gmin }}
|
||||||
|
{{ if eq $span 0.0 }}{{ $span = 1.0 }}{{ end }}
|
||||||
|
|
||||||
<div class="idokep-daily">
|
<div class="idokep-daily">
|
||||||
{{ range .JSON.Array "daily" }}
|
{{ range $daily }}
|
||||||
<div class="idokep-d">
|
{{ $tmin := .Float "tmin_c" }}
|
||||||
<div class="dow">{{ .String "dow" }}</div>
|
{{ $tmax := .Float "tmax_c" }}
|
||||||
{{ if .Exists "icon_url" }}<img src="{{ .String "icon_url" }}" alt="">{{ end }}
|
{{ $left := mul (div (sub $tmin $gmin) $span) 100.0 }}
|
||||||
<div class="idokep-bar">
|
{{ $wid := mul (div (sub $tmax $tmin) $span) 100.0 }}
|
||||||
{{ if and (.Exists "bar_left_pct") (.Exists "bar_width_pct") }}
|
|
||||||
<span style="left: {{ .Float "bar_left_pct" }}%; width: {{ .Float "bar_width_pct" }}%; opacity:.95;"></span>
|
<div class="idokep-row">
|
||||||
{{ else }}
|
<div class="idokep-dow">{{ .String "dow" }}</div>
|
||||||
<span style="left: 0%; width: 100%; opacity:.6;"></span>
|
|
||||||
{{ end }}
|
{{ $dicon := .String "icon_url" }}
|
||||||
</div>
|
<div class="idokep-dayicon">
|
||||||
<div style="display:flex; align-items:center; gap:8px;">
|
{{ if $dicon }}<img src="{{ $dicon }}" alt="" />{{ end }}
|
||||||
{{ if .Exists "tmin_c" }}<div style="font-weight:600;">{{ .Float "tmin_c" | toInt }}°</div>{{ else }}<div style="font-weight:600;">—</div>{{ end }}
|
|
||||||
{{ if .Exists "tmax_c" }}<div style="opacity:.85;">{{ .Float "tmax_c" | toInt }}°</div>{{ else }}<div style="opacity:.85;">—</div>{{ end }}
|
|
||||||
{{ if .Exists "prec_mm" }}<div class="idokep-mm">{{ .Float "prec_mm" }}mm</div>{{ end }}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{{ end }}
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="idokep-foot">
|
<div class="idokep-bar">
|
||||||
Forrás: <a href="{{ .JSON.String "source.url" }}" target="_blank">{{ .JSON.String "source.name" }}</a>
|
<div class="idokep-bar-track"></div>
|
||||||
|
<div class="idokep-bar-fill" style="left: {{ printf "%.1f" $left }}%; width: {{ printf "%.1f" $wid }}%;"></div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="idokep-min">{{ printf "%.0f" $tmin }}°</div>
|
||||||
|
<div class="idokep-max">{{ printf "%.0f" $tmax }}°</div>
|
||||||
</div>
|
</div>
|
||||||
{{ end }}
|
{{ end }}
|
||||||
|
</div>
|
||||||
|
{{ end }}
|
||||||
|
</div>
|
||||||
|
|
||||||
# Calendar Widget
|
# Calendar Widget
|
||||||
- type: calendar
|
- type: calendar
|
||||||
@@ -1683,6 +1633,44 @@ data:
|
|||||||
color: #7ed9e6 !important;
|
color: #7ed9e6 !important;
|
||||||
border-color: #5ac8d8 !important;
|
border-color: #5ac8d8 !important;
|
||||||
}
|
}
|
||||||
|
/* =========================================================================
|
||||||
|
Időkép custom-api widget
|
||||||
|
========================================================================= */
|
||||||
|
.idokep { display: flex; flex-direction: column; gap: 10px; }
|
||||||
|
|
||||||
|
.idokep-top { display: flex; justify-content: space-between; align-items: center; gap: 10px; }
|
||||||
|
.idokep-top-left { display: flex; align-items: center; gap: 10px; }
|
||||||
|
.idokep-icon { width: 42px; height: 42px; opacity: 0.95; }
|
||||||
|
.idokep-temp { font-size: 42px; font-weight: 700; letter-spacing: 0.5px; line-height: 1; }
|
||||||
|
|
||||||
|
.idokep-top-right { text-align: right; }
|
||||||
|
.idokep-loc { opacity: 0.85; font-weight: 600; }
|
||||||
|
.idokep-src { opacity: 0.6; font-size: 12px; margin-top: 2px; }
|
||||||
|
.idokep-src a { opacity: 0.9; }
|
||||||
|
|
||||||
|
.idokep-hourly { display: flex; gap: 10px; padding-top: 4px; }
|
||||||
|
.idokep-hour { width: 54px; display: flex; flex-direction: column; align-items: center; gap: 6px; opacity: 0.9; }
|
||||||
|
.idokep-hour-time { font-size: 12px; opacity: 0.65; }
|
||||||
|
.idokep-hour-icon { width: 26px; height: 26px; }
|
||||||
|
.idokep-hour-temp { font-weight: 700; }
|
||||||
|
|
||||||
|
.idokep-muted { opacity: 0.6; font-size: 12px; padding: 4px 0; }
|
||||||
|
|
||||||
|
.idokep-daily { display: flex; flex-direction: column; gap: 8px; margin-top: 2px; }
|
||||||
|
.idokep-row { display: grid; grid-template-columns: 28px 26px 1fr 36px 36px; gap: 10px; align-items: center; }
|
||||||
|
.idokep-dow { opacity: 0.7; font-weight: 700; }
|
||||||
|
.idokep-dayicon img { width: 22px; height: 22px; opacity: 0.95; }
|
||||||
|
|
||||||
|
.idokep-bar { position: relative; height: 10px; }
|
||||||
|
.idokep-bar-track { position: absolute; inset: 0; border-radius: 999px; background: rgba(255,255,255,0.10); }
|
||||||
|
.idokep-bar-fill {
|
||||||
|
position: absolute; top: 0; bottom: 0;
|
||||||
|
border-radius: 999px;
|
||||||
|
background: linear-gradient(90deg, #8BE28B 0%, #FFE17A 50%, #FF9B7A 100%);
|
||||||
|
box-shadow: 0 0 0 1px rgba(0,0,0,0.08) inset;
|
||||||
|
}
|
||||||
|
|
||||||
|
.idokep-min, .idokep-max { text-align: right; font-weight: 700; opacity: 0.8; }
|
||||||
|
|
||||||
---
|
---
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
|
|||||||
+184
-245
@@ -1,283 +1,222 @@
|
|||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: ConfigMap
|
kind: Namespace
|
||||||
metadata:
|
metadata:
|
||||||
name: idokep-proxy
|
name: glance-system
|
||||||
namespace: glance-system
|
|
||||||
data:
|
|
||||||
app.py: |
|
|
||||||
import os
|
|
||||||
import time
|
|
||||||
import re
|
|
||||||
from typing import Any, Dict, Optional, Tuple, List
|
|
||||||
|
|
||||||
import requests
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
from fastapi import FastAPI, Query, Response
|
|
||||||
from fastapi.responses import JSONResponse
|
|
||||||
|
|
||||||
from prometheus_client import Gauge, Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST
|
|
||||||
|
|
||||||
app = FastAPI()
|
|
||||||
|
|
||||||
IDOKEP_BASE = "https://www.idokep.hu"
|
|
||||||
DEFAULT_PLACE = os.getenv("IDOKEP_PLACE", "Budapest VIII. ker")
|
|
||||||
USER_AGENT = os.getenv(
|
|
||||||
"IDOKEP_UA",
|
|
||||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
|
|
||||||
)
|
|
||||||
|
|
||||||
CACHE_TTL_SEC = int(os.getenv("CACHE_TTL_SEC", "900")) # 15 minutes
|
|
||||||
_cache: Dict[str, Tuple[float, Dict[str, Any]]] = {}
|
|
||||||
|
|
||||||
# --- Prometheus metrics (low-cardinality, place as label) ---
|
|
||||||
SCRAPES_TOTAL = Counter("idokep_scrapes_total", "Total Időkép scrapes", ["place", "status"])
|
|
||||||
SCRAPE_SECONDS = Histogram("idokep_scrape_seconds", "Időkép scrape duration in seconds", ["place"])
|
|
||||||
CURRENT_TEMP_C = Gauge("idokep_current_temp_c", "Current temperature in Celsius", ["place"])
|
|
||||||
DAILY_TMIN_C = Gauge("idokep_daily_tmin_c", "Daily minimum temperature in Celsius", ["place", "dow"])
|
|
||||||
DAILY_TMAX_C = Gauge("idokep_daily_tmax_c", "Daily maximum temperature in Celsius", ["place", "dow"])
|
|
||||||
DAILY_PREC_MM = Gauge("idokep_daily_precip_mm", "Daily precipitation in mm", ["place", "dow"])
|
|
||||||
|
|
||||||
def _num(s: str) -> Optional[float]:
|
|
||||||
if s is None:
|
|
||||||
return None
|
|
||||||
m = re.search(r"-?\d+(\.\d+)?", s.replace(",", "."))
|
|
||||||
return float(m.group(0)) if m else None
|
|
||||||
|
|
||||||
def _abs_url(url: Optional[str]) -> Optional[str]:
|
|
||||||
if not url:
|
|
||||||
return None
|
|
||||||
if url.startswith("//"):
|
|
||||||
return "https:" + url
|
|
||||||
if url.startswith("/"):
|
|
||||||
return IDOKEP_BASE + url
|
|
||||||
return url
|
|
||||||
|
|
||||||
def _pick_text(el) -> Optional[str]:
|
|
||||||
if not el:
|
|
||||||
return None
|
|
||||||
return el.get_text(" ", strip=True)
|
|
||||||
|
|
||||||
def _fetch_place_html(place: str) -> str:
|
|
||||||
place_path = requests.utils.requote_uri(place)
|
|
||||||
url = f"{IDOKEP_BASE}/idojaras/{place_path}"
|
|
||||||
r = requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=20)
|
|
||||||
r.raise_for_status()
|
|
||||||
return r.text
|
|
||||||
|
|
||||||
def _parse_idokep(html: str, place: str) -> Dict[str, Any]:
|
|
||||||
soup = BeautifulSoup(html, "lxml")
|
|
||||||
|
|
||||||
# CURRENT
|
|
||||||
temp_el = soup.select_one(".current-temperature")
|
|
||||||
temp_c = _num(_pick_text(temp_el) or "")
|
|
||||||
|
|
||||||
icon_el = soup.select_one(".forecast-bigicon")
|
|
||||||
icon_url = _abs_url(icon_el.get("src") if icon_el else None)
|
|
||||||
|
|
||||||
cond_hu_el = soup.select_one(".weather-short-desc")
|
|
||||||
condition_hu = _pick_text(cond_hu_el)
|
|
||||||
|
|
||||||
# HOURLY (first 6)
|
|
||||||
hourly_cards = soup.select(".new-hourly-forecast-card")
|
|
||||||
hourly: List[Dict[str, Any]] = []
|
|
||||||
for card in hourly_cards[:6]:
|
|
||||||
hour_txt = _pick_text(card.select_one(".new-hourly-forecast-hour"))
|
|
||||||
|
|
||||||
htemp_c = _num(_pick_text(card.select_one(".tempValue .hover-over")) or "")
|
|
||||||
|
|
||||||
hicon_url = _abs_url((card.select_one(".forecast-icon") or {}).get("src")) if card.select_one(".forecast-icon") else None
|
|
||||||
|
|
||||||
hprec_pct = _num(_pick_text(card.select_one(".hourly-rain-chance a")) or "")
|
|
||||||
|
|
||||||
hourly.append(
|
|
||||||
{
|
|
||||||
"hour": hour_txt, # e.g. "15:00"
|
|
||||||
"temp_c": htemp_c,
|
|
||||||
"icon_url": hicon_url,
|
|
||||||
"precip_pct": hprec_pct,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# DAILY (next 5, skip first like your HA template did)
|
|
||||||
daily_cols = soup.select(".dailyForecastCol")
|
|
||||||
cols = daily_cols[1:] if len(daily_cols) >= 2 else daily_cols
|
|
||||||
|
|
||||||
daily_raw: List[Dict[str, Any]] = []
|
|
||||||
for col in cols[:5]:
|
|
||||||
dow = _pick_text(col.select_one(".dfDay"))
|
|
||||||
daynum = _pick_text(col.select_one(".dfDayNum"))
|
|
||||||
|
|
||||||
dicon_url = _abs_url((col.select_one(".forecast") or {}).get("src")) if col.select_one(".forecast") else None
|
|
||||||
|
|
||||||
# various layouts: try a few
|
|
||||||
max_el = col.select_one(".max a") or col.select_one(".min-max-close a:nth-child(1)")
|
|
||||||
min_el = col.select_one(".min a") or col.select_one(".min-max-close a:nth-child(2)")
|
|
||||||
tmax_c = _num(_pick_text(max_el) or "")
|
|
||||||
tmin_c = _num(_pick_text(min_el) or "")
|
|
||||||
|
|
||||||
prec_mm = _num(_pick_text(col.select_one(".mm")) or "")
|
|
||||||
|
|
||||||
daily_raw.append(
|
|
||||||
{
|
|
||||||
"dow": dow, # e.g. "Sze"
|
|
||||||
"daynum": daynum, # e.g. "14"
|
|
||||||
"tmax_c": tmax_c,
|
|
||||||
"tmin_c": tmin_c,
|
|
||||||
"prec_mm": prec_mm,
|
|
||||||
"icon_url": dicon_url,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# Compute weekly min/max for HA-like bars (left/width)
|
|
||||||
mins = [d["tmin_c"] for d in daily_raw if d.get("tmin_c") is not None]
|
|
||||||
maxs = [d["tmax_c"] for d in daily_raw if d.get("tmax_c") is not None]
|
|
||||||
week_min = min(mins) if mins else None
|
|
||||||
week_max = max(maxs) if maxs else None
|
|
||||||
denom = (week_max - week_min) if (week_min is not None and week_max is not None and week_max != week_min) else None
|
|
||||||
|
|
||||||
daily: List[Dict[str, Any]] = []
|
|
||||||
for d in daily_raw:
|
|
||||||
left = None
|
|
||||||
width = None
|
|
||||||
if denom is not None and d.get("tmin_c") is not None and d.get("tmax_c") is not None:
|
|
||||||
left = ((d["tmin_c"] - week_min) / denom) * 100.0
|
|
||||||
width = ((d["tmax_c"] - d["tmin_c"]) / denom) * 100.0
|
|
||||||
# clamp for safety
|
|
||||||
left = max(0.0, min(100.0, left))
|
|
||||||
width = max(1.0, min(100.0, width))
|
|
||||||
d2 = dict(d)
|
|
||||||
d2["bar_left_pct"] = left
|
|
||||||
d2["bar_width_pct"] = width
|
|
||||||
daily.append(d2)
|
|
||||||
|
|
||||||
return {
|
|
||||||
"source": {
|
|
||||||
"name": "Időkép",
|
|
||||||
"url": f"{IDOKEP_BASE}/idojaras/{requests.utils.requote_uri(place)}",
|
|
||||||
},
|
|
||||||
"place": place,
|
|
||||||
"current": {
|
|
||||||
"temp_c": temp_c,
|
|
||||||
"condition_hu": condition_hu,
|
|
||||||
"icon_url": icon_url,
|
|
||||||
},
|
|
||||||
"hourly": hourly,
|
|
||||||
"daily": daily,
|
|
||||||
"weekly": {
|
|
||||||
"tmin_c": week_min,
|
|
||||||
"tmax_c": week_max,
|
|
||||||
},
|
|
||||||
"fetched_at_unix": int(time.time()),
|
|
||||||
}
|
|
||||||
|
|
||||||
@app.get("/api/idokep")
|
|
||||||
def api_idokep(place: str = Query(default=DEFAULT_PLACE, description="Időkép place name as in /idojaras/<place>")):
|
|
||||||
now = time.time()
|
|
||||||
|
|
||||||
cached = _cache.get(place)
|
|
||||||
if cached and cached[0] > now:
|
|
||||||
return JSONResponse(cached[1])
|
|
||||||
|
|
||||||
with SCRAPE_SECONDS.labels(place=place).time():
|
|
||||||
try:
|
|
||||||
html = _fetch_place_html(place)
|
|
||||||
payload = _parse_idokep(html, place)
|
|
||||||
_cache[place] = (now + CACHE_TTL_SEC, payload)
|
|
||||||
|
|
||||||
# update metrics (best-effort)
|
|
||||||
t = payload.get("current", {}).get("temp_c")
|
|
||||||
if t is not None:
|
|
||||||
CURRENT_TEMP_C.labels(place=place).set(float(t))
|
|
||||||
|
|
||||||
for d in payload.get("daily", []):
|
|
||||||
dow = d.get("dow") or "?"
|
|
||||||
if d.get("tmin_c") is not None:
|
|
||||||
DAILY_TMIN_C.labels(place=place, dow=dow).set(float(d["tmin_c"]))
|
|
||||||
if d.get("tmax_c") is not None:
|
|
||||||
DAILY_TMAX_C.labels(place=place, dow=dow).set(float(d["tmax_c"]))
|
|
||||||
if d.get("prec_mm") is not None:
|
|
||||||
DAILY_PREC_MM.labels(place=place, dow=dow).set(float(d["prec_mm"]))
|
|
||||||
|
|
||||||
SCRAPES_TOTAL.labels(place=place, status="ok").inc()
|
|
||||||
return JSONResponse(payload)
|
|
||||||
|
|
||||||
except Exception:
|
|
||||||
SCRAPES_TOTAL.labels(place=place, status="error").inc()
|
|
||||||
# return a structured error Glance can show
|
|
||||||
return JSONResponse(
|
|
||||||
{
|
|
||||||
"place": place,
|
|
||||||
"error": "Failed to scrape Időkép. Check the place string or Időkép page layout changes.",
|
|
||||||
"fetched_at_unix": int(time.time()),
|
|
||||||
},
|
|
||||||
status_code=502,
|
|
||||||
)
|
|
||||||
|
|
||||||
@app.get("/metrics")
|
|
||||||
def metrics():
|
|
||||||
return Response(generate_latest(), media_type=CONTENT_TYPE_LATEST)
|
|
||||||
|
|
||||||
---
|
---
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
metadata:
|
metadata:
|
||||||
name: idokep-proxy
|
name: idokep-scraper
|
||||||
namespace: glance-system
|
namespace: glance-system
|
||||||
spec:
|
spec:
|
||||||
replicas: 1
|
replicas: 1
|
||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
app: idokep-proxy
|
app: idokep-scraper
|
||||||
template:
|
template:
|
||||||
metadata:
|
metadata:
|
||||||
labels:
|
labels:
|
||||||
app: idokep-proxy
|
app: idokep-scraper
|
||||||
annotations:
|
|
||||||
prometheus.io/scrape: "true"
|
|
||||||
prometheus.io/port: "8000"
|
|
||||||
prometheus.io/path: "/metrics"
|
|
||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: idokep-proxy
|
- name: idokep-scraper
|
||||||
image: python:3.12-slim
|
image: python:3.12-slim
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
env:
|
||||||
|
- name: IDOKEP_URL
|
||||||
|
value: "https://www.idokep.hu/idojaras/Budapest%20VIII.%20ker"
|
||||||
|
- name: PLACE_NAME
|
||||||
|
value: "Budapest VIII. ker"
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 8000
|
- containerPort: 8000
|
||||||
env:
|
command: ["/bin/sh", "-lc"]
|
||||||
- name: IDOKEP_PLACE
|
args:
|
||||||
value: "Budapest VIII. ker"
|
- |
|
||||||
- name: CACHE_TTL_SEC
|
pip install --no-cache-dir fastapi uvicorn requests beautifulsoup4 prometheus-client &&
|
||||||
value: "900"
|
python -c "import uvicorn; uvicorn.run('app:APP', host='0.0.0.0', port=8000)"
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 25m
|
|
||||||
memory: 128Mi
|
|
||||||
limits:
|
|
||||||
memory: 256Mi
|
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- name: app
|
- name: app
|
||||||
mountPath: /app
|
mountPath: /app
|
||||||
workingDir: /app
|
workingDir: /app
|
||||||
command: ["/bin/sh","-lc"]
|
|
||||||
args:
|
|
||||||
- |
|
|
||||||
pip install --no-cache-dir fastapi uvicorn requests beautifulsoup4 lxml prometheus_client &&
|
|
||||||
uvicorn app:app --host 0.0.0.0 --port 8000
|
|
||||||
volumes:
|
volumes:
|
||||||
- name: app
|
- name: app
|
||||||
configMap:
|
configMap:
|
||||||
name: idokep-proxy
|
name: idokep-scraper-app
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: idokep-scraper-app
|
||||||
|
namespace: glance-system
|
||||||
|
data:
|
||||||
|
app.py: |
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from typing import List, Dict, Any, Optional
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from fastapi import FastAPI, Response
|
||||||
|
from prometheus_client import Counter, Histogram, Gauge, generate_latest, CONTENT_TYPE_LATEST
|
||||||
|
|
||||||
|
APP = FastAPI()
|
||||||
|
|
||||||
|
IDOKEP_URL = os.getenv(
|
||||||
|
"IDOKEP_URL",
|
||||||
|
"https://www.idokep.hu/idojaras/Budapest%20VIII.%20ker",
|
||||||
|
)
|
||||||
|
PLACE_NAME = os.getenv("PLACE_NAME", "Budapest VIII. ker")
|
||||||
|
SOURCE_NAME = "Időkép"
|
||||||
|
|
||||||
|
UA = os.getenv(
|
||||||
|
"USER_AGENT",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome Safari",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Prometheus metrics (optional)
|
||||||
|
SCRAPES = Counter("idokep_scrapes_total", "Total Időkép scrapes", ["place", "status"])
|
||||||
|
SCRAPE_SECONDS = Histogram("idokep_scrape_seconds", "Időkép scrape duration in seconds", ["place"])
|
||||||
|
CURRENT_TEMP = Gauge("idokep_current_temp_c", "Current temperature in Celsius", ["place"])
|
||||||
|
DAILY_TMIN = Gauge("idokep_daily_tmin_c", "Daily minimum temperature in Celsius", ["place", "dow"])
|
||||||
|
DAILY_TMAX = Gauge("idokep_daily_tmax_c", "Daily maximum temperature in Celsius", ["place", "dow"])
|
||||||
|
HOURLY_TEMP = Gauge("idokep_hourly_temp_c", "Hourly temperature in Celsius", ["place", "time"])
|
||||||
|
|
||||||
|
|
||||||
|
def _abs_url(maybe_relative: Optional[str]) -> Optional[str]:
|
||||||
|
if not maybe_relative:
|
||||||
|
return None
|
||||||
|
if maybe_relative.startswith("http://") or maybe_relative.startswith("https://"):
|
||||||
|
return maybe_relative
|
||||||
|
# Időkép uses /assets/... paths
|
||||||
|
return "https://www.idokep.hu" + maybe_relative
|
||||||
|
|
||||||
|
|
||||||
|
def _to_int_temp(s: str) -> Optional[float]:
|
||||||
|
if not s:
|
||||||
|
return None
|
||||||
|
s = s.strip().replace("˚C", "").replace("°C", "").replace("°", "")
|
||||||
|
try:
|
||||||
|
return float(s)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def scrape() -> Dict[str, Any]:
|
||||||
|
headers = {"User-Agent": UA}
|
||||||
|
r = requests.get(IDOKEP_URL, headers=headers, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
|
||||||
|
soup = BeautifulSoup(r.text, "html.parser")
|
||||||
|
|
||||||
|
# Current
|
||||||
|
cur_temp_el = soup.select_one(".current-temperature")
|
||||||
|
cur_cond_el = soup.select_one(".current-weather")
|
||||||
|
cur_icon_el = soup.select_one(".forecast-bigicon")
|
||||||
|
|
||||||
|
cur_temp = _to_int_temp(cur_temp_el.get_text(strip=True) if cur_temp_el else "")
|
||||||
|
cur_cond = cur_cond_el.get_text(strip=True) if cur_cond_el else ""
|
||||||
|
cur_icon = _abs_url(cur_icon_el.get("src") if cur_icon_el else None)
|
||||||
|
|
||||||
|
# Hourly cards (the block you highlighted in devtools: .ik.hourly-forecast-card)
|
||||||
|
hourly: List[Dict[str, Any]] = []
|
||||||
|
for card in soup.select(".ik.hourly-forecast-card")[:8]:
|
||||||
|
t_el = card.select_one(".ik.hourly-forecast-hour")
|
||||||
|
temp_el = card.select_one(".ik.temperature-circled")
|
||||||
|
icon_el = card.select_one("img.ik.forecast-icon")
|
||||||
|
|
||||||
|
t = t_el.get_text(strip=True) if t_el else ""
|
||||||
|
temp = _to_int_temp(temp_el.get_text(strip=True) if temp_el else "")
|
||||||
|
icon = _abs_url(icon_el.get("src") if icon_el else None)
|
||||||
|
|
||||||
|
if t and temp is not None:
|
||||||
|
hourly.append(
|
||||||
|
{
|
||||||
|
"time": t, # e.g. "18:00"
|
||||||
|
"temp_c": temp, # e.g. -2
|
||||||
|
"icon_url": icon, # absolute URL
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Daily columns (bottom forecast table: .ik.daily-forecast-container .ik.dailyForecastCol)
|
||||||
|
daily: List[Dict[str, Any]] = []
|
||||||
|
for col in soup.select(".ik.daily-forecast-container .ik.dailyForecastCol")[:7]:
|
||||||
|
dow_el = col.select_one(".ik.dfDay")
|
||||||
|
icon_el = col.select_one("img.ik.forecast-icon")
|
||||||
|
tmax_el = col.select_one("div.ik.max")
|
||||||
|
tmin_el = col.select_one("div.ik.min")
|
||||||
|
|
||||||
|
dow = dow_el.get_text(strip=True) if dow_el else ""
|
||||||
|
icon = _abs_url(icon_el.get("src") if icon_el else None)
|
||||||
|
tmax = _to_int_temp(tmax_el.get_text(strip=True) if tmax_el else "")
|
||||||
|
tmin = _to_int_temp(tmin_el.get_text(strip=True) if tmin_el else "")
|
||||||
|
|
||||||
|
# Keep only rows that look valid
|
||||||
|
if dow and (tmin is not None) and (tmax is not None):
|
||||||
|
daily.append(
|
||||||
|
{
|
||||||
|
"dow": dow, # e.g. "Cs", "P", "Sz"
|
||||||
|
"tmin_c": tmin,
|
||||||
|
"tmax_c": tmax,
|
||||||
|
"icon_url": icon,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Limit to 5 days for your widget
|
||||||
|
daily = daily[:5]
|
||||||
|
|
||||||
|
return {
|
||||||
|
"source": {"name": SOURCE_NAME, "url": IDOKEP_URL},
|
||||||
|
"location": {"name": PLACE_NAME},
|
||||||
|
"current": {"temp_c": cur_temp, "condition": cur_cond, "icon_url": cur_icon},
|
||||||
|
"hourly": hourly,
|
||||||
|
"daily": daily,
|
||||||
|
"fetched_at_unix": int(time.time()),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@APP.get("/api")
|
||||||
|
def api():
|
||||||
|
status = "ok"
|
||||||
|
with SCRAPE_SECONDS.labels(place=PLACE_NAME).time():
|
||||||
|
try:
|
||||||
|
data = scrape()
|
||||||
|
except Exception:
|
||||||
|
status = "error"
|
||||||
|
SCRAPES.labels(place=PLACE_NAME, status=status).inc()
|
||||||
|
raise
|
||||||
|
|
||||||
|
SCRAPES.labels(place=PLACE_NAME, status=status).inc()
|
||||||
|
|
||||||
|
# Update Prometheus gauges (best-effort)
|
||||||
|
try:
|
||||||
|
if data.get("current", {}).get("temp_c") is not None:
|
||||||
|
CURRENT_TEMP.labels(place=PLACE_NAME).set(float(data["current"]["temp_c"]))
|
||||||
|
for d in data.get("daily", []):
|
||||||
|
DAILY_TMIN.labels(place=PLACE_NAME, dow=d["dow"]).set(float(d["tmin_c"]))
|
||||||
|
DAILY_TMAX.labels(place=PLACE_NAME, dow=d["dow"]).set(float(d["tmax_c"]))
|
||||||
|
for h in data.get("hourly", []):
|
||||||
|
HOURLY_TEMP.labels(place=PLACE_NAME, time=h["time"]).set(float(h["temp_c"]))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# IMPORTANT: force JSON content-type so Glance exposes `.JSON`
|
||||||
|
import json
|
||||||
|
return Response(content=json.dumps(data, ensure_ascii=False), media_type="application/json; charset=utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
@APP.get("/metrics")
|
||||||
|
def metrics():
|
||||||
|
return Response(generate_latest(), media_type=CONTENT_TYPE_LATEST)
|
||||||
---
|
---
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: Service
|
kind: Service
|
||||||
metadata:
|
metadata:
|
||||||
name: idokep-proxy
|
name: idokep-scraper
|
||||||
namespace: glance-system
|
namespace: glance-system
|
||||||
spec:
|
spec:
|
||||||
selector:
|
selector:
|
||||||
app: idokep-proxy
|
app: idokep-scraper
|
||||||
ports:
|
ports:
|
||||||
- name: http
|
- name: http
|
||||||
port: 8000
|
port: 8000
|
||||||
targetPort: 8000
|
targetPort: 8000
|
||||||
type: ClusterIP
|
|
||||||
Reference in New Issue
Block a user