From e7fb10200e2a5d8dabf1b68c3ad126335b818a34 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Wed, 7 Oct 2026 10:11:45 +0200 Subject: [PATCH] =?UTF-8?q?R-812=20option=20A=20(hub):=20the=20Proxmox=20p?= =?UTF-8?q?ackage=20set=20=E2=80=94=20candidate,=20operator=20approval,=20?= =?UTF-8?q?System=20page?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Layer pve: never auto-approved; the candidate is the Proxmox userspace set every ring-0 box reports (kernel / boot / firmware names left out); the operator's "Approve Proxmox set" button appears only after 2 healthy night pve steps on every ring-0 box; an approval nudges no box (ring 1 by a signed os_pve_step). 11 §5.10 written (BUILT, unreleased, not yet proven live); §8 step 6 split (userspace §5.10, kernel R-836). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- documentation/architecture/11-os-updates.md | 35 ++++++- hub/CHANGELOG.md | 12 +++ hub/internal/osupdates/pve_test.go | 97 +++++++++++++++++ hub/internal/osupdates/service.go | 109 ++++++++++++++++++-- hub/internal/web/os_updates.go | 9 ++ hub/internal/web/r135_csrf_test.go | 1 + hub/internal/web/system.go | 1 + hub/internal/web/system_test.go | 28 +++++ hub/internal/web/templates/system.html | 5 + 9 files changed, 289 insertions(+), 8 deletions(-) create mode 100644 hub/internal/osupdates/pve_test.go diff --git a/documentation/architecture/11-os-updates.md b/documentation/architecture/11-os-updates.md index da091198..63f5cf70 100644 --- a/documentation/architecture/11-os-updates.md +++ b/documentation/architecture/11-os-updates.md @@ -469,6 +469,38 @@ Decision 88 (R-851): *"Yes, but maybe not indefinitely."* Evidence `audits/os-do --- +### 5.10 The Proxmox package lane — BUILT 2026-10-07, unreleased, not yet proven live (R-812 option A, `09` §3 decision 163) `[FACT]` + +**What it updates:** the host's Proxmox USERSPACE packages (pve-manager, qemu-server, pve-container, lxc-pve, +libpve-*, proxmox-widget-toolkit, the ceph client libraries …) — the 77–78 packages behind on the demo boxes on +2026-10-07. **What it never touches:** a kernel, boot, firmware or microcode name (`HOST_SLOW_RE`: `proxmox-kernel*`, +`proxmox-default-kernel`, `pve-kernel*`, `pve-firmware`, `shim*`, `grub*`, `systemd-boot`, `*-microcode`, `efibootmgr`) +— that is R-836's lane (decision 164). No reboot. + +- **Wrapper** (`felhom-agent/configs/felhom-os-apply`): layer `pve`, lane `slow` only. Origin `Proxmox Debian + Repository` only (R2; a Debian package in a pve plan or simulation is refused); R14 on every name in the plan and the + simulation; no removal (R4); no undo (R5 — put a version back by hand); a NEW package only from `PVE_NEW_ALLOW` + (`proxmox-firewall-data`, the one new userspace package a full upgrade adds on demo-felhom); an appliance only (R12); + authority R3 — a signed `os_pve_step` (op bound to the exact package list, host, time window, nonce) or the root-owned + ring-0 mark (`ring0_slow_lane`). Ring 0 selects `pending-pve`: every installed Proxmox-origin package with a pending + upgrade. The report carries `pve_manager` (pveversion after the step). Tests: `PVELane` (17). +- **Agent** (`internal/osupdate`): ring 0 runs it in the night leg AFTER a healthy host step (an appliance; a Docker + step's outcome does not gate it); ring 1 never in the night leg — only a signed `os_pve_step` (`PVEStepExecutor`, under + the heavy-op gate). Health (`PVEHealthVerdict`) = the host rule (§8.2) + every running container keeps its id (the + household's apps were not restarted) + pveversion reads the pve-manager the step installed. +- **pmxcfs** (`internal/pvegate`): pve-cluster's postinst restarts pmxcfs (the filesystem behind /etc/pve). While a pve + step runs, the agent's own /etc/pve writes WAIT — every non-GET Proxmox API call (`Client.doBody`) and every root CLI + that writes /etc/pve (`ExecRunner.RunStdin`: pct config verbs, pvesm, pveum, felhom-pbs-apply create/reconcile). The + step first waits for writes in flight; after 2 minutes it gives up and does not run (`failed`). Backups and + restore-tests are already out (the heavy-op gate). +- **Hub** (`osupdates`): layer `pve` — never auto-approved. The candidate is the Proxmox userspace set every ring-0 box + agrees on (kernel names left out). The operator's **„Approve Proxmox set"** button on the System page appears only + when every ring-0 box ran the set in `DockerNightsEffective` (2) healthy night steps since it was first seen and none + was unhealthy — „healthy" is the box's own verdict above (no memory-kill check; that is the Docker engine's, R-528). + An approval nudges no box: ring 1 takes the set only by a signed `os_pve_step` (signed per box, as `os_docker_step`). +- **Not yet:** the live proof on demo-felhom (ring 0); the undo runbook (by hand: `apt-get install =` — + Proxmox keeps 30–66 old versions, C2). + ## 6. Risks and edge cases | # | What can go wrong | What the design does | @@ -547,7 +579,8 @@ Each step returns to the operator for go or no-go. not needed:** measured on Tester 2, the agent's first OS leg ran right after the box's FIRST whole-guest backup, 17 min after enrolment (16:24/16:25 UTC: 49 guest + 106 host packages, 110 s + 44 s, healthy). An installer pass would cost ~45 s and run BEFORE any whole-guest backup exists (no undo) — not built. -6. **Slow lane: host kernel and Proxmox packages, with the reboot.** +6. **Slow lane: host kernel and Proxmox packages, with the reboot.** **Split 2026-10-07 (decisions 163–164):** the Proxmox + USERSPACE packages are §5.10 (BUILT, unreleased, no reboot); the kernel lane is R-836 (a spike with reboots first). 7. **Later:** the Proxmox major upgrade (PVE 9 → 10), drilled on ring 0 first. --- diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index ac4277f9..b1e9f32e 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,3 +1,15 @@ +## Unreleased (2026-10-07) — the Proxmox package set (R-812 option A, `09` §3 decision 163) + +- hub (osupdates): layer `pve` — the host's Proxmox userspace set. Never auto-approved; the candidate is what every + ring-0 box reports (kernel / boot / firmware names left out); `PVEStatus` / `ApprovePVE`: approvable only after every + ring-0 box ran the set in 2 healthy night pve steps since it was first seen and none was unhealthy (the box's own + verdict: the host rule + unchanged container ids + pveversion). An approval nudges no box — ring 1 takes it by a signed + `os_pve_step`. Stamped on the minute's tick like the Docker set. Operator undo text names the by-hand route. +- hub (web): „Approve Proxmox set" on the System page (`POST /os/approve-pve`, CSRF-gated like every operator POST), + shown only when the rule allows it. +- Tests: `TestPVE_*` (4), `TestSystemPage_PVEButtonOnlyWhenReady`, the R-135 route list. Red-proofs: + `documentation/audits/day-2026-10-07/B/red-hub-pve-mutations.txt`. + ## v0.141.0 — a reinstall's new backup key no longer overwrites the old one in the escrow; a second off-site Save is refused, not raced; the SMART counters are modelled (R-366, R-31, R-330) (2026-10-07) **Built and deployed 2026-10-07 (morning), on the operator's word (`09` §3 decision 161).** The night session's image build was refused by its permission check at 00:12, so the night window was not used. **Operator action on deploy: none.** diff --git a/hub/internal/osupdates/pve_test.go b/hub/internal/osupdates/pve_test.go new file mode 100644 index 00000000..3ffa6c1d --- /dev/null +++ b/hub/internal/osupdates/pve_test.go @@ -0,0 +1,97 @@ +package osupdates + +import ( + "strings" + "testing" + "time" +) + +// R-812 option A (`09` §3 decision 163): the Proxmox package set — approved only by the operator's button, after every +// ring-0 box ran it in healthy night steps (the host health rule, judged on the box), never a kernel, never pushed to +// ring 1 by the desired state (a signed os_pve_step only). + +var pveSet = []Package{{Name: "pve-manager", Version: "9.2.21", Origin: "Proxmox"}, + {Name: "qemu-server", Version: "9.0.9", Origin: "Proxmox"}} + +func (f *fix) pveNight(t *testing.T, host string, healthy bool) { + t.Helper() + out := "nothing" + if !healthy { + out = "health_failed" + } + f.ingest(t, host, Report{Layer: LayerPVE, Trigger: "night", Mode: "apply", Outcome: out, Healthy: healthy, + Installed: append([]Package{pk("libc6", "x"), {Name: "proxmox-kernel-helper", Version: "9.0.6", Origin: "Proxmox"}}, pveSet...)}) +} + +// COMPANION RED-PROOF (observed): add LayerPVE to Layers (the auto-approved list) → this fails. +func TestPVE_NeverAutoApproved(t *testing.T) { + f := newFix(t) + for i := 0; i < 3; i++ { + f.pveNight(t, "hp", true) + f.pveNight(t, "n100", true) + f.now = f.now.Add(25 * time.Hour) + f.s.Evaluate() + } + if rel, _ := f.s.Store.LatestOSRelease(LayerPVE); rel != nil { + t.Fatalf("a Proxmox set was auto-approved: %+v", rel) + } +} + +// The button works only after two healthy nights on every ring-0 box; the release holds Proxmox userspace only (no +// kernel name, no Debian package) and nudges no box. +// +// COMPANION RED-PROOF (observed): drop the hostSlowRE check for the pve layer in candidate → "proxmox-kernel-helper". +func TestPVE_ApproveNeedsTwoHealthyNightsOnEveryRing0Box(t *testing.T) { + f := newFix(t) + f.pveNight(t, "hp", true) + f.pveNight(t, "n100", true) + if _, err := f.s.ApprovePVE(); err == nil || !strings.Contains(err.Error(), "1 of 2") { + t.Fatalf("approved after one night: %v", err) + } + f.now = f.now.Add(24 * time.Hour) + f.pveNight(t, "hp", true) + f.pveNight(t, "n100", true) + id, err := f.s.ApprovePVE() + if err != nil || !strings.HasPrefix(id, "os-pve-") { + t.Fatalf("%q %v", id, err) + } + rel, _ := f.s.Store.LatestOSRelease(LayerPVE) + if rel.ApprovedBy != "operator" || !strings.Contains(rel.PackagesJSON, "pve-manager") || + strings.Contains(rel.PackagesJSON, "libc6") || strings.Contains(rel.PackagesJSON, "proxmox-kernel") { + t.Fatalf("release %+v", rel) + } + if len(f.bumps) != 0 { + t.Fatalf("a Proxmox approval must nudge no box (ring 1 takes it by a signed job): %v", f.bumps) + } + if b := f.s.DesiredBlock("cust1"); b.Release != nil || b.HostRelease != nil { + t.Fatalf("the Proxmox set leaked into the desired block: %+v", b) + } +} + +func TestPVE_UnhealthyStepBlocksTheButton(t *testing.T) { + f := newFix(t) + f.pveNight(t, "hp", true) + f.pveNight(t, "n100", true) + f.now = f.now.Add(24 * time.Hour) + f.pveNight(t, "hp", false) + f.pveNight(t, "n100", true) + if _, err := f.s.ApprovePVE(); err == nil || !strings.Contains(err.Error(), "health_failed") { + t.Fatalf("an unhealthy Proxmox step did not block: %v", err) + } +} + +// The System page lists the Proxmox candidate beside the Docker one. +func TestPVE_InTheCandidates(t *testing.T) { + f := newFix(t) + f.pveNight(t, "hp", true) + f.pveNight(t, "n100", true) + found := false + for _, c := range f.s.Candidates() { + if c.Layer == LayerPVE && c.Packages == 2 { + found = true + } + } + if !found { + t.Fatalf("no pve candidate with 2 packages: %+v", f.s.Candidates()) + } +} diff --git a/hub/internal/osupdates/service.go b/hub/internal/osupdates/service.go index 806a9215..6eb4ca4c 100644 --- a/hub/internal/osupdates/service.go +++ b/hub/internal/osupdates/service.go @@ -38,13 +38,16 @@ const ( LayerGuest = "guest" LayerHost = "host" LayerDocker = "docker" // the guest's Docker engine set — slow lane, OPERATOR-approved only (`11` §5.8, hub v0.132.0) + // LayerPVE is the host's Proxmox USERSPACE set (R-812 option A, `09` §3 decision 163, `11` §5.10): slow lane, + // OPERATOR-approved only, never a kernel / boot / firmware name, ring 1 only through a signed os_pve_step. + LayerPVE = "pve" ) // Layers lists the layers the hub approves AUTOMATICALLY (the fast lane). var Layers = []string{LayerGuest, LayerHost} // AllLayers adds the Docker engine set (approved only by the operator's button, ApproveDocker). -var AllLayers = []string{LayerGuest, LayerHost, LayerDocker} +var AllLayers = []string{LayerGuest, LayerHost, LayerDocker, LayerPVE} // dockerNames are the six packages of the Docker engine set (the agent wrapper's DOCKER_NAMES). var dockerNames = map[string]bool{"containerd.io": true, "docker-buildx-plugin": true, "docker-ce": true, @@ -241,7 +244,7 @@ func (s *Service) event(customerID, typ, sev, msg string, details any) { func layerOf(r Report) string { switch r.Layer { - case LayerHost, LayerDocker: + case LayerHost, LayerDocker, LayerPVE: return r.Layer } return LayerGuest @@ -270,6 +273,8 @@ func (s *Service) Ingest(hostID string, r Report) error { where = "the box's base system" case LayerDocker: where = "the box's app engine (Docker " + r.DockerEngine + ")" + case LayerPVE: + where = "the box's Proxmox system" } switch r.Outcome { case "applied", "health_failed": @@ -282,6 +287,8 @@ func (s *Service) Ingest(hostID string, r Report) error { undo = "put a host package back by hand from the previous release's snapshot (runbook `os-updates-host-undo.md`)" case LayerDocker: undo = "sign an os_docker_step with undo for the previous engine set (runbook `os-updates-docker-undo.md`)" + case LayerPVE: + undo = "install the previous Proxmox package versions by hand (`apt-get install =`; Proxmox keeps them; there is no automatic undo, R-812)" } s.event(h.CustomerID, EventHealthFailed, "error", fmt.Sprintf("OS update (%s) on %s: NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically; %s.", @@ -310,12 +317,20 @@ func (s *Service) candidate(layer string, ring0 []string) (map[string]Package, e return nil, err } for _, p := range r.Installed { - if layer == LayerDocker { + switch layer { + case LayerDocker: if !dockerNames[p.Name] { continue } - } else if p.Origin != "Debian" && p.Origin != "Debian-Security" { - continue + case LayerPVE: + // the wrapper's inventory names download.proxmox.com "Proxmox"; never a kernel / boot / firmware name + if (p.Origin != "Proxmox" && p.Origin != "Proxmox Debian Repository") || dockerNames[p.Name] || hostSlowRE.MatchString(p.Name) { + continue + } + default: + if p.Origin != "Debian" && p.Origin != "Debian-Security" { + continue + } } if layer == LayerHost && hostSlowRE.MatchString(p.Name) { continue @@ -402,6 +417,9 @@ func (s *Service) Evaluate() ([]Status, error) { if _, err := s.DockerStatus(); err != nil { return out, err } + if _, err := s.PVEStatus(); err != nil { // R-812 option A: stamped on the tick too, like the Docker set + return out, err + } return out, nil } @@ -508,7 +526,8 @@ func (s *Service) Candidates() []Status { out = append(out, st) } d, _ := s.DockerStatus() - return append(out, d) + p, _ := s.PVEStatus() + return append(out, d, p) } // BundleThreshold is the config-bundle alarm's wait (the System page turns the cell red at it). @@ -642,6 +661,82 @@ func oomCheckWaiting(host string, reps []store.OSReport) string { return "" } +// PVEStatus is the Proxmox package set ring 0 runs now and whether the operator's button may approve it (R-812 option +// A): every ring-0 box has run it in DockerNightsEffective healthy night pve steps since it was first seen, and none +// unhealthy. "Healthy" is the box's own verdict on the step — the host rule (every host service active, the guest +// running and healthy, the tunnel running) plus unchanged container ids and pveversion reading the installed +// pve-manager (felhom-agent osupdate.PVEHealthVerdict). No memory-kill check: that is the Docker engine's (R-528). +// Pinned by TestPVE_*. +func (s *Service) PVEStatus() (Status, error) { + st := Status{Layer: LayerPVE} + ring0, err := s.ring0Hosts() + if err != nil || len(ring0) == 0 { + st.Waiting = "no ring-0 box" + return st, err + } + cand, err := s.candidate(LayerPVE, ring0) + if err != nil || len(cand) == 0 { + st.Waiting = "a ring-0 box has not reported a Proxmox step" + return st, err + } + fp, list := fingerprint(LayerPVE, cand) + pj, _ := json.Marshal(list) + first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()) + if err != nil { + return st, err + } + st.Fingerprint, st.FirstSeen, st.Packages = fp, first, len(list) + if rel, _ := s.Store.LatestOSRelease(LayerPVE); rel != nil && rel.Fingerprint == fp { + st.Approved, st.Waiting = rel.ID, "already approved" + return st, nil + } + need := s.DockerNightsEffective() + for _, h := range ring0 { + reps, err := s.Store.OSReportsSince(h, LayerPVE, first) + if err != nil { + return st, err + } + nights := 0 + for _, r := range reps { + if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" { + st.Waiting = fmt.Sprintf("%s reported a Proxmox step %s (healthy=%v) at %s", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339)) + return st, nil + } + if r.Trigger == "night" { + nights++ + } + } + if nights < need { + st.Waiting = fmt.Sprintf("%s has %d of %d healthy night Proxmox step(s) with this set", h, nights, need) + return st, nil + } + } + return st, nil +} + +// ApprovePVE is the operator's button for the Proxmox set (R-812 option A). A ring-1 box takes it only through a signed +// os_pve_step — approval alone installs nothing. +func (s *Service) ApprovePVE() (string, error) { + st, err := s.PVEStatus() + if err != nil { + return "", err + } + if st.Waiting != "" { + return "", fmt.Errorf("osupdates: the Proxmox set cannot be approved yet: %s", st.Waiting) + } + ring0, _ := s.ring0Hosts() + cand, err := s.candidate(LayerPVE, ring0) + if err != nil { + return "", err + } + fp, list := fingerprint(LayerPVE, cand) + if err := s.approve(LayerPVE, fp, list, "operator"); err != nil { + return "", err + } + rel, _ := s.Store.LatestOSRelease(LayerPVE) + return rel.ID, nil +} + // ApproveDocker is the operator's button: it approves the Docker engine set ring 0 runs, only when DockerStatus allows. // A ring-1 box then takes it only through a signed operator job (`11` §5.8) — approval alone installs nothing. func (s *Service) ApproveDocker() (string, error) { @@ -681,7 +776,7 @@ func (s *Service) approve(layer, fp string, list []Package, by string) error { s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)%s", id, layer, by, len(list), fp, mark) s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages)%s.", id, layer, by, len(list), mark), map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp, "test": test}) - if s.Bump != nil && layer != LayerDocker { // a Docker set reaches ring 1 only by a signed job, not the desired state + if s.Bump != nil && layer != LayerDocker && layer != LayerPVE { // the slow-lane sets reach ring 1 only by a signed job hosts, _ := s.Store.ListHosts() for _, h := range hosts { if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled { diff --git a/hub/internal/web/os_updates.go b/hub/internal/web/os_updates.go index ddcb9fbe..f107249a 100644 --- a/hub/internal/web/os_updates.go +++ b/hub/internal/web/os_updates.go @@ -70,6 +70,15 @@ func (s *Server) handleOSAdmin(w http.ResponseWriter, r *http.Request, path stri } id, err := view.ApproveDocker() reply(map[string]string{"release_id": id}, err) + case r.Method == http.MethodPost && path == "/os/approve-pve": + // R-812 option A (`09` §3 decision 163): the operator approves the Proxmox package set ring 0 ran. + view, ok := s.osUpdates.(OSSystemView) + if !ok { + reply(nil, fmt.Errorf("proxmox approval not available")) + return + } + id, err := view.ApprovePVE() + reply(map[string]string{"release_id": id}, err) default: http.Error(w, "not found", http.StatusNotFound) } diff --git a/hub/internal/web/r135_csrf_test.go b/hub/internal/web/r135_csrf_test.go index e5a7cd93..743c3fc6 100644 --- a/hub/internal/web/r135_csrf_test.go +++ b/hub/internal/web/r135_csrf_test.go @@ -59,6 +59,7 @@ var r135PostRoutes = []string{ "/os/enabled/h1", "/os/approve-now", "/os/approve-docker", + "/os/approve-pve", // Not a route: the gate must refuse BEFORE routing, so even an unknown path is 403, never 404. "/no-such-route", } diff --git a/hub/internal/web/system.go b/hub/internal/web/system.go index 3e9d3035..cea759aa 100644 --- a/hub/internal/web/system.go +++ b/hub/internal/web/system.go @@ -57,6 +57,7 @@ type OSSystemView interface { BundleThreshold() time.Duration AgentThreshold() time.Duration ApproveDocker() (string, error) + ApprovePVE() (string, error) // R-812 option A: the Proxmox package set } // agentCell is the "Agent" cell (R-530, hub v0.135.0): the box's agent against the vouched one, how far behind and diff --git a/hub/internal/web/system_test.go b/hub/internal/web/system_test.go index 8fbd197d..0a2627c5 100644 --- a/hub/internal/web/system_test.go +++ b/hub/internal/web/system_test.go @@ -185,3 +185,31 @@ func TestHostsPage_ProxmoxKernelColumn(t *testing.T) { t.Fatalf("hosts column missing:\n%s", b[:min(len(b), 400)]) } } + +// R-812 option A: the "Approve Proxmox set" button appears ONLY when the rule allows it (one render test per branch). +// +// COMPANION RED-PROOF (observed): drop the `(eq .Waiting "")` from the pve button's condition → "button shown after ONE night". +func TestSystemPage_PVEButtonOnlyWhenReady(t *testing.T) { + s, st, svc := systemServer(t) + if strings.Contains(getSystem(t, s), `action="/os/approve-pve"`) { + t.Fatal("button shown with no Proxmox candidate") + } + _ = st.SetOSRing("full-1", 0) + clock := time.Date(2026, 10, 7, 3, 0, 0, 0, time.UTC) + svc.Now = func() time.Time { return clock } + night := func() { + r := osupdates.Report{RunID: time.Now().String(), Layer: "pve", Trigger: "night", Mode: "apply", Outcome: "nothing", + Healthy: true, Installed: []osupdates.Package{{Name: "pve-manager", Version: "9.2.21", Origin: "Proxmox"}}} + if err := svc.Ingest("full-1", r); err != nil { + t.Fatal(err) + } + } + night() + if strings.Contains(getSystem(t, s), `action="/os/approve-pve"`) { + t.Fatal("button shown after ONE night") + } + night() + if !strings.Contains(getSystem(t, s), `action="/os/approve-pve"`) { + t.Fatal("button missing after two healthy nights") + } +} diff --git a/hub/internal/web/templates/system.html b/hub/internal/web/templates/system.html index 5d5e8dcd..b12d8159 100644 --- a/hub/internal/web/templates/system.html +++ b/hub/internal/web/templates/system.html @@ -67,6 +67,11 @@ {{end}} + {{if and (eq .Layer "pve") .Fingerprint (not .Approved) (eq .Waiting "")}} +
+ + +
{{end}} {{end}}