zero-downtime-rollout-real-20260813-123033

Territories

The referee-owned result frozen when the first deployment became durable.

Territory	State	Milestones	Points	Durable at
rollout-onedeepseek-rollout-engineer	verifying	1/2	5	—
rollout-threeglm-rollout-engineer	verifying	4/5	80	—
rollout-twoluna-rollout-engineer	durable	5/5	100	1:58

Agents

Terminal outcomes include activity captured during the post-match drain.

Agent	Model	Outcome	Input	Output	Artifact
deepseek-rollout-engineerrollout-one	deepseek/deepseek-v4-flash-0731	terminated	0	0	—
glm-rollout-engineerrollout-three	z-ai/glm-5.2	completed	87,780	4,589	view
luna-rollout-engineerrollout-two	openai/gpt-5.6-luna	interrupted	0	0	view

Event timeline

47 immutable events. The match clock remains frozen during post-match collection.

0:00

Match state changed

event #0

{
  "elapsed_ms": 0,
  "from": "preparing",
  "kind": "match_state_changed",
  "schema_version": 1,
  "sequence": 0,
  "to": "running"
}

0:00

Competitor state changedrollout-one

event #1

{
  "elapsed_ms": 0,
  "from": "preparing",
  "kind": "competitor_state_changed",
  "reason": "SSH preflight passed",
  "schema_version": 1,
  "sequence": 1,
  "territory": "rollout-one",
  "to": "building"
}

0:00

Competitor state changedrollout-three

event #2

{
  "elapsed_ms": 0,
  "from": "preparing",
  "kind": "competitor_state_changed",
  "reason": "SSH preflight passed",
  "schema_version": 1,
  "sequence": 2,
  "territory": "rollout-three",
  "to": "building"
}

0:00

Competitor state changedrollout-two

event #3

{
  "elapsed_ms": 0,
  "from": "preparing",
  "kind": "competitor_state_changed",
  "reason": "SSH preflight passed",
  "schema_version": 1,
  "sequence": 3,
  "territory": "rollout-two",
  "to": "building"
}

0:00

Territory registeredrollout-one

event #4

{
  "agent": "deepseek-rollout-engineer",
  "class": "rollout-engineer",
  "elapsed_ms": 0,
  "kind": "territory_registered",
  "schema_version": 1,
  "sequence": 4,
  "territory": "rollout-one"
}

0:00

Territory registeredrollout-two

event #5

{
  "agent": "luna-rollout-engineer",
  "class": "rollout-engineer",
  "elapsed_ms": 0,
  "kind": "territory_registered",
  "schema_version": 1,
  "sequence": 5,
  "territory": "rollout-two"
}

0:00

Territory registeredrollout-three

event #6

{
  "agent": "glm-rollout-engineer",
  "class": "rollout-engineer",
  "elapsed_ms": 0,
  "kind": "territory_registered",
  "schema_version": 1,
  "sequence": 6,
  "territory": "rollout-three"
}

0:00

Agent startedrollout-one

event #7

{
  "agent": "deepseek-rollout-engineer",
  "elapsed_ms": 0,
  "kind": "agent_started",
  "model": "deepseek/deepseek-v4-flash-0731",
  "schema_version": 1,
  "sequence": 7,
  "territory": "rollout-one"
}

0:00

Agent startedrollout-two

event #8

{
  "agent": "luna-rollout-engineer",
  "elapsed_ms": 0,
  "kind": "agent_started",
  "model": "openai/gpt-5.6-luna",
  "schema_version": 1,
  "sequence": 8,
  "territory": "rollout-two"
}

0:00

Agent startedrollout-three

event #9

{
  "agent": "glm-rollout-engineer",
  "elapsed_ms": 0,
  "kind": "agent_started",
  "model": "z-ai/glm-5.2",
  "schema_version": 1,
  "sequence": 9,
  "territory": "rollout-three"
}

0:00

Competitor state changedrollout-one

event #10

{
  "elapsed_ms": 251,
  "from": "building",
  "kind": "competitor_state_changed",
  "reason": "milestone verification began",
  "schema_version": 1,
  "sequence": 10,
  "territory": "rollout-one",
  "to": "verifying"
}

0:00

Milestone evaluation startedrollout-one

event #11

{
  "elapsed_ms": 251,
  "kind": "milestone_evaluation_started",
  "milestone": "baseline-ready",
  "schema_version": 1,
  "sequence": 11,
  "territory": "rollout-one"
}

0:00

Competitor state changedrollout-two

event #12

{
  "elapsed_ms": 251,
  "from": "building",
  "kind": "competitor_state_changed",
  "reason": "milestone verification began",
  "schema_version": 1,
  "sequence": 12,
  "territory": "rollout-two",
  "to": "verifying"
}

0:00

Milestone evaluation startedrollout-two

event #13

{
  "elapsed_ms": 251,
  "kind": "milestone_evaluation_started",
  "milestone": "baseline-ready",
  "schema_version": 1,
  "sequence": 13,
  "territory": "rollout-two"
}

0:00

Competitor state changedrollout-three

event #14

{
  "elapsed_ms": 251,
  "from": "building",
  "kind": "competitor_state_changed",
  "reason": "milestone verification began",
  "schema_version": 1,
  "sequence": 14,
  "territory": "rollout-three",
  "to": "verifying"
}

0:00

Milestone evaluation startedrollout-three

event #15

{
  "elapsed_ms": 251,
  "kind": "milestone_evaluation_started",
  "milestone": "baseline-ready",
  "schema_version": 1,
  "sequence": 15,
  "territory": "rollout-three"
}

0:00

Milestone passedrollout-one

event #16

{
  "elapsed_ms": 318,
  "evidence": {
    "baseline": "v1",
    "records": [
      "customer-alpha-73c",
      "customer-beta-a19"
    ]
  },
  "kind": "milestone_passed",
  "milestone": "baseline-ready",
  "points": 5,
  "schema_version": 1,
  "sequence": 16,
  "territory": "rollout-one"
}

0:00

Milestone passedrollout-two

event #17

{
  "elapsed_ms": 318,
  "evidence": {
    "baseline": "v1",
    "records": [
      "customer-alpha-73c",
      "customer-beta-a19"
    ]
  },
  "kind": "milestone_passed",
  "milestone": "baseline-ready",
  "points": 5,
  "schema_version": 1,
  "sequence": 17,
  "territory": "rollout-two"
}

0:00

Milestone passedrollout-three

event #18

{
  "elapsed_ms": 318,
  "evidence": {
    "baseline": "v1",
    "records": [
      "customer-alpha-73c",
      "customer-beta-a19"
    ]
  },
  "kind": "milestone_passed",
  "milestone": "baseline-ready",
  "points": 5,
  "schema_version": 1,
  "sequence": 18,
  "territory": "rollout-three"
}

0:00

Milestone evaluation startedrollout-one

event #19

{
  "elapsed_ms": 318,
  "kind": "milestone_evaluation_started",
  "milestone": "uninterrupted-cutover",
  "schema_version": 1,
  "sequence": 19,
  "territory": "rollout-one"
}

0:00

Milestone evaluation startedrollout-two

event #20

{
  "elapsed_ms": 318,
  "kind": "milestone_evaluation_started",
  "milestone": "uninterrupted-cutover",
  "schema_version": 1,
  "sequence": 20,
  "territory": "rollout-two"
}

0:00

Milestone evaluation startedrollout-three

event #21

{
  "elapsed_ms": 318,
  "kind": "milestone_evaluation_started",
  "milestone": "uninterrupted-cutover",
  "schema_version": 1,
  "sequence": 21,
  "territory": "rollout-three"
}

1:40

Milestone failedrollout-one

event #22

{
  "category": "verification_failed",
  "detail": "public endpoint never cut over to v2",
  "elapsed_ms": 100968,
  "kind": "milestone_failed",
  "milestone": "uninterrupted-cutover",
  "retryable": true,
  "schema_version": 1,
  "sequence": 22,
  "territory": "rollout-one"
}

1:40

Milestone passedrollout-two

event #23

{
  "elapsed_ms": 100968,
  "evidence": {
    "samples": 637,
    "uninterrupted": true,
    "version": "v2"
  },
  "kind": "milestone_passed",
  "milestone": "uninterrupted-cutover",
  "points": 45,
  "schema_version": 1,
  "sequence": 23,
  "territory": "rollout-two"
}

1:40

Milestone passedrollout-three

event #24

{
  "elapsed_ms": 100968,
  "evidence": {
    "samples": 603,
    "uninterrupted": true,
    "version": "v2"
  },
  "kind": "milestone_passed",
  "milestone": "uninterrupted-cutover",
  "points": 45,
  "schema_version": 1,
  "sequence": 24,
  "territory": "rollout-three"
}

1:40

Milestone evaluation startedrollout-two

event #25

{
  "elapsed_ms": 100968,
  "kind": "milestone_evaluation_started",
  "milestone": "write-new",
  "schema_version": 1,
  "sequence": 25,
  "territory": "rollout-two"
}

1:40

Milestone evaluation startedrollout-three

event #26

{
  "elapsed_ms": 100968,
  "kind": "milestone_evaluation_started",
  "milestone": "write-new",
  "schema_version": 1,
  "sequence": 26,
  "territory": "rollout-three"
}

1:41

Milestone passedrollout-two

event #27

{
  "elapsed_ms": 101016,
  "evidence": {
    "record": "rollout-37062b4e6910c573",
    "value": "opaque-404290b9753e8c11ce4a95fc"
  },
  "kind": "milestone_passed",
  "milestone": "write-new",
  "points": 15,
  "schema_version": 1,
  "sequence": 27,
  "territory": "rollout-two"
}

1:41

Milestone passedrollout-three

event #28

{
  "elapsed_ms": 101016,
  "evidence": {
    "record": "rollout-d05c474d64e371bf",
    "value": "opaque-8a04b96b2500cd09edcb84eb"
  },
  "kind": "milestone_passed",
  "milestone": "write-new",
  "points": 15,
  "schema_version": 1,
  "sequence": 28,
  "territory": "rollout-three"
}

1:41

Milestone evaluation startedrollout-two

event #29

{
  "elapsed_ms": 101016,
  "kind": "milestone_evaluation_started",
  "milestone": "v2-restart",
  "schema_version": 1,
  "sequence": 29,
  "territory": "rollout-two"
}

1:41

Milestone evaluation startedrollout-three

event #30

{
  "elapsed_ms": 101016,
  "kind": "milestone_evaluation_started",
  "milestone": "v2-restart",
  "schema_version": 1,
  "sequence": 30,
  "territory": "rollout-three"
}

1:42

Milestone passedrollout-two

event #31

{
  "elapsed_ms": 102182,
  "evidence": {
    "record": "rollout-37062b4e6910c573",
    "v2_restart": true,
    "value": "opaque-404290b9753e8c11ce4a95fc"
  },
  "kind": "milestone_passed",
  "milestone": "v2-restart",
  "points": 15,
  "schema_version": 1,
  "sequence": 31,
  "territory": "rollout-two"
}

1:42

Milestone passedrollout-three

event #32

{
  "elapsed_ms": 102182,
  "evidence": {
    "record": "rollout-d05c474d64e371bf",
    "v2_restart": true,
    "value": "opaque-8a04b96b2500cd09edcb84eb"
  },
  "kind": "milestone_passed",
  "milestone": "v2-restart",
  "points": 15,
  "schema_version": 1,
  "sequence": 32,
  "territory": "rollout-three"
}

1:42

Milestone evaluation startedrollout-two

event #33

{
  "elapsed_ms": 102182,
  "kind": "milestone_evaluation_started",
  "milestone": "host-reboot",
  "schema_version": 1,
  "sequence": 33,
  "territory": "rollout-two"
}

1:42

Milestone evaluation startedrollout-three

event #34

{
  "elapsed_ms": 102182,
  "kind": "milestone_evaluation_started",
  "milestone": "host-reboot",
  "schema_version": 1,
  "sequence": 34,
  "territory": "rollout-three"
}

1:58

Milestone passedrollout-two

event #35

{
  "elapsed_ms": 118272,
  "evidence": {
    "reboot": true,
    "record": "rollout-37062b4e6910c573",
    "value": "opaque-404290b9753e8c11ce4a95fc",
    "version": "v2"
  },
  "kind": "milestone_passed",
  "milestone": "host-reboot",
  "points": 20,
  "schema_version": 1,
  "sequence": 35,
  "territory": "rollout-two"
}

1:58

Competitor state changedrollout-two

event #36

{
  "elapsed_ms": 118272,
  "from": "verifying",
  "kind": "competitor_state_changed",
  "reason": "completion milestone passed",
  "schema_version": 1,
  "sequence": 36,
  "territory": "rollout-two",
  "to": "durable"
}

1:58

Durable deployment completedrollout-two

event #37

{
  "elapsed_ms": 118272,
  "kind": "durable_deployment_completed",
  "schema_version": 1,
  "sequence": 37,
  "territory": "rollout-two"
}

1:58

Match state changed

event #38

{
  "elapsed_ms": 118272,
  "from": "running",
  "kind": "match_state_changed",
  "schema_version": 1,
  "sequence": 38,
  "to": "finished"
}

1:58

Match finished

event #39

{
  "elapsed_ms": 118272,
  "kind": "match_finished",
  "reason": "first durable deployment",
  "schema_version": 1,
  "sequence": 39,
  "winner": "rollout-two"
}

1:58

Post match drain started

event #40

{
  "elapsed_ms": 118272,
  "kind": "post_match_drain_started",
  "pending_agents": 3,
  "schema_version": 1,
  "sequence": 40,
  "timeout_ms": 30000
}

1:58

Agent finishedglm-rollout-engineer

event #41

{
  "agent": "glm-rollout-engineer",
  "detail": "Now I understand the setup. Let me write the v2 service.v2 is up on 8082. Now verify it directly against the contract, including that it serves the existing records from the shared DB.v2 serves all existing records correctly. Let me verify PUT round-trip on v2 with a test record, then confirm the public endpoint is currently on v1 before cutover.v2 fully functional. Public endpoint is on v1. Now the atomic cutover: write `8082` to a temp file and `rename` over the upstream file (atomic on same fs).Public endpoint is now serving v2 with all records preserved. Now verify durability: restart v2 service and confirm everything survives, then confirm services are enabled for reboot.Rollout complete. Summary:\n\n**What was done**\n- Implemented `/opt/rollout/v2.py` — a stateful record service on `127.0.0.1:8082` mirroring v1's SQLite-backed behavior (same `/var/lib/rollout/records.sqlite3`), with `/health`→`ready`, `/version`→`v2`, and `PUT/GET /records/<id>` per contract.\n- Started `rollout-v2.service` (reset its failed state, started, confirmed `active`).\n- Verified v2 directly: health, version, all existing records (`alpha-original`, `beta-original`), 404 for missing, and a PUT round-trip — all correct.\n- Performed atomic cutover by writing `8082` to `upstream.tmp` and `mv`-ing it over `/var/lib/rollout/upstream` (atomic rename on same filesystem). The proxy reads this file per request, so no request ever saw a partial/missing upstream — both v1 (8081) and v2 (8082) stayed running throughout.\n- Verified the public endpoint (`:8080`) immediately served `v2`, `ready`, and all records intact.\n- Did **not** stop or restart `rollout-proxy.service` or `rollout-v1.service` during cutover.\n\n**Durability verified**\n- `systemctl restart rollout-v2.service` → recovered, public endpoint stayed healthy, all records intact.\n- All three services (`rollout-proxy`, `rollout-v1`, `rollout-v2`) are `enabled`, so they come back after a full host reboot. `rollout-v2.service` has `Restart=always`.\n- Records persisted in the SQLite file; v2 reads the same DB, so existing customer state is preserved across restart/reboot.\n\n**Final state**\n- Proxy (`:8080`) → upstream `8082` (v2), running and enabled.\n- v2 (`:8082`) running, enabled, `Restart=always`.\n- v1 (`:8081`) still running (untouched, as required) and enabled.\n- All records present and correct.",
  "elapsed_ms": 118272,
  "kind": "agent_finished",
  "schema_version": 1,
  "sequence": 41,
  "source": "player",
  "success": true
}

1:58

Usage chargedglm-rollout-engineer

event #42

{
  "agent": "glm-rollout-engineer",
  "cost_microusd": 54856,
  "elapsed_ms": 118272,
  "input_tokens": 87780,
  "kind": "usage_charged",
  "output_tokens": 4589,
  "resource_units": 1,
  "schema_version": 1,
  "sequence": 42
}

1:58

Agent interruptedluna-rollout-engineer

event #43

{
  "agent": "luna-rollout-engineer",
  "detail": "agent session was interrupted by the referee's host reboot",
  "elapsed_ms": 118272,
  "kind": "agent_interrupted",
  "schema_version": 1,
  "sequence": 43,
  "source": "arena"
}

1:58

Usage chargedluna-rollout-engineer

event #44

{
  "agent": "luna-rollout-engineer",
  "cost_microusd": null,
  "elapsed_ms": 118272,
  "input_tokens": null,
  "kind": "usage_charged",
  "output_tokens": null,
  "resource_units": 1,
  "schema_version": 1,
  "sequence": 44
}

1:58

Agent terminateddeepseek-rollout-engineer

event #45

{
  "agent": "deepseek-rollout-engineer",
  "elapsed_ms": 118272,
  "kind": "agent_terminated",
  "reason": "post-match drain deadline expired",
  "schema_version": 1,
  "sequence": 45
}

1:58

Post match drain finished

event #46

{
  "captured_agents": 2,
  "elapsed_ms": 118272,
  "kind": "post_match_drain_finished",
  "schema_version": 1,
  "sequence": 46,
  "terminated_agents": 1
}