feat: log aggregation — the machine ships its logs to Loki
The gameplay logs sat on /dump on the machine, readable only over SSH. Now they land in Loki for query and tail: - k8s/loki/loki.yaml: Loki 3.4.4 monolithic on the home cluster's monitoring namespace — filesystem storage on CephFS, 30-day retention, traefik ingress at loki.k8s-home.lqx.net with a cert-manager cert, Grafana datasource auto-registered via the sidecar. reject_old_samples_max_age is a year: the machines backfill their history on first ship. - shipper/: logshipper, a ~300-line static Go service for the machine. Tails /dump/mode.log and /dump/switch.log, batches, pushes to Loki's push API. Per-file offsets are saved only after a successful push (down Loki = retry-then-catch-up, never loss); first run starts at each file's end, SHIP_FROM_START=1 ships history; truncation is noticed. Lines keep their own RFC 3339 stamp as the Loki timestamp; unstamped lines (the boot banner) inherit the next stamped line's, so a stream never starts at "now" and jumps back. After a backfill it calls /flush — old entries are invisible in the ingester's head until they reach the store (found the hard way: Loki answers 204 to an old entry on a new stream and then never serves it). Labels: job=pinball, machine, source, game (from the boot line). /etc/init.d/log_shipper runs it at boot. Deployed: Loki is live and verified (query through the ingress). The shipper is verified end-to-end against the cluster from this Mac; the machine was unreachable at deploy time — the README has the install commands. Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
Claude Code
@@ -0,0 +1,191 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ---- timestamps -----------------------------------------------------------------------
|
||||
|
||||
func TestExtractsTheRfc3339StampFromEveryLineShape(t *testing.T) {
|
||||
// the runtime writes an uptime-ms prefix before [who]; gamelog puts its stamp after the tag;
|
||||
// the switch log starts with the stamp. All three must parse, wherever the stamp sits.
|
||||
lines := []string{
|
||||
` 1234 [pad] [GAMELOG] 2026-10-12T20:45:12.123Z EV ball_start player=1 ball=1 score=0`,
|
||||
` 1300 [pad] gamelog: 2026-10-12T20:45:12.500Z MODE START 5 (Island Scene) player=1`,
|
||||
`2026-10-12T20:45:12.678Z SW 62 "RIGHT FLIPPER BUTTON" level=1 hit=1 game=1`,
|
||||
` 1400 [pad] port /usr/local/padmode/ports/king_kong_le-0.97.port: king_kong_le 0.97, 121 sites, 55 shots`,
|
||||
}
|
||||
want := []bool{true, true, true, false}
|
||||
wanted := []time.Time{
|
||||
time.Date(2026, 10, 12, 20, 45, 12, 123000000, time.UTC),
|
||||
time.Date(2026, 10, 12, 20, 45, 12, 500000000, time.UTC),
|
||||
time.Date(2026, 10, 12, 20, 45, 12, 678000000, time.UTC),
|
||||
}
|
||||
for i, line := range lines {
|
||||
ts, ok := extractTimestamp(line)
|
||||
if ok != want[i] {
|
||||
t.Fatalf("line %d: ok=%v, want %v", i, ok, want[i])
|
||||
}
|
||||
if ok && !ts.Equal(wanted[i]) {
|
||||
t.Fatalf("line %d: wrong stamp %v", i, ts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the game label --------------------------------------------------------------------
|
||||
|
||||
func TestExtractsTheGameFromTheBootLine(t *testing.T) {
|
||||
for _, line := range []string{
|
||||
` 1234 [pad] [GAMELOG] 2026-10-12T20:45:12.123Z up: game king_kong_le 0.97, 55 named shots; event ids 77/37/52/40/200/198; extra_ball_award=13`,
|
||||
`[GAMELOG] 2026-10-13T09:00:00.000Z up: game godzilla_le 1.16, 40 named shots`,
|
||||
} {
|
||||
if g := extractGame(line); g != "king_kong_le" && g != "godzilla_le" {
|
||||
t.Fatalf("extractGame(%q) = %q", line, g)
|
||||
}
|
||||
}
|
||||
if g := extractGame(` 1234 [pad] [GAMELOG] 2026-10-12T20:45:12.123Z EV bonus_end`); g != "" {
|
||||
t.Fatalf("a non-boot line named a game: %q", g)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the Loki payload -------------------------------------------------------------------
|
||||
|
||||
func TestPayloadGroupsByLabelsAndCarriesNanosecondStamps(t *testing.T) {
|
||||
ts := time.Date(2026, 10, 12, 20, 45, 12, 123000000, time.UTC)
|
||||
entries := []Entry{
|
||||
{Source: "mode", Game: "king_kong_le", TS: ts, Line: "one"},
|
||||
{Source: "mode", Game: "king_kong_le", TS: ts.Add(time.Second), Line: "two"},
|
||||
{Source: "switch", Game: "", TS: ts, Line: "three"},
|
||||
}
|
||||
payloads := buildPayloads(entries, "stern-spike2", 1<<20)
|
||||
if len(payloads) != 1 {
|
||||
t.Fatalf("got %d payloads, want 1 (both sources fit under the cap)", len(payloads))
|
||||
}
|
||||
var parsed struct {
|
||||
Streams []struct {
|
||||
Stream map[string]string `json:"stream"`
|
||||
Values [][]string `json:"values"`
|
||||
} `json:"streams"`
|
||||
}
|
||||
if err := json.Unmarshal(payloads[0], &parsed); err != nil {
|
||||
t.Fatalf("payload is not JSON: %v", err)
|
||||
}
|
||||
if len(parsed.Streams) != 2 {
|
||||
t.Fatalf("got %d streams, want 2 (mode and switch have different labels)", len(parsed.Streams))
|
||||
}
|
||||
for _, s := range parsed.Streams {
|
||||
if s.Stream["job"] != "pinball" || s.Stream["machine"] != "stern-spike2" {
|
||||
t.Fatalf("bad labels: %v", s.Stream)
|
||||
}
|
||||
}
|
||||
// find the mode stream and check ordering, stamps and the game label
|
||||
var mode struct {
|
||||
values [][]string
|
||||
}
|
||||
for _, s := range parsed.Streams {
|
||||
if s.Stream["source"] == "mode" {
|
||||
if s.Stream["game"] != "king_kong_le" {
|
||||
t.Fatalf("mode stream game label = %q", s.Stream["game"])
|
||||
}
|
||||
mode.values = s.Values
|
||||
}
|
||||
}
|
||||
if len(mode.values) != 2 || mode.values[0][1] != "one" || mode.values[1][1] != "two" {
|
||||
t.Fatalf("mode values wrong: %v", mode.values)
|
||||
}
|
||||
if mode.values[0][0] != "1791837912123000000" { // the line's own stamp, nanoseconds, as a string
|
||||
t.Fatalf("stamp = %q", mode.values[0][0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPayloadSplitsPastTheCap(t *testing.T) {
|
||||
ts := time.Date(2026, 10, 12, 20, 45, 12, 0, time.UTC)
|
||||
var entries []Entry
|
||||
for i := 0; i < 100; i++ {
|
||||
entries = append(entries, Entry{Source: "mode", TS: ts, Line: strings.Repeat("x", 1000)})
|
||||
}
|
||||
payloads := buildPayloads(entries, "stern-spike2", 4096)
|
||||
if len(payloads) < 2 {
|
||||
t.Fatalf("got %d payloads, want several", len(payloads))
|
||||
}
|
||||
for _, p := range payloads {
|
||||
if len(p) > 4096 {
|
||||
t.Fatalf("payload of %d bytes exceeds the cap", len(p))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- stamps for unstamped lines ---------------------------------------------------------
|
||||
|
||||
func TestUnstampedLinesInheritTheirNeighboursStamps(t *testing.T) {
|
||||
stamp := func(s string) time.Time {
|
||||
ts, _ := time.Parse(stampLayout, s)
|
||||
return ts
|
||||
}
|
||||
fallback := stamp("2026-10-22T12:00:00.000Z")
|
||||
lines := []string{
|
||||
` 1234 [pad] port /usr/local/padmode/ports/king_kong_le-0.97.port: king_kong_le 0.97, 121 sites, 55 shots`,
|
||||
` 1234 [pad] [GAMELOG] 2026-10-12T20:45:12.123Z up: game king_kong_le 0.97, 55 named shots`,
|
||||
` 1250 [pad] [GAMELOG] 2026-10-12T20:45:12.500Z EV game_start - GAME START`,
|
||||
` 1300 [pad] [GAMELOG] 2026-10-12T20:45:13.000Z EV ball_start player=1 ball=1 score=0`,
|
||||
` 1400 [pad] gamelog: no stamp on this one`,
|
||||
}
|
||||
got := assignTimestamps(lines, fallback)
|
||||
// the banner comes BEFORE the first stamp: it takes the next line's stamp, not the
|
||||
// fallback - a stream must not start at "now" and then jump ten days back
|
||||
if !got[0].Equal(stamp("2026-10-12T20:45:12.123Z")) {
|
||||
t.Fatalf("banner stamp = %v, want the next line's stamp", got[0])
|
||||
}
|
||||
for i, want := range []string{"2026-10-12T20:45:12.123Z", "2026-10-12T20:45:12.123Z",
|
||||
"2026-10-12T20:45:12.500Z", "2026-10-12T20:45:13.000Z", "2026-10-12T20:45:13.000Z"} {
|
||||
if !got[i].Equal(stamp(want)) {
|
||||
t.Fatalf("line %d stamp = %v, want %v", i, got[i], want)
|
||||
}
|
||||
}
|
||||
|
||||
// a batch with no stamps at all falls back to the shipping time
|
||||
got = assignTimestamps([]string{"one", "two"}, fallback)
|
||||
if !got[0].Equal(fallback) || !got[1].Equal(fallback) {
|
||||
t.Fatalf("fallback stamps wrong: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the flush after a backfill ---------------------------------------------------------
|
||||
|
||||
func TestOldEntriesNeedAFlush(t *testing.T) {
|
||||
// entries older than the out-of-order window (1h) sit invisible in the ingester's head
|
||||
// until a flush; live entries never need one
|
||||
now := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)
|
||||
fresh := []Entry{{Source: "mode", TS: now.Add(-time.Minute), Line: "x"}}
|
||||
old := []Entry{{Source: "mode", TS: now.Add(-10 * 24 * time.Hour), Line: "x"}}
|
||||
if needsFlush(fresh, now) {
|
||||
t.Fatal("a fresh batch must not trigger a flush")
|
||||
}
|
||||
if !needsFlush(old, now) {
|
||||
t.Fatal("an old batch must trigger a flush")
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the offset state -------------------------------------------------------------------
|
||||
|
||||
func TestStateRoundTrips(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := dir + "/state"
|
||||
st := state{"mode.log": 12345, "switch.log": 67890}
|
||||
if err := saveState(path, st); err != nil {
|
||||
t.Fatalf("save: %v", err)
|
||||
}
|
||||
got, err := loadState(path)
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if got["mode.log"] != 12345 || got["switch.log"] != 67890 {
|
||||
t.Fatalf("round trip lost offsets: %v", got)
|
||||
}
|
||||
if _, err := loadState(path + "-missing"); err != nil {
|
||||
t.Fatalf("a missing state file is not an error: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user