mattlqxandClaude Code
95e7317b81
The gameplay logs sat on /dump on the machine, readable only over SSH. Now they land in Loki for query and tail: - k8s/loki/loki.yaml: Loki 3.4.4 monolithic on the home cluster's monitoring namespace — filesystem storage on CephFS, 30-day retention, traefik ingress at loki.k8s-home.lqx.net with a cert-manager cert, Grafana datasource auto-registered via the sidecar. reject_old_samples_max_age is a year: the machines backfill their history on first ship. - shipper/: logshipper, a ~300-line static Go service for the machine. Tails /dump/mode.log and /dump/switch.log, batches, pushes to Loki's push API. Per-file offsets are saved only after a successful push (down Loki = retry-then-catch-up, never loss); first run starts at each file's end, SHIP_FROM_START=1 ships history; truncation is noticed. Lines keep their own RFC 3339 stamp as the Loki timestamp; unstamped lines (the boot banner) inherit the next stamped line's, so a stream never starts at "now" and jumps back. After a backfill it calls /flush — old entries are invisible in the ingester's head until they reach the store (found the hard way: Loki answers 204 to an old entry on a new stream and then never serves it). Labels: job=pinball, machine, source, game (from the boot line). /etc/init.d/log_shipper runs it at boot. Deployed: Loki is live and verified (query through the ingress). The shipper is verified end-to-end against the cluster from this Mac; the machine was unreachable at deploy time — the README has the install commands. Co-Authored-By: Claude Code <noreply@anthropic.com>
164 lines
3.6 KiB
YAML
164 lines
3.6 KiB
YAML
# Loki, monolithic: one binary, filesystem storage on a CephFS PVC, 30-day retention.
|
|
# Sized for the pinball machines' logs (a few MB a day), not for a fleet.
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: loki-config
|
|
namespace: monitoring
|
|
labels:
|
|
app: loki
|
|
data:
|
|
loki.yaml: |
|
|
auth_enabled: false
|
|
server:
|
|
http_listen_port: 3100
|
|
grpc_listen_port: 9096
|
|
common:
|
|
path_prefix: /var/lib/loki
|
|
replication_factor: 1
|
|
ring:
|
|
kvstore:
|
|
store: inmemory
|
|
storage:
|
|
filesystem:
|
|
chunks_directory: /var/lib/loki/chunks
|
|
rules_directory: /var/lib/loki/rules
|
|
schema_config:
|
|
configs:
|
|
- from: 2024-01-01
|
|
store: tsdb
|
|
object_store: filesystem
|
|
schema: v13
|
|
index:
|
|
prefix: index_
|
|
period: 24h
|
|
limits_config:
|
|
retention_period: 720h # 30 days
|
|
# the machines backfill their history on first ship: accept old entries
|
|
reject_old_samples: true
|
|
reject_old_samples_max_age: 8760h # a year
|
|
compactor:
|
|
working_directory: /var/lib/loki/compactor
|
|
retention_enabled: true
|
|
delete_request_store: filesystem
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: loki
|
|
namespace: monitoring
|
|
labels:
|
|
app: loki
|
|
spec:
|
|
serviceName: loki
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: loki
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: loki
|
|
spec:
|
|
securityContext:
|
|
fsGroup: 10001
|
|
runAsNonRoot: true
|
|
runAsUser: 10001
|
|
runAsGroup: 10001
|
|
containers:
|
|
- name: loki
|
|
image: grafana/loki:3.4.4
|
|
args: ["-config.file=/etc/loki/loki.yaml"]
|
|
ports:
|
|
- containerPort: 3100
|
|
name: http
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 256Mi
|
|
limits:
|
|
cpu: "1"
|
|
memory: 1Gi
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /ready
|
|
port: http
|
|
initialDelaySeconds: 15
|
|
volumeMounts:
|
|
- name: config
|
|
mountPath: /etc/loki
|
|
- name: data
|
|
mountPath: /var/lib/loki
|
|
volumes:
|
|
- name: config
|
|
configMap:
|
|
name: loki-config
|
|
volumeClaimTemplates:
|
|
- metadata:
|
|
name: data
|
|
spec:
|
|
accessModes: ["ReadWriteOnce"]
|
|
storageClassName: csi-cephfs
|
|
resources:
|
|
requests:
|
|
storage: 10Gi
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: loki
|
|
namespace: monitoring
|
|
labels:
|
|
app: loki
|
|
spec:
|
|
selector:
|
|
app: loki
|
|
ports:
|
|
- name: http
|
|
port: 3100
|
|
targetPort: http
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: loki
|
|
namespace: monitoring
|
|
annotations:
|
|
cert-manager.io/cluster-issuer: letsencrypt
|
|
spec:
|
|
ingressClassName: traefik-public
|
|
rules:
|
|
- host: loki.k8s-home.lqx.net
|
|
http:
|
|
paths:
|
|
- backend:
|
|
service:
|
|
name: loki
|
|
port:
|
|
number: 3100
|
|
path: /
|
|
pathType: Prefix
|
|
tls:
|
|
- hosts:
|
|
- loki.k8s-home.lqx.net
|
|
secretName: loki-tls
|
|
---
|
|
# picked up by the kube-prom Grafana's datasource sidecar
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: grafana-loki-datasource
|
|
namespace: monitoring
|
|
labels:
|
|
grafana_datasource: "1"
|
|
data:
|
|
loki-datasource.yaml: |
|
|
apiVersion: 1
|
|
datasources:
|
|
- name: Loki
|
|
type: loki
|
|
access: proxy
|
|
url: http://loki.monitoring.svc.cluster.local:3100
|
|
isDefault: false
|