chore: configure alerting and monitoring (#25857)

This commit is contained in:
Victor Navarro 2026-05-05 16:08:28 +02:00 committed by GitHub
parent 773078e81f
commit 726ae6f541
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 471 additions and 0 deletions

View file

@ -36,6 +36,8 @@ jobs:
PLANETSCALE_SERVICE_TOKEN_NAME: ${{ secrets.PLANETSCALE_SERVICE_TOKEN_NAME }}
PLANETSCALE_SERVICE_TOKEN: ${{ secrets.PLANETSCALE_SERVICE_TOKEN }}
STRIPE_SECRET_KEY: ${{ github.ref_name == 'production' && secrets.STRIPE_SECRET_KEY_PROD || secrets.STRIPE_SECRET_KEY_DEV }}
HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY }}
INCIDENT_API_KEY: ${{ secrets.INCIDENT_API_KEY }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_ORG: ${{ vars.SENTRY_ORG }}
SENTRY_PROJECT: ${{ vars.WEB_SENTRY_PROJECT }}

View file

@ -107,6 +107,7 @@
"solid-js": "catalog:",
"solid-list": "0.3.0",
"solid-stripe": "0.8.1",
"svix": "1.92.2",
"vite": "catalog:",
"zod": "catalog:",
},
@ -2159,6 +2160,8 @@
"@speed-highlight/core": ["@speed-highlight/core@1.2.15", "", {}, "sha512-BMq1K3DsElxDWawkX6eLg9+CKJrTVGCBAWVuHXVUV2u0s2711qiChLSId6ikYPfxhdYocLNt3wWwSvDiTvFabw=="],
"@stablelib/base64": ["@stablelib/base64@1.0.1", "", {}, "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ=="],
"@standard-community/standard-json": ["@standard-community/standard-json@0.3.5", "", { "peerDependencies": { "@standard-schema/spec": "^1.0.0", "@types/json-schema": "^7.0.15", "@valibot/to-json-schema": "^1.3.0", "arktype": "^2.1.20", "effect": "^3.16.8", "quansync": "^0.2.11", "sury": "^10.0.0", "typebox": "^1.0.17", "valibot": "^1.1.0", "zod": "^3.25.0 || ^4.0.0", "zod-to-json-schema": "^3.24.5" }, "optionalPeers": ["@valibot/to-json-schema", "arktype", "effect", "sury", "typebox", "valibot", "zod", "zod-to-json-schema"] }, "sha512-4+ZPorwDRt47i+O7RjyuaxHRK/37QY/LmgxlGrRrSTLYoFatEOzvqIc85GTlM18SFZ5E91C+v0o/M37wZPpUHA=="],
"@standard-community/standard-openapi": ["@standard-community/standard-openapi@0.2.9", "", { "peerDependencies": { "@standard-community/standard-json": "^0.3.5", "@standard-schema/spec": "^1.0.0", "arktype": "^2.1.20", "effect": "^3.17.14", "openapi-types": "^12.1.3", "sury": "^10.0.0", "typebox": "^1.0.0", "valibot": "^1.1.0", "zod": "^3.25.0 || ^4.0.0", "zod-openapi": "^4" }, "optionalPeers": ["arktype", "effect", "sury", "typebox", "valibot", "zod", "zod-openapi"] }, "sha512-htj+yldvN1XncyZi4rehbf9kLbu8os2Ke/rfqoZHCMHuw34kiF3LP/yQPdA0tQ940y8nDq3Iou8R3wG+AGGyvg=="],
@ -3169,6 +3172,8 @@
"fast-querystring": ["fast-querystring@1.1.2", "", { "dependencies": { "fast-decode-uri-component": "^1.0.1" } }, "sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg=="],
"fast-sha256": ["fast-sha256@1.3.0", "", {}, "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ=="],
"fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="],
"fast-xml-builder": ["fast-xml-builder@1.1.4", "", { "dependencies": { "path-expression-matcher": "^1.1.3" } }, "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg=="],
@ -4643,6 +4648,8 @@
"standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="],
"standardwebhooks": ["standardwebhooks@1.0.0", "", { "dependencies": { "@stablelib/base64": "^1.0.0", "fast-sha256": "^1.3.0" } }, "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg=="],
"stat-mode": ["stat-mode@1.0.0", "", {}, "sha512-jH9EhtKIjuXZ2cWxmXS8ZP80XyC3iasQxMDV8jzhNJpfDb7VbQLVW4Wvsxz9QZvzV+G4YoSfBUVKDOyxLzi/sg=="],
"statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="],
@ -4711,6 +4718,8 @@
"sury": ["sury@11.0.0-alpha.4", "", { "peerDependencies": { "rescript": "12.x" }, "optionalPeers": ["rescript"] }, "sha512-oeG/GJWZvQCKtGPpLbu0yCZudfr5LxycDo5kh7SJmKHDPCsEPJssIZL2Eb4Tl7g9aPEvIDuRrkS+L0pybsMEMA=="],
"svix": ["svix@1.92.2", "", { "dependencies": { "standardwebhooks": "1.0.0" } }, "sha512-ZmuA3UVvlnF9EgxlzmPtF7CKjQb64Z6OFlyfdDfU0sdcC7dJa+3aOYX5B9mA+RS6ch1AxBa4UP/l6KmqfGtWBQ=="],
"system-architecture": ["system-architecture@0.1.0", "", {}, "sha512-ulAk51I9UVUyJgxlv9M6lFot2WP3e7t8Kz9+IS6D4rVba1tR9kON+Ey69f+1R4Q8cd45Lod6a4IcJIxnzGc/zA=="],
"tailwindcss": ["tailwindcss@4.1.11", "", {}, "sha512-2E9TBm6MDD/xKYe+dvJZAmg3yxIEDNRc0jwlNyDg/4Fil2QcSLjFKGVff0lAf1jjeaArlG/M75Ey/EYr/OJtBA=="],

View file

@ -221,6 +221,9 @@ const AUTH_API_URL = new sst.Linkable("AUTH_API_URL", {
const STRIPE_WEBHOOK_SECRET = new sst.Linkable("STRIPE_WEBHOOK_SECRET", {
properties: { value: stripeWebhook.secret },
})
const INCIDENT_WEBHOOK_SIGNING_SECRET = new sst.Secret("INCIDENT_WEBHOOK_SIGNING_SECRET")
const DISCORD_INCIDENT_WEBHOOK_URL = new sst.Secret("DISCORD_INCIDENT_WEBHOOK_URL")
const gatewayKv = new sst.cloudflare.Kv("GatewayKv")
////////////////
@ -251,6 +254,8 @@ new sst.cloudflare.x.SolidStart("Console", {
database,
AUTH_API_URL,
STRIPE_WEBHOOK_SECRET,
INCIDENT_WEBHOOK_SIGNING_SECRET,
DISCORD_INCIDENT_WEBHOOK_URL,
STRIPE_SECRET_KEY,
EMAILOCTOPUS_API_KEY,
AWS_SES_ACCESS_KEY_ID,

320
infra/monitoring.ts Normal file
View file

@ -0,0 +1,320 @@
const displayName = (s: string) =>
s
.split("-")
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
.join(" ")
.replace(/(?<=\d) (?=\d)/g, ".")
const resourceName = (s: string) => displayName(s).replace(/[^a-zA-Z0-9]/g, "")
const varSpec = (label: string, name: string) =>
$jsonStringify({
content: [
{
content: [
{
attrs: {
name,
label,
missing: false,
},
type: "varSpec",
},
],
type: "paragraph",
},
],
type: "doc",
})
const fields = {
model: incident.getAlertAttributeOutput({ name: "Model" }),
product: incident.getAlertAttributeOutput({ name: "Product" }),
}
const alertSource = new incident.AlertSource("HoneycombAlertSource", {
name: $app.stage === "production" ? "Honeycomb" : `Honeycomb (${$app.stage})`,
sourceType: "honeycomb",
template: {
title: {
literal: varSpec("Payload -> Title", "title"),
},
description: {
literal: varSpec("Payload -> Description", "description"),
},
attributes: [
{
alertAttributeId: fields.model.id,
binding: {
value: {
reference: 'expressions["model"]',
},
mergeStrategy: "first_wins",
},
},
{
alertAttributeId: fields.product.id,
binding: {
value: {
reference: 'expressions["product"]',
},
mergeStrategy: "first_wins",
},
},
],
expressions: [
{
label: "Model",
operations: [
{
operationType: "parse",
parse: {
returns: {
array: false,
type: fields.model.type,
},
source: "$['model']",
},
},
],
reference: "model",
rootReference: "payload",
},
{
label: "Product",
operations: [
{
operationType: "parse",
parse: {
returns: {
array: false,
type: fields.product.type,
},
source: "$['product']",
},
},
],
reference: "product",
rootReference: "payload",
},
],
},
})
const webhookRecipient = new honeycomb.WebhookRecipient(`IncidentWebhook`, {
name: $app.stage === "production" ? "Incident.io" : `Incident.io (${$app.stage})`,
url: alertSource.alertEventsUrl,
secret: alertSource.secretToken,
templates: [
{
type: "trigger",
body: $jsonStringify({
title: "{{ .Name }}",
description: "{{ .Description }}",
status: "{{ .Alert.Status }}",
deduplication_key: "{{ .Alert.InstanceID }}",
source_url: "{{ .Result.URL }}",
model: "{{ .Vars.model }}",
product: "{{ .Vars.product }}",
}),
},
],
variables: [
{
name: "model",
},
{
name: "product",
},
],
})
new incident.AlertRoute("HoneycombAlertRoute", {
name: $app.stage === "production" ? "Honeycomb" : `Honeycomb (${$app.stage})`,
enabled: true,
isPrivate: false,
alertSources: [
{
alertSourceId: alertSource.id,
conditionGroups: [
{
conditions: [
{
subject: "alert.title",
operation: "is_set",
paramBindings: [],
},
],
},
],
},
],
conditionGroups: [
{
conditions: [
{
subject: "alert.title",
operation: "is_set",
paramBindings: [],
},
],
},
],
expressions: [],
escalationConfig: {
autoCancelEscalations: true,
escalationTargets: [],
},
incidentConfig: {
autoDeclineEnabled: true,
enabled: true,
conditionGroups: [],
deferTimeSeconds: 0,
groupingKeys: [
{
reference: $interpolate`alert.attributes.${fields.model.id}`,
},
{
reference: $interpolate`alert.attributes.${fields.product.id}`,
},
],
groupingWindowSeconds: 900,
},
incidentTemplate: {
name: {
value: {
literal: varSpec("Alert -> Title", "alert.title"),
},
},
summary: {
value: {
literal: varSpec("Alert -> Description", "alert.description"),
},
},
startInTriage: {
value: {
literal: "true",
},
},
severity: {
mergeStrategy: "first-wins",
},
incidentMode: {
value: {
literal: $app.stage === "production" ? "standard" : "test",
},
},
},
})
type Product = "go" | "zen"
type Trigger = (opts: { model: string; product: Product }) => {
id: string
title: string
description: string
json: honeycomb.GetQuerySpecificationOutputArgs
threshold: { op: ">=" | "<="; value: number }
baseline: 3600 | 86400
}
type Model = { id: string; products: Product[]; triggers: Trigger[] }
const httpErrors: Trigger = ({ model, product }) => ({
id: "increased-http-errors",
title: `Increased HTTP Errors for ${displayName(model)} on ${displayName(product)}`,
description: `Detected increased rate of HTTP errors for ${displayName(model)} on OpenCode ${displayName(product)}`,
json: {
calculations: [
{
op: "COUNT",
name: "TOTAL",
filterCombination: "AND",
filters: [
{ column: "model", op: "=", value: model },
{ column: "isGoTier", op: "=", value: product === "go" ? "true" : "false" },
],
},
{
op: "COUNT",
name: "FAILED",
filterCombination: "AND",
filters: [
{ column: "model", op: "=", value: model },
{ column: "isGoTier", op: "=", value: product === "go" ? "true" : "false" },
{ column: "status", op: ">=", value: "400" },
{ column: "status", op: "!=", value: "401" },
],
},
],
formulas: [{ name: "ERROR", expression: "$FAILED / $TOTAL" }],
timeRange: 900,
},
// Alert when errors surge 50% compared to the previous period
threshold: { op: ">=", value: 50 },
// What previous time period to evaluate against
baseline: 3600,
})
const models: Model[] = [
{ id: "kimi-k2.6", products: ["go", "zen"], triggers: [httpErrors] },
{ id: "kimi-k2.5", products: ["go", "zen"], triggers: [httpErrors] },
{ id: "deepseek-v4-flash", products: ["go", "zen"], triggers: [httpErrors] },
{ id: "deepseek-v4-pro", products: ["go", "zen"], triggers: [httpErrors] },
{ id: "glm-5.1", products: ["go", "zen"], triggers: [httpErrors] },
// { id: "glm-5", products: ["go"], triggers: [httpErrors] },
{ id: "qwen3.6-plus", products: ["go", "zen"], triggers: [httpErrors] },
{ id: "qwen3.5-plus", products: ["go"], triggers: [httpErrors] },
{ id: "minimax-m2.7", products: ["go", "zen"], triggers: [httpErrors] },
// { id: "minimax-m2.5", products: ["go", "zen"], triggers: [httpErrors] },
{ id: "mimo-v2.5-pro", products: ["go"], triggers: [httpErrors] },
// { id: "mimo-v2.5", products: ["go"], triggers: [httpErrors] },
// { id: "mimo-v2-omni", products: ["go"], triggers: [httpErrors] },
// { id: "mimo-v2-pro", products: ["go"], triggers: [httpErrors] },
{ id: "claude-opus-4-7", products: ["zen"], triggers: [httpErrors] },
// { id: "claude-opus-4-6", products: ["zen"], triggers: [httpErrors] },
// { id: "claude-sonnet-4-6", products: ["zen"], triggers: [httpErrors] },
{ id: "gpt-5.5", products: ["zen"], triggers: [httpErrors] },
{ id: "big-pickle", products: ["zen"], triggers: [httpErrors] },
// { id: "minimax-m2.5-free", products: ["zen"], triggers: [httpErrors] },
// { id: "hy3-preview-free", products: ["zen"], triggers: [httpErrors] },
// { id: "nemotron-3-super-free", products: ["zen"], triggers: [httpErrors] },
// { id: "trinity-large-preview-free", products: ["zen"], triggers: [httpErrors] },
// { id: "ling-2.6-flash-free", products: ["zen"], triggers: [httpErrors] },
]
if ($app.stage !== "production") {
models.splice(1)
}
for (const model of models) {
for (const product of model.products) {
for (const trigger of model.triggers) {
const spec = trigger({ model: model.id, product })
new honeycomb.Trigger(resourceName(`${spec.id}-${product}-${model.id}`), {
name: spec.title,
description: spec.description,
queryJson: honeycomb.getQuerySpecificationOutput(spec.json).json,
alertType: "on_change",
// This is the minimum when using % change detection
frequency: 900,
baselineDetails: [{ type: "percentage", offsetMinutes: spec.baseline / 60 }],
thresholds: [{ ...spec.threshold, exceededLimit: 1 }],
recipients: [
{
id: webhookRecipient.id,
notificationDetails: [
{
variables: [
{ name: "model", value: model.id },
{ name: "product", value: product },
],
},
],
},
],
})
}
}
}

View file

@ -31,6 +31,7 @@
"solid-js": "catalog:",
"solid-list": "0.3.0",
"solid-stripe": "0.8.1",
"svix": "1.92.2",
"vite": "catalog:",
"zod": "catalog:"
},

View file

@ -0,0 +1,75 @@
import type { APIEvent } from "@solidjs/start/server"
import { Resource } from "@opencode-ai/console-resource"
import { Webhook } from "svix"
type Incident = {
mode?: "test" | "standard"
name?: string
permalink?: string
summary?: string
}
type IncidentWebhookPayload = {
event_type?: string
"public_incident.incident_created_v2"?: Incident
}
const verifyWebhook = async (request: Request) => {
const body = await request.text()
try {
return new Webhook(Resource.INCIDENT_WEBHOOK_SIGNING_SECRET.value).verify(
body,
Object.fromEntries(request.headers.entries()),
) as IncidentWebhookPayload
} catch {
return undefined
}
}
const postDiscordMessage = async (incident: Incident) => {
return fetch(Resource.DISCORD_INCIDENT_WEBHOOK_URL.value, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
content: [
`**${incident.mode === "test" ? "[TEST] " : ""}${incident.name ?? "Incident has been created"}**`,
incident.summary,
"",
"@everyone",
"",
incident.permalink,
]
.filter((line) => line !== undefined)
.join("\n"),
allowed_mentions: {
parse: ["everyone"],
},
flags: 4,
}),
})
}
export async function POST(input: APIEvent) {
const payload = await verifyWebhook(input.request)
if (!payload) {
return Response.json({ message: "invalid signature" }, { status: 401 })
}
if (payload.event_type !== "public_incident.incident_created_v2") {
return Response.json({ message: "ignored event" }, { status: 200 })
}
const incident = payload["public_incident.incident_created_v2"]
if (!incident) {
return Response.json({ message: "missing incident" }, { status: 400 })
}
const response = await postDiscordMessage(incident)
if (!response.ok) {
return Response.json({ message: "discord webhook failed" }, { status: 502 })
}
return Response.json({ message: "sent" }, { status: 200 })
}

View file

@ -35,6 +35,10 @@ declare module "sst" {
"type": "sst.cloudflare.SolidStart"
"url": string
}
"DISCORD_INCIDENT_WEBHOOK_URL": {
"type": "sst.sst.Secret"
"value": string
}
"DISCORD_SUPPORT_BOT_TOKEN": {
"type": "sst.sst.Secret"
"value": string
@ -87,6 +91,10 @@ declare module "sst" {
"type": "sst.sst.Secret"
"value": string
}
"INCIDENT_WEBHOOK_SIGNING_SECRET": {
"type": "sst.sst.Secret"
"value": string
}
"R2AccessKey": {
"type": "sst.sst.Secret"
"value": string

View file

@ -35,6 +35,10 @@ declare module "sst" {
"type": "sst.cloudflare.SolidStart"
"url": string
}
"DISCORD_INCIDENT_WEBHOOK_URL": {
"type": "sst.sst.Secret"
"value": string
}
"DISCORD_SUPPORT_BOT_TOKEN": {
"type": "sst.sst.Secret"
"value": string
@ -87,6 +91,10 @@ declare module "sst" {
"type": "sst.sst.Secret"
"value": string
}
"INCIDENT_WEBHOOK_SIGNING_SECRET": {
"type": "sst.sst.Secret"
"value": string
}
"R2AccessKey": {
"type": "sst.sst.Secret"
"value": string

View file

@ -35,6 +35,10 @@ declare module "sst" {
"type": "sst.cloudflare.SolidStart"
"url": string
}
"DISCORD_INCIDENT_WEBHOOK_URL": {
"type": "sst.sst.Secret"
"value": string
}
"DISCORD_SUPPORT_BOT_TOKEN": {
"type": "sst.sst.Secret"
"value": string
@ -87,6 +91,10 @@ declare module "sst" {
"type": "sst.sst.Secret"
"value": string
}
"INCIDENT_WEBHOOK_SIGNING_SECRET": {
"type": "sst.sst.Secret"
"value": string
}
"R2AccessKey": {
"type": "sst.sst.Secret"
"value": string

View file

@ -35,6 +35,10 @@ declare module "sst" {
"type": "sst.cloudflare.SolidStart"
"url": string
}
"DISCORD_INCIDENT_WEBHOOK_URL": {
"type": "sst.sst.Secret"
"value": string
}
"DISCORD_SUPPORT_BOT_TOKEN": {
"type": "sst.sst.Secret"
"value": string
@ -87,6 +91,10 @@ declare module "sst" {
"type": "sst.sst.Secret"
"value": string
}
"INCIDENT_WEBHOOK_SIGNING_SECRET": {
"type": "sst.sst.Secret"
"value": string
}
"R2AccessKey": {
"type": "sst.sst.Secret"
"value": string

View file

@ -35,6 +35,10 @@ declare module "sst" {
"type": "sst.cloudflare.SolidStart"
"url": string
}
"DISCORD_INCIDENT_WEBHOOK_URL": {
"type": "sst.sst.Secret"
"value": string
}
"DISCORD_SUPPORT_BOT_TOKEN": {
"type": "sst.sst.Secret"
"value": string
@ -87,6 +91,10 @@ declare module "sst" {
"type": "sst.sst.Secret"
"value": string
}
"INCIDENT_WEBHOOK_SIGNING_SECRET": {
"type": "sst.sst.Secret"
"value": string
}
"R2AccessKey": {
"type": "sst.sst.Secret"
"value": string

8
sst-env.d.ts vendored
View file

@ -50,6 +50,10 @@ declare module "sst" {
"type": "sst.cloudflare.SolidStart"
"url": string
}
"DISCORD_INCIDENT_WEBHOOK_URL": {
"type": "sst.sst.Secret"
"value": string
}
"DISCORD_SUPPORT_BOT_TOKEN": {
"type": "sst.sst.Secret"
"value": string
@ -110,6 +114,10 @@ declare module "sst" {
"type": "sst.sst.Secret"
"value": string
}
"INCIDENT_WEBHOOK_SIGNING_SECRET": {
"type": "sst.sst.Secret"
"value": string
}
"LogProcessor": {
"type": "sst.cloudflare.Worker"
}

View file

@ -12,6 +12,14 @@ export default $config({
apiKey: process.env.STRIPE_SECRET_KEY!,
},
planetscale: "0.4.1",
honeycomb: {
version: "0.49.0",
apiKey: process.env.HONEYCOMB_API_KEY!,
},
incident: {
version: "5.35.0",
apiKey: process.env.INCIDENT_API_KEY!,
},
},
}
},
@ -19,5 +27,8 @@ export default $config({
await import("./infra/app.js")
await import("./infra/console.js")
await import("./infra/enterprise.js")
if ($app.stage === "production") {
await import("./infra/monitoring.js")
}
},
})