fix(reporting): translate UTF-8 report strings to cp1252 before PDF render

Free-form strings entering the PDF generator (AI narrative prose, resource
names, alert messages, brand display names) were written to fpdf core fonts
as raw UTF-8, and the cp1252-decoding fonts rendered em dashes and curly
quotes as mojibake. Generate and GenerateMulti now run every string field
reachable from ReportData/MultiReportData through fpdf's cp1252 translator
once before rendering, so write sites stay encoding-free. The translator is
built per call: fpdf's closure reuses an internal buffer and the generator
is shared across concurrent requests. Runes outside cp1252 degrade to '.'.
Tests render AI-shaped narratives with em dashes and curly quotes for both
the single-resource and fleet paths and assert the extracted content
streams decode without mojibake.
This commit is contained in:
rcourtman 2026-06-10 17:18:19 +01:00
parent e6c0c4d385
commit 3dc06bea71
3 changed files with 235 additions and 0 deletions

View file

@ -156,6 +156,14 @@ func (g *PDFGenerator) Generate(data *ReportData) ([]byte, error) {
pdf.SetMargins(20, 20, 20)
pdf.SetAutoPageBreak(true, 25)
// Core fonts are cp1252: translate the UTF-8 report strings (AI
// narrative prose, resource names, alert messages) once up front so
// em dashes and curly quotes render as glyphs instead of mojibake.
// The translator is built per call because fpdf's closure reuses an
// internal buffer and this generator is shared across concurrent
// requests; see pdf_codepage.go.
translateReportStrings(data, pdf.UnicodeTranslatorFromDescriptor(""))
// Cover page
g.writeCoverPage(pdf, data)
@ -1677,6 +1685,10 @@ func (g *PDFGenerator) GenerateMulti(data *MultiReportData) ([]byte, error) {
pdf.SetMargins(20, 20, 20)
pdf.SetAutoPageBreak(true, 25)
// Translate UTF-8 strings to the cp1252 space the core fonts expect;
// see the matching call in Generate and pdf_codepage.go.
translateMultiReportStrings(data, pdf.UnicodeTranslatorFromDescriptor(""))
// Page 1: Cover page
g.writeMultiCoverPage(pdf, data)

View file

@ -0,0 +1,97 @@
package reporting
import "reflect"
// The PDF renderer uses fpdf core fonts (Arial), which read text as cp1252
// bytes. Free-form strings entering ReportData/MultiReportData are UTF-8:
// AI narrative prose full of em dashes and curly quotes, user-controlled
// resource names, alert messages, brand display names. Writing them
// untranslated renders mojibake in the PDF (an em dash becomes "—").
//
// translateReportStrings and translateMultiReportStrings rewrite every
// reachable string field through fpdf's cp1252 translator once, before
// rendering, so individual write sites never have to think about encoding.
// Reflection (rather than a hand-maintained field list) keeps newly added
// narrative, branding, and enrichment fields covered without anyone having
// to remember this layer exists. Runes cp1252 cannot represent degrade to
// "." (fpdf's documented behaviour), which is unfortunate but readable;
// raw UTF-8 bytes are neither.
//
// The translator must be created per Generate call: fpdf's translator
// closure reuses an internal buffer, so it is not safe for concurrent use,
// and one PDFGenerator is shared across concurrent requests. Translation
// mutates the data in place; report data is built per request, so no caller
// observes the rewrite.
// translateReportStrings rewrites every string field reachable from data
// into the cp1252 byte space the core PDF fonts expect. tr is the output of
// pdf.UnicodeTranslatorFromDescriptor for the document being generated.
func translateReportStrings(data *ReportData, tr func(string) string) {
if data == nil || tr == nil {
return
}
translateStringFields(reflect.ValueOf(data).Elem(), tr, map[uintptr]bool{})
}
// translateMultiReportStrings is the fleet-report counterpart of
// translateReportStrings, covering the fleet narrative and every
// per-resource ReportData.
func translateMultiReportStrings(data *MultiReportData, tr func(string) string) {
if data == nil || tr == nil {
return
}
translateStringFields(reflect.ValueOf(data).Elem(), tr, map[uintptr]bool{})
}
// translateStringFields walks v depth-first and rewrites every settable
// string through tr. visited guards pointer aliasing (e.g. a *ReportBrand
// shared between MultiReportData and its per-resource ReportData entries):
// cp1252 translation is not idempotent, because a second pass would read
// the raw high bytes as invalid UTF-8 and flatten them to ".".
func translateStringFields(v reflect.Value, tr func(string) string, visited map[uintptr]bool) {
switch v.Kind() {
case reflect.String:
if v.CanSet() {
v.SetString(tr(v.String()))
}
case reflect.Pointer:
if v.IsNil() || visited[v.Pointer()] {
return
}
visited[v.Pointer()] = true
translateStringFields(v.Elem(), tr, visited)
case reflect.Struct:
t := v.Type()
for i := 0; i < v.NumField(); i++ {
// Unexported fields (time.Time internals) are read-only
// through reflection and hold nothing user-visible.
if !t.Field(i).IsExported() {
continue
}
translateStringFields(v.Field(i), tr, visited)
}
case reflect.Slice, reflect.Array:
for i := 0; i < v.Len(); i++ {
translateStringFields(v.Index(i), tr, visited)
}
case reflect.Map:
// Keys stay untouched: they are lookup identifiers (metric
// names), and translating them would break renderer lookups.
for _, k := range v.MapKeys() {
mv := v.MapIndex(k)
switch mv.Kind() {
case reflect.String:
v.SetMapIndex(k, reflect.ValueOf(tr(mv.String())).Convert(mv.Type()))
case reflect.Struct:
// Map values are not addressable; translate a
// copy and store it back.
cp := reflect.New(mv.Type()).Elem()
cp.Set(mv)
translateStringFields(cp, tr, visited)
v.SetMapIndex(k, cp)
case reflect.Pointer, reflect.Slice, reflect.Map:
translateStringFields(mv, tr, visited)
}
}
}
}

View file

@ -0,0 +1,126 @@
package reporting
import (
"strings"
"testing"
"time"
)
// Free-form report strings arrive as UTF-8 but fpdf core fonts read cp1252
// bytes. These tests render narratives full of typographic punctuation (the
// exact characters AI narrators emit) plus a non-ASCII resource name, then
// assert the content streams carry cp1252 encodings rather than raw UTF-8
// byte sequences, which a PDF viewer displays as mojibake (an em dash
// becomes "—", an é becomes "é").
func TestGenerate_NarrativeUTF8RendersWithoutMojibake(t *testing.T) {
now := time.Now()
data := &ReportData{
Title: "Monthly Review — “Production”",
ResourceType: "vm",
ResourceID: "vm-7f8b2b6cd98c2089",
Resource: &ResourceInfo{Name: "café-web-01", Status: "running"},
Start: now.Add(-time.Hour),
End: now,
GeneratedAt: now,
Metrics: map[string][]MetricDataPoint{},
Summary: MetricSummary{ByMetric: map[string]MetricStats{
"cpu": {Avg: 10, Max: 20, Count: 60},
}},
TotalPoints: 60,
Narrative: &Narrative{
Source: NarrativeSourceAI,
HealthStatus: "HEALTHY",
HealthMessage: "Quiet period — no incidents recorded",
ExecutiveSummary: "Utilisation stayed “well within” limits — the hosts capacity is sufficient.",
Observations: []NarrativeBullet{{Text: "CPU averaged 10% — flat across the window", Severity: NarrativeSeverityOK}},
Recommendations: []string{"Keep monitoring — no changes required"},
Disclaimer: "Narrative generated by Pulse Assistant.",
},
Alerts: []AlertInfo{{
Type: "cpu",
Level: "warning",
Message: "CPU high — sustained above threshold",
StartTime: now.Add(-30 * time.Minute),
}},
}
assertNoMojibake(t, renderExecutiveSummaryText(t, data),
[]string{"—", "“", "”", "", "café-web-01"})
}
func TestGenerateMulti_FleetNarrativeUTF8RendersWithoutMojibake(t *testing.T) {
now := time.Now()
multi := &MultiReportData{
Title: "Fleet Review — “June”",
Start: now.Add(-time.Hour),
End: now,
GeneratedAt: now,
Resources: []*ReportData{{
ResourceID: "vm-a",
ResourceType: "vm",
Resource: &ResourceInfo{Name: "café-web-01", Status: "running"},
Summary: MetricSummary{ByMetric: map[string]MetricStats{"cpu": {Avg: 10, Max: 15, Count: 60}}},
TotalPoints: 60,
}},
FleetNarrative: &FleetNarrative{
Source: NarrativeSourceAI,
HealthStatus: "HEALTHY",
HealthMessage: "Fleet stable — “no outliers” this period",
Outliers: []FleetOutlier{{
ResourceID: "vm-a",
ResourceName: "café-web-01",
Reason: "Memory averaging 91.2% — sustained pressure",
Severity: NarrativeSeverityWarning,
}},
Disclaimer: "Narrative generated by Pulse Assistant.",
},
}
assertNoMojibake(t, renderFleetSummaryText(t, multi),
[]string{"—", "“", "”", "café-web-01"})
}
// assertNoMojibake decodes the raw content-stream bytes the way a PDF
// viewer decodes core-font text (cp1252/WinAnsi) and asserts the
// typographic characters survived translation. Untranslated UTF-8 decodes
// to fragments led by "â€" (punctuation range) or "Ã" (latin-1 range).
func assertNoMojibake(t *testing.T, raw string, want []string) {
t.Helper()
rendered := decodeWinAnsi(raw)
for _, frag := range []string{"â€", "é"} {
if strings.Contains(rendered, frag) {
t.Errorf("rendered PDF text contains mojibake %q:\n%s", frag, rendered)
}
}
for _, w := range want {
if !strings.Contains(rendered, w) {
t.Errorf("rendered PDF text missing %q after cp1252 translation:\n%s", w, rendered)
}
}
}
// winAnsiSpecials covers the 0x80-0x9F block where cp1252 places the
// typographic punctuation that latin-1 lacks; every other byte maps to the
// identical Unicode code point.
var winAnsiSpecials = map[byte]rune{
0x80: '€', 0x82: '', 0x83: 'ƒ', 0x84: '„', 0x85: '…', 0x86: '†', 0x87: '‡',
0x88: 'ˆ', 0x89: '‰', 0x8A: 'Š', 0x8B: '', 0x8C: 'Œ', 0x8E: 'Ž',
0x91: '', 0x92: '', 0x93: '“', 0x94: '”', 0x95: '•', 0x96: '', 0x97: '—',
0x98: '˜', 0x99: '™', 0x9A: 'š', 0x9B: '', 0x9C: 'œ', 0x9E: 'ž', 0x9F: 'Ÿ',
}
func decodeWinAnsi(s string) string {
var b strings.Builder
for i := 0; i < len(s); i++ {
c := s[i]
if c < 0x80 {
b.WriteByte(c)
continue
}
if r, ok := winAnsiSpecials[c]; ok {
b.WriteRune(r)
continue
}
b.WriteRune(rune(c))
}
return b.String()
}