What the Staged Rollout Plan Generator actually produces

{ "phase1_internal": { "name": "Internal Dogfood (Week 1)", "cohort": "Product, Engineering, and Customer Success teams (n~50); US-based only; non-regulated verticals", "trafficPercentage": "0% of production", "duration": "7 days", "promotionCriteria": [ "Zero P0 incidents (hallucinations, credential leaks, format errors) in 50 sessions", "Manual quality review: 3 subject-matter experts rate ≥80% of suggestions as on-brand and contextually appropriate (score ≥4/5)", "Composer latency P95 ≤800ms in 100% of dogfood sessions", "Team consensus sign-off from PM, Eng Lead, and Legal" ], "killCriteria": "Any credential/confidential data exposure, model generating offensive/discriminatory content, or P95 latency >1500ms → immediate feature halt pending investigation" }, "phase2_closed_beta": { "name": "Closed Beta Opt-In (Weeks 2-3)", "cohort": "US-only customers, Enterprise+ plan holders (n~120), self-selected into beta via settings toggle; exclude: HIPAA/financial-regulated, custom data residency, active escalations, users with AI opt-out flag", "trafficPercentage": "0.8% of MAU (3.8K users)", "duration": "14 days", "promotionCriteria": [ "Suggestion acceptance rate ≥15% (within 3pp of 18% target)", "7-day feature retention ≥35%", "Composer session length within ±8% of baseline (allows for latency variance)", "Support ticket volume for 'AI reply' issues ≤5 tickets/1K users over 14 days", "Zero confirmed hallucinations, confidentiality breaches, or regulatory incidents", "NPS for feature (post-session survey, n≥50) ≥40" ], "pauseCriteria": "Acceptance rate <12% or retention <30% for 5+ consecutive days → pause expansion, maintain cohort, increase feedback collection", "killCriteria": "Acceptance rate <10% for 3+ consecutive days; support volume >8/1K users; any confidentiality incident; P95 latency >1200ms sustained" }, "phase3_regional_expansion": { "name": "US Regional Expansion (Weeks 4-5)", "cohort": "Expand Phase 2 cohort + non-regulated US mid-market (Enterprise and Growth plans only); exclude same groups as Phase 2", "trafficPercentage": "3.5% of MAU (16.8K users)", "duration": "14 days", "promotionCriteria": [ "Acceptance rate ≥16.5% (sustained over 7+ days)", "7-day retention ≥38%", "Composer latency P95 ≤850ms", "Support ticket escalation rate ≤4/1K users; zero critical incidents", "Model performance stable: suggestion relevance score (internal eval) ≥0.72" ], "investigateCriteria": "Acceptance rate dips to 15-16.5% or retention drops to 36-38% → increase monitoring frequency to daily, collect user feedback on UX friction, assess model drift" }, "phase4_segment_targeting": { "name": "Segment-Specific Targeting (Weeks 6-8)", "cohort": "Phase 3 + all US non-regulated users (exclude HIPAA, financial, custom data residency, escalations); segment by use case: Sales (primary), Customer Success (secondary), Operations (tertiary)", "trafficPercentage": "15% of MAU (72K users)", "duration": "14 days", "promotionCriteria": [ "Acceptance rate ≥17.5% across all segments; minimum 16% in lowest-performing segment", "7-day retention ≥40%", "No difference >2pp between desktop/mobile acceptance (ensure parity)", "Latency P95 ≤800ms sustained", "Segment-specific support volume <3/1K users", "GDPR-readiness confirmed: DPA addendum signed, EU shadow evaluation passing legal review" ], "investigateCriteria": "Mobile acceptance <14% while desktop >17% → investigate composer UX on mobile, assess if suggestion ranking differs" }, "phase5_eu_launch": { "name": "EU/GDPR Launch (Week 9+)", "cohort": "Phase 4 + EU users (only after DPA addendum signed and verified); exclude UK financial services and any HIPAA-equivalent regulated users", "trafficPercentage": "35% of MAU (168K users)", "duration": "21 days (longer due to regulatory context)", "promotionCriteria": [ "Acceptance rate ≥17% across US + EU", "7-day retention ≥40%", "Zero GDPR compliance incidents; audit trail confirms data residency compliance", "Support volume stable or declining vs. Phase 4", "User consent opt-in rate ≥60% (EU users must explicitly enable feature)" ], "killCriteria": "GDPR incident; acceptance rate <15%; retention <37%; EU support volume spike >5/1K users" }, "phase6_general_availability": { "name": "General Availability (Week 11+)", "cohort": "All users except: HIPAA/financial-regulated, free-tier, accounts with custom data-residency, active escalations, explicit AI opt-out", "trafficPercentage": "100% of eligible MAU (~420K users)", "duration": "Ongoing with staged roll-in (10% daily growth over 10 days to monitor infrastructure)", "promotionCriteria": [ "Acceptance rate ≥18% sustained over 14 days", "7-day retention ≥40%", "Latency P95 ≤800ms under load", "Support volume <2.5/1K users; escalation rate <0.8/1K" ], "ongoingKillCriteria": "Acceptance rate drops below 16% for 7+ days; retention <37% sustained; critical safety incident; model quality score <0.70" } }

{ "kill_immediate_rollback": [ { "metric": "Confidentiality/security incident", "threshold": "Any confirmed instance of credential exposure, PII leakage, or prompt injection success", "timeframe": "Immediate upon confirmation", "action": "Disable feature server-side, revert to Phase 1, notify Legal and affected users within 2 hours, forensic review before re-enabling" }, { "metric": "Model hallucination rate", "threshold": ">2% of sampled suggestions contain factually false or offensive content (manual review of 500-suggestion sample)", "timeframe": "Roll back immediately if observed", "action": "Pause feature, retrain model, shadow-test for 5 days before re-enabling" }, { "metric": "Composer latency spike", "threshold": "P95 latency >1500ms sustained for >2 hours", "timeframe": "Immediate", "action": "Disable AI suggestions server-side (feature remains, defaults to 'No suggestions available'), deploy fix, shadow re-test" }, { "metric": "Regulatory/compliance breach", "threshold": "Any GDPR, HIPAA, or contract-data-residency violation confirmed", "timeframe": "Immediate", "action": "Kill feature for affected cohort, escalate to Legal, notify customers within SLA" } ], "pause_halt_expansion": [ { "metric": "Acceptance rate decline", "threshold": "Drops to <12% (Phase 2) or <15% (Phase 3+) for 5+ consecutive days", "timeframe": "Evaluate daily; trigger pause if threshold met", "action": "Stop phase expansion, maintain current cohort, increase user feedback collection (surveys, support review), investigate UX friction or model drift" }, { "metric": "Feature retention collapse", "threshold": "7-day retention <30% (Phase 2) or <37% (Phase 3+)", "timeframe": "Evaluate every 48 hours", "action": "Pause expansion, dig into churn cohorts, assess if suggestions lack personalization or relevance" }, { "metric": "Support volume spike", "threshold": ">8/1K users in Phase 2, >5/1K in Phase 3+ reporting AI-related issues in any 7-day window", "timeframe": "Real-time alert; evaluate daily", "action": "Pause expansion, triage support backlog to identify root cause, prioritize fixes" } ], "investigate_increase_monitoring": [ { "metric": "Acceptance rate at risk", "threshold": "15-16.5% in Phase 3 (within 1.5pp of pause threshold)", "timeframe": "Daily check", "action": "Increase monitoring to hourly; collect user feedback on why suggestions are rejected; assess if personalization is weak; do not expand but maintain cohort" }, { "metric": "Latency degradation", "threshold": "P95 creeps above 900ms but below 1200ms", "timeframe": "Daily monitoring", "action": "Profile model inference, check for caching issues, optimize tokenization; increase frequency of performance monitoring to hourly" }, { "metric": "Segment performance variance", "threshold": "Difference in acceptance rate >2pp between segments (e.g., Sales 18%, CS 16%)", "timeframe": "Weekly cohort analysis", "action": "Investigate whether segment-specific training data or personas are needed; do not expand to underperforming segment yet; A/B test UX variations" }, { "metric": "Mobile vs. Desktop gap", "threshold": "Mobile acceptance <14% while desktop >17%", "timeframe": "Weekly check", "action": "Deep dive into mobile composer UX, latency differences, and suggestion ranking; do not expand mobile cohort until parity achieved" } ] }

{ "quality_metrics": { "suggestion_relevance_score": { "definition": "Manual expert review: 3 reviewers rate 100-suggestion sample weekly on relevance to thread context and user tone match (1-5 scale); threshold ≥4.0", "dataSource": "Internal evaluation dashboard; labeled dataset in data lake", "cadence": "Weekly (Phase 1-3), bi-weekly (Phase 4+)", "owner": "ML/Product", "action": "Score <3.8 → investigate model drift, retrain; score <3.5 → shadow test before re-enabling" }, "hallucination_rate": { "definition": "Percentage of suggestions containing factually false, nonsensical, or offensive content; sampled and manually reviewed", "dataSource": "User feedback flags + weekly expert audit of 500-suggestion random sample", "cadence": "Real-time alerts on user reports; full audit weekly", "owner": "Safety/ML", "action": ">2% → pause expansion; >5% → rollback" }, "confidentiality_exposure": { "definition": "Count of incidents where model suggests information that should remain confidential (credentials, internal names, sensitive business data)", "dataSource": "User reports, support tickets tagged 'confidentiality concern', automated content filtering flagging PII-like patterns in suggestions", "cadence": "Real-time alert system", "owner": "Legal/Safety", "action": "Any confirmed incident → immediate rollback; suspected incident → shadow-only mode until cleared" } }, "business_metrics": { "acceptance_rate": { "definition": "Percentage of users who select (or partially use) at least one suggestion in a composer session", "dataSource": "Event logging: suggestion_displayed → suggestion_clicked or suggestion_edited", "cadence": "Daily dashboard; promotion gates check 7-day rolling average", "owner": "Product", "action": "Hit targets = proceed; below pause threshold = investigate; below kill threshold = rollback" }, "7day_feature_retention": { "definition": "Percentage of users who open composer with feature enabled and trigger suggestion generation in both week 1 and week 2", "dataSource": "User cohort tracking; weekly active feature users query", "cadence": "Daily tracking; promotion gates check at day 7 and day 14", "owner": "Product/Analytics", "action": "Sustained retention <target → pause expansion; investigate churn drivers via exit surveys" }, "composer_session_length": { "definition": "Time from composer open to send; baseline measured pre-launch; track ±5% tolerance", "dataSource": "Event logging; time_composer_open → time_message_sent", "cadence": "Daily P50, P95; weekly trend analysis", "owner": "Product/Performance", "action": "Session length increases >8% → investigate latency impact; may indicate friction in UX" }, "support_volume": { "definition": "Support tickets mentioning 'reply suggestion,' 'AI reply,' or 'composer' per 1K active users per week", "dataSource": "Support ticket system filtered by keywords + AI feature tags", "cadence": "Daily count; weekly trend; escalation spike alerts", "owner": "CS/Support", "action": "Spike >phase threshold → pause; correlate with acceptance drop to identify systemic UX issues" }, "nps_cohort": { "definition": "Net Promoter Score collected via post-session survey for users who interacted with suggestions (n≥30/week)", "dataSource": "In-app survey (optional, shown to 5% of feature users)", "cadence": "Weekly aggregate; target ≥40", "owner": "Product", "action": "NPS <30 for 2+ weeks → investigate satisfaction drivers, may correlate with acceptance decline" } }, "safety_and_compliance_metrics": { "gdpr_compliance": { "definition": "Audit trail confirming: (1) EU users see consent prompt before feature activation, (2) data residency constraints honored, (3) DPA addendum in place", "dataSource": "Consent event logs; infrastructure config; Legal contract tracking", "cadence": "Pre-Phase 5 (legal sign-off); weekly spot-check post-Phase 5", "owner": "Legal/Privacy", "action": "Any breach → kill feature for EU cohort; fix and re-test before re-enabling" }, "hipaa_regulated_user_exposure": { "definition": "Count of users flagged as HIPAA-regulated who somehow received AI suggestions (exclusion list bypass)", "dataSource": "Real-time: user account plan check before feature render; daily audit of feature_active flag vs. user compliance tags", "cadence": "Real-time gate check; daily audit query", "owner": "Legal/Eng", "action": "Any exposure → immediate rollback for that user, review exclusion logic" }, "prompt_injection_attempts": { "definition": "Incoming email content that contains patterns consistent with prompt injection attacks (e.g., 'ignore previous instructions')", "dataSource": "Content filter logs; flagged text samples reviewed by security team", "cadence": "Weekly review; real-time alerts if injection attempt succeeds in generating malicious suggestion", "owner": "Security/ML", "action": "Successful injection (user reports malicious output) → immediate shadow-only mode; patch tokenization/filtering" }, "model_drift": { "definition": "Statistical comparison of suggestion quality metrics (relevance, tone match, acceptance) between baseline (Phase 1) and current phase; flagged if <5% decline", "dataSource": "Internal quality score + acceptance rate trend", "cadence": "Bi-weekly analysis", "owner": "ML", "action": "Drift detected → retraining cycle; pause expansion until drift reversed" } }, "shadow_evaluation_and_reversibility": { "shadow_mode_phases": "Phase 3 (10% of traffic): Run model on incoming emails, generate suggestions, store in logs but do NOT show to users; compare model output to expert baseline evaluations; roll forward to Phase 4 only if shadow quality ≥0.72", "dataSource": "Shadow suggestion logs; expert manual eval of 300-suggestion sample", "cadence": "Daily in shadow; eval results at end of Phase 3", "reversibility": "Model version decoupled from feature flag; if model quality drops, revert to previous version server-side within 5 minutes without rolling back feature; rollback testing done weekly", "dual_run_capability": "Phase 2-4 can toggle between AI-enabled and AI-disabled for A/B comparison; acceptance and retention rates measured side-by-side to isolate AI impact" } }