feat: rebuild Experiments tab with Difference-in-Differences analysis and Bayesian verdicts

Replace the basic CRUD experiment tracker with a scientifically rigorous A/B testing system:

- Add DiD analysis engine (services/experimentAnalysis.ts) that computes treatment vs control
  group comparisons across 7 metrics (units, sessions, CVR, CTR, ROAS, revenue, ACOS)
- Implement Bayesian verdict system (Winner/Loser/Inconclusive) using posterior probability
  with normal CDF approximation (Abramowitz & Stegun erf, no external deps)
- Build counterfactual time series for trend charts (actual vs estimated without change)
- Rewrite ExperimentsView as single component with 3 inline sub-views (list, detail, create)
  replacing the previous modal-based ExperimentDetail and ExperimentForm
- Add control group support, change annotations (before→after diffs), and SEO experiment type
- New types: ExperimentChangeAnnotation, DiDMetricResult, DifferenceInDifferencesResult,
  ExperimentVerdict
- Simplify App.tsx by removing experiment modal state (5 useState hooks eliminated)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Christian Vidal Wolf
2026-02-23 15:42:40 +01:00
co-authored by Claude Opus 4.6
parent 7ee33671cc
commit 24df5935cc
6 changed files with 1495 additions and 433 deletions
+37 -5
View File
@@ -281,9 +281,31 @@ export interface VendorCSVRow {
}
// Experiment Tracking Types
export type ExperimentType = 'pricing' | 'advertising' | 'content' | 'promotion';
export type ExperimentType = 'pricing' | 'advertising' | 'content' | 'promotion' | 'seo';
export type ExperimentStatus = 'planned' | 'active' | 'completed' | 'paused';
export type ExperimentMetric = 'units' | 'revenue' | 'acos' | 'ctr' | 'cvr' | 'bsr';
export type ExperimentMetric = 'units' | 'sessions' | 'cvr' | 'ctr' | 'roas' | 'revenue' | 'acos';
export type ExperimentVerdict = 'winner' | 'loser' | 'inconclusive';
export interface ExperimentChangeAnnotation {
field: string;
before_value: string;
after_value: string;
}
export interface DiDMetricResult {
treatment_before: number;
treatment_after: number;
control_before: number;
control_after: number;
did_estimate: number;
lift_percent: number;
posterior_prob_positive: number;
}
export interface DifferenceInDifferencesResult {
metrics: Record<string, DiDMetricResult>;
computed_at: string;
}
export interface Experiment {
id: string;
@@ -292,6 +314,7 @@ export interface Experiment {
type: ExperimentType;
status: ExperimentStatus;
asins: string[];
control_asins: string[];
marketplace: string;
start_date: string;
end_date?: string;
@@ -300,23 +323,27 @@ export interface Experiment {
hypothesis?: string;
primary_metric: ExperimentMetric;
target_lift_percent?: number;
changes: ExperimentChangeAnnotation[];
// Results
// Legacy results (kept for backward compat)
baseline_units?: number;
baseline_revenue?: number;
baseline_gv?: number;
baseline_cvr?: number;
baseline_acos?: number;
experiment_units?: number;
experiment_revenue?: number;
experiment_gv?: number;
experiment_cvr?: number;
experiment_acos?: number;
actual_lift_percent?: number;
statistical_significance?: number;
// DiD results
did_results?: DifferenceInDifferencesResult;
verdict?: ExperimentVerdict;
verdict_probability?: number;
learnings?: string;
owner?: string;
}
@@ -326,12 +353,14 @@ export interface ExperimentCreateInput {
description?: string;
type: ExperimentType;
asins: string[];
control_asins: string[];
marketplace: string;
start_date: string;
end_date?: string;
hypothesis?: string;
primary_metric: ExperimentMetric;
target_lift_percent?: number;
changes: ExperimentChangeAnnotation[];
owner?: string;
}
@@ -342,11 +371,14 @@ export interface ExperimentListItem {
status: ExperimentStatus;
asin_count: number;
asins: string[];
control_asin_count: number;
marketplace: string;
start_date: string;
end_date?: string;
progress_percent: number;
primary_metric: ExperimentMetric;
verdict?: ExperimentVerdict;
verdict_probability?: number;
actual_lift_percent?: number;
owner?: string;
}