feat: rebuild Experiments tab with Difference-in-Differences analysis and Bayesian verdicts

Replace the basic CRUD experiment tracker with a scientifically rigorous A/B testing system:

- Add DiD analysis engine (services/experimentAnalysis.ts) that computes treatment vs control
  group comparisons across 7 metrics (units, sessions, CVR, CTR, ROAS, revenue, ACOS)
- Implement Bayesian verdict system (Winner/Loser/Inconclusive) using posterior probability
  with normal CDF approximation (Abramowitz & Stegun erf, no external deps)
- Build counterfactual time series for trend charts (actual vs estimated without change)
- Rewrite ExperimentsView as single component with 3 inline sub-views (list, detail, create)
  replacing the previous modal-based ExperimentDetail and ExperimentForm
- Add control group support, change annotations (before→after diffs), and SEO experiment type
- New types: ExperimentChangeAnnotation, DiDMetricResult, DifferenceInDifferencesResult,
  ExperimentVerdict
- Simplify App.tsx by removing experiment modal state (5 useState hooks eliminated)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Christian Vidal Wolf
2026-02-23 15:42:40 +01:00
co-authored by Claude Opus 4.6
parent 7ee33671cc
commit 24df5935cc
6 changed files with 1495 additions and 433 deletions
+9 -54
View File
@@ -23,8 +23,7 @@ const AdsPerformance = lazy(() => import('./components/AdsPerformance'));
const ForecastView = lazy(() => import('./components/ForecastView'));
const VendorDataView = lazy(() => import('./components/VendorDataView'));
const ExperimentsView = lazy(() => import('./components/ExperimentsView'));
const ExperimentDetail = lazy(() => import('./components/ExperimentDetail'));
const ExperimentForm = lazy(() => import('./components/ExperimentForm'));
// ExperimentDetail and ExperimentForm are now integrated into ExperimentsView
// Loading fallback component
const LoadingSpinner = () => (
@@ -64,17 +63,7 @@ const App: React.FC = () => {
// Experiments state
const [experimentMap, setExperimentMap] = useState<Map<string, ActiveExperiment[]>>(new Map());
const [selectedExperimentId, setSelectedExperimentId] = useState<string | null>(null);
const [showCreateExperiment, setShowCreateExperiment] = useState(false);
const [preselectedAsins, setPreselectedAsins] = useState<string[]>([]);
const [preselectedMarketplace, setPreselectedMarketplace] = useState<string>('');
const [preselectedLine, setPreselectedLine] = useState<string>('');
// Available product lines (for experiment form)
const availableProductLines = useMemo(() => {
const lines = new Set(rawData.map(r => r.line).filter(Boolean));
return Array.from(lines).sort();
}, [rawData]);
// Experiment modal state removed — detail/create are now inline in ExperimentsView
// Modal State
const [isDataModalOpen, setIsDataModalOpen] = useState(false);
@@ -349,13 +338,10 @@ const App: React.FC = () => {
setView('ads');
}, []);
// Create experiment for a specific line
const handleCreateExperimentForLine = useCallback((line: string) => {
if (!line) return;
setPreselectedLine(line);
setPreselectedMarketplace(filters.customer[0] || '');
setShowCreateExperiment(true);
}, [filters.customer]);
// Create experiment for a specific line — now handled inline by ExperimentsView
const handleCreateExperimentForLine = useCallback((_line: string) => {
setView('experiments');
}, []);
// 1. Initial Load from Cache (IndexedDB) or Auto-Fetch Permanent URL
useEffect(() => {
@@ -902,7 +888,7 @@ const App: React.FC = () => {
velocityMap={velocityMap}
buyBoxLostMap={buyBoxLostMap}
experimentMap={experimentMap}
onOpenExperiment={(id) => setSelectedExperimentId(id)}
onOpenExperiment={() => setView('experiments')}
/>
</div>
</Suspense>
@@ -962,13 +948,8 @@ const App: React.FC = () => {
<Suspense fallback={<LoadingSpinner />}>
<div className={view === 'experiments' ? '' : 'hidden'}>
<ExperimentsView
onOpenDetail={(id) => setSelectedExperimentId(id)}
onOpenCreate={() => {
setPreselectedAsins([]);
setPreselectedMarketplace(filters.customer[0] || '');
setShowCreateExperiment(true);
}}
salesData={unfilteredCombinedData}
onExperimentsFetch={handleExperimentsFetch}
/>
</div>
</Suspense>
@@ -1036,33 +1017,7 @@ const App: React.FC = () => {
)
}
{/* Experiment Detail Modal */}
{selectedExperimentId && (
<ExperimentDetail
experimentId={selectedExperimentId}
onClose={() => setSelectedExperimentId(null)}
salesData={unfilteredCombinedData}
/>
)}
{/* Create Experiment Modal */}
{showCreateExperiment && (
<ExperimentForm
onClose={() => {
setShowCreateExperiment(false);
setPreselectedAsins([]);
setPreselectedMarketplace('');
setPreselectedLine('');
}}
onSuccess={() => {
handleExperimentsFetch();
}}
initialAsins={preselectedAsins}
initialMarketplace={preselectedMarketplace}
initialLine={preselectedLine}
availableLines={availableProductLines}
/>
)}
{/* Experiment modals removed — detail/create are now inline in ExperimentsView */}
</div >
);