christianV/CrazeAnalytix
1
0
Fork 0
mirror of https://github.com/christianvidalwolf-prog/CrazeAnalytix.git synced 2026-08-03 13:55:23 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: rebuild Experiments tab with Difference-in-Differences analysis and Bayesian verdicts

Browse Source 
Replace the basic CRUD experiment tracker with a scientifically rigorous A/B testing system:

- Add DiD analysis engine (services/experimentAnalysis.ts) that computes treatment vs control
  group comparisons across 7 metrics (units, sessions, CVR, CTR, ROAS, revenue, ACOS)
- Implement Bayesian verdict system (Winner/Loser/Inconclusive) using posterior probability
  with normal CDF approximation (Abramowitz & Stegun erf, no external deps)
- Build counterfactual time series for trend charts (actual vs estimated without change)
- Rewrite ExperimentsView as single component with 3 inline sub-views (list, detail, create)
  replacing the previous modal-based ExperimentDetail and ExperimentForm
- Add control group support, change annotations (before→after diffs), and SEO experiment type
- New types: ExperimentChangeAnnotation, DiDMetricResult, DifferenceInDifferencesResult,
  ExperimentVerdict
- Simplify App.tsx by removing experiment modal state (5 useState hooks eliminated)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Christian Vidal Wolf
2026-02-23 15:42:40 +01:00
co-authored by Claude Opus 4.6
parent 7ee33671cc
commit 24df5935cc
6 changed files with 1495 additions and 433 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/experiments.ts
+67 -140
View File
@@ -5,12 +5,13 @@ import {
ActiveExperiment,
ExperimentType,
ExperimentStatus,
ExperimentVerdict,
CombinedKPIs,
} from '../types';
import { CombinedKPIs } from '../types';
const API_BASE = '/api/experiments';
// ============ Helper Functions ============
// ============ ASIN Resolution ============
export const getExperimentAsins = (experimentAsins: string[], salesData: CombinedKPIs[]): Set<string> => {
const explicitAsins = new Set<string>();
@@ -59,27 +60,22 @@ export const updateExperiment = async (
id: string,
updates: Partial<Experiment>
): Promise<Experiment> => {
try {
const response = await fetch(`${API_BASE}?id=${id}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(updates),
});
const response = await fetch(`${API_BASE}?id=${id}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(updates),
});
if (!response.ok) {
let errText = await response.text();
try {
const errJson = JSON.parse(errText);
errText = errJson.error || errText;
} catch (e) { }
throw new Error(errText);
}
return response.json();
} catch (err: any) {
console.error('API Error in updateExperiment:', err);
throw err;
if (!response.ok) {
let errText = await response.text();
try {
const errJson = JSON.parse(errText);
errText = errJson.error || errText;
} catch (e) { }
throw new Error(errText);
}
return response.json();
};
export const deleteExperiment = async (id: string): Promise<void> => {
@@ -95,9 +91,7 @@ export const deleteExperiment = async (id: string): Promise<void> => {
export const getExperiment = async (id: string): Promise<Experiment | null> => {
const response = await fetch(`${API_BASE}?id=${id}`);
if (!response.ok) return null;
return response.json();
};
@@ -141,7 +135,6 @@ export const listExperiments = async (
const data = await response.json();
// Transform to ExperimentListItem
return data.map((exp: any) => {
const today = new Date();
const startDate = new Date(exp.start_date);
@@ -161,11 +154,14 @@ export const listExperiments = async (
status: exp.status,
asin_count: exp.asins?.length || 0,
asins: exp.asins || [],
control_asin_count: exp.control_asins?.length || 0,
marketplace: exp.marketplace,
start_date: exp.start_date,
end_date: exp.end_date,
progress_percent: Math.round(progressPercent),
primary_metric: exp.primary_metric,
verdict: exp.verdict,
verdict_probability: exp.verdict_probability,
actual_lift_percent: exp.actual_lift_percent,
owner: exp.owner,
};
@@ -185,7 +181,6 @@ export const getActiveExperiments = async (
const data = await response.json();
const today = new Date().toISOString().split('T')[0];
// Filter experiments that are currently active
const activeExperiments = data.filter((exp: any) =>
exp.status === 'active' &&
exp.start_date <= today &&
@@ -203,7 +198,7 @@ export const getActiveExperiments = async (
: undefined;
const activeExp: ActiveExperiment = {
asin: '', // placeholder
asin: '',
experiment_id: exp.id,
experiment_name: exp.name,
type: exp.type,
@@ -223,120 +218,7 @@ export const getActiveExperiments = async (
return map;
};
// ============ Performance Calculations ============
export const calculateExperimentPerformance = async (
experiment: Experiment,
salesData: CombinedKPIs[]
): Promise<{
baseline_units: number;
baseline_revenue: number;
baseline_gv: number;
baseline_cvr: number;
baseline_acos: number;
experiment_units: number;
experiment_revenue: number;
experiment_gv: number;
experiment_cvr: number;
experiment_acos: number;
actual_lift_percent: number;
}> => {
const parseLocalDate = (dateStr?: string) => {
if (!dateStr) return new Date();
const [y, m, d] = dateStr.split('T')[0].split('-');
return new Date(Number(y), Number(m) - 1, Number(d));
};
const startDate = parseLocalDate(experiment.start_date);
const endDate = experiment.end_date ? parseLocalDate(experiment.end_date) : new Date();
// Calculate baseline period (same duration before experiment)
const durationMs = endDate.getTime() - startDate.getTime();
const baselineStart = new Date(startDate.getTime() - durationMs);
const baselineEnd = startDate;
// Filter sales data for experiment ASINs
const asinSet = getExperimentAsins(experiment.asins, salesData);
const baselineData = salesData.filter(r => {
const recordDate = new Date(r.year, 0, 1 + (r.week - 1) * 7);
const mkt = (r.marketplace || (r as any).customer || '').toLowerCase();
const isMarket = !experiment.marketplace || experiment.marketplace === 'All' || mkt.includes(experiment.marketplace.toLowerCase());
return isMarket && asinSet.has(r.asin.toUpperCase()) &&
recordDate >= baselineStart && recordDate < baselineEnd;
});
const experimentData = salesData.filter(r => {
const recordDate = new Date(r.year, 0, 1 + (r.week - 1) * 7);
const mkt = (r.marketplace || (r as any).customer || '').toLowerCase();
const isMarket = !experiment.marketplace || experiment.marketplace === 'All' || mkt.includes(experiment.marketplace.toLowerCase());
return isMarket && asinSet.has(r.asin.toUpperCase()) &&
recordDate >= startDate && recordDate <= endDate;
});
const baseline_units = baselineData.reduce((sum, r) => sum + (r.unitsTotal ?? (r as any).units ?? 0), 0);
const baseline_revenue = baselineData.reduce((sum, r) => sum + (r.salesTotal ?? (r as any).sellOut ?? 0), 0);
const baseline_gv = baselineData.reduce((sum, r) => sum + (r.glanceViews || 0), 0);
const baseline_cvr = baseline_gv > 0 ? (baseline_units / baseline_gv) * 100 : 0;
const baseline_spend = baselineData.reduce((sum, r) => sum + (r.cost || 0), 0);
const baseline_acos = baseline_revenue > 0 ? (baseline_spend / baseline_revenue) * 100 : 0;
const experiment_units = experimentData.reduce((sum, r) => sum + (r.unitsTotal ?? (r as any).units ?? 0), 0);
const experiment_revenue = experimentData.reduce((sum, r) => sum + (r.salesTotal ?? (r as any).sellOut ?? 0), 0);
const experiment_gv = experimentData.reduce((sum, r) => sum + (r.glanceViews || 0), 0);
const experiment_cvr = experiment_gv > 0 ? (experiment_units / experiment_gv) * 100 : 0;
const experiment_spend = experimentData.reduce((sum, r) => sum + (r.cost || 0), 0);
const experiment_acos = experiment_revenue > 0 ? (experiment_spend / experiment_revenue) * 100 : 0;
let actual_lift_percent = 0;
if (experiment.primary_metric === 'cvr' && baseline_cvr > 0) {
actual_lift_percent = ((experiment_cvr - baseline_cvr) / baseline_cvr) * 100;
} else if (experiment.primary_metric === 'revenue' && baseline_revenue > 0) {
actual_lift_percent = ((experiment_revenue - baseline_revenue) / baseline_revenue) * 100;
} else if (experiment.primary_metric === 'acos' && baseline_acos > 0) {
actual_lift_percent = ((baseline_acos - experiment_acos) / Math.max(0.1, baseline_acos)) * 100; // inverted, lower is better
} else if (baseline_units > 0) {
actual_lift_percent = ((experiment_units - baseline_units) / baseline_units) * 100;
}
return {
baseline_units,
baseline_revenue,
baseline_gv,
baseline_cvr: Math.round(baseline_cvr * 10) / 10,
baseline_acos: Math.round(baseline_acos * 10) / 10,
experiment_units,
experiment_revenue,
experiment_gv,
experiment_cvr: Math.round(experiment_cvr * 10) / 10,
experiment_acos: Math.round(experiment_acos * 10) / 10,
actual_lift_percent: Math.round(actual_lift_percent * 10) / 10,
};
};
export const updateExperimentResults = async (
experimentId: string,
salesData: CombinedKPIs[]
): Promise<Experiment> => {
const experiment = await getExperiment(experimentId);
if (!experiment) throw new Error('Experiment not found');
const performance = await calculateExperimentPerformance(experiment, salesData);
return updateExperiment(experimentId, {
...performance,
});
};
// ============ Helper Functions ============
function getMonthNumber(monthStr: string): number {
const months: Record<string, number> = {
'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6,
'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12
};
return months[monthStr.split('-')[0]] || 1;
}
// ============ Display Helpers ============
export const getExperimentStatusColor = (status: ExperimentStatus): string => {
switch (status) {
@@ -354,6 +236,7 @@ export const getExperimentTypeColor = (type: ExperimentType): string => {
case 'advertising': return 'bg-purple-500/20 text-purple-400 border-purple-500/30';
case 'content': return 'bg-pink-500/20 text-pink-400 border-pink-500/30';
case 'promotion': return 'bg-orange-500/20 text-orange-400 border-orange-500/30';
case 'seo': return 'bg-cyan-500/20 text-cyan-400 border-cyan-500/30';
default: return 'bg-slate-500/20 text-slate-400';
}
};
@@ -364,6 +247,50 @@ export const getExperimentIcon = (type: ExperimentType): string => {
case 'advertising': return '📢';
case 'content': return '📝';
case 'promotion': return '🏷️';
case 'seo': return '🔍';
default: return '🧪';
}
};
export const getVerdictColor = (verdict?: ExperimentVerdict): string => {
switch (verdict) {
case 'winner': return 'bg-emerald-500/15 text-emerald-400 border-emerald-500/30';
case 'loser': return 'bg-red-500/15 text-red-400 border-red-500/30';
case 'inconclusive': return 'bg-slate-500/15 text-slate-400 border-slate-500/30';
default: return 'bg-slate-800/50 text-slate-500 border-slate-700';
}
};
export const getVerdictLabel = (verdict?: ExperimentVerdict): string => {
switch (verdict) {
case 'winner': return 'Winner';
case 'loser': return 'Loser';
case 'inconclusive': return 'Inconclusive';
default: return 'Pending';
}
};
export const METRIC_LABELS: Record<string, string> = {
units: 'Units Sold',
sessions: 'Sessions',
cvr: 'Conversion Rate',
ctr: 'Click-Through Rate',
roas: 'ROAS',
revenue: 'Revenue',
acos: 'ACOS',
};
export const formatMetricValue = (value: number, metric: string): string => {
switch (metric) {
case 'cvr':
case 'ctr':
case 'acos':
return `${value.toFixed(1)}%`;
case 'roas':
return `${value.toFixed(2)}x`;
case 'revenue':
return `€${value.toLocaleString('de-DE', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`;
default:
return value.toLocaleString('de-DE', { minimumFractionDigits: 0, maximumFractionDigits: 0 });
}
};