diff --git a/services/experimentAnalysis.ts b/services/experimentAnalysis.ts index 774c575..cc24e2a 100644 --- a/services/experimentAnalysis.ts +++ b/services/experimentAnalysis.ts @@ -40,7 +40,6 @@ interface ComputedWeeklyMetrics extends WeeklyMetrics { cvr: number; ctr: number; roas: number; - fraction?: number; } function getISOWeekStart(year: number, week: number): number { @@ -126,67 +125,32 @@ function splitPeriods( startTs: number, endTs: number, beforeStartTs: number, - beforeEndTs: number // Added to support separated custom periods + beforeEndTs: number ): { before: ComputedWeeklyMetrics[]; after: ComputedWeeklyMetrics[] } { const before: ComputedWeeklyMetrics[] = []; const after: ComputedWeeklyMetrics[] = []; for (const w of data) { const weekStartTs = w.timestamp; - const weekEndTs = w.timestamp + 6 * 86400000 + 86399999; // End of the 7th day + const weekEndTs = w.timestamp + 6 * 86400000 + 86399999; - // A week overlaps the treatment period if it starts before the period ends, - // AND ends after the period starts. const overlapsAfter = weekEndTs >= startTs && weekStartTs <= endTs; const overlapsBefore = weekEndTs >= beforeStartTs && weekStartTs < beforeEndTs; - // We prioritize assigning to 'after' so that we capture all units happening - // closely around the experiment dates for accurate reporting. if (overlapsAfter) { - const overlapStart = Math.max(weekStartTs, startTs); - const overlapEnd = Math.min(weekEndTs, endTs); - const overlapFraction = Math.max(0, Math.min(1, (overlapEnd - overlapStart) / (7 * 86400000))); - - if (overlapFraction > 0) { - after.push({ - ...w, - units: w.units * overlapFraction, - revenue: w.revenue * overlapFraction, - sessions: w.sessions * overlapFraction, - cost: w.cost * overlapFraction, - clicks: w.clicks * overlapFraction, - impressions: w.impressions * overlapFraction, - fraction: overlapFraction - }); - } + after.push(w); } else if (overlapsBefore) { - const overlapStart = Math.max(weekStartTs, beforeStartTs); - const overlapEnd = Math.min(weekEndTs, beforeEndTs); - const overlapFraction = Math.max(0, Math.min(1, (overlapEnd - overlapStart) / (7 * 86400000))); - - if (overlapFraction > 0) { - before.push({ - ...w, - units: w.units * overlapFraction, - revenue: w.revenue * overlapFraction, - sessions: w.sessions * overlapFraction, - cost: w.cost * overlapFraction, - clicks: w.clicks * overlapFraction, - impressions: w.impressions * overlapFraction, - fraction: overlapFraction - }); - } + before.push(w); } } return { before, after }; } -function avgMetric(data: ComputedWeeklyMetrics[], metric: string): number { - if (data.length === 0) return 0; +function avgMetric(data: ComputedWeeklyMetrics[], metric: string, durationWeeks: number): number { + if (data.length === 0 || durationWeeks <= 0) return 0; const sum = data.reduce((s, w) => s + getMetricValue(w, metric), 0); - const totalFraction = data.reduce((s, w) => s + (w.fraction || 1), 0); - return totalFraction > 0 ? sum / totalFraction : 0; + return sum / durationWeeks; } const METRICS = ['units', 'sessions', 'cvr', 'ctr', 'roas', 'revenue', 'acos']; @@ -198,12 +162,14 @@ function computeMetricDiD( controlBefore: ComputedWeeklyMetrics[], controlAfter: ComputedWeeklyMetrics[], metric: string, - hasControlGroup: boolean + hasControlGroup: boolean, + treatmentDurationWeeks: number, + baselineDurationWeeks: number ): DiDMetricResult { - const tBefore = avgMetric(treatmentBefore, metric); - const tAfter = avgMetric(treatmentAfter, metric); - const cBefore = hasControlGroup ? avgMetric(controlBefore, metric) : 0; - const cAfter = hasControlGroup ? avgMetric(controlAfter, metric) : 0; + const tBefore = avgMetric(treatmentBefore, metric, baselineDurationWeeks); + const tAfter = avgMetric(treatmentAfter, metric, treatmentDurationWeeks); + const cBefore = hasControlGroup ? avgMetric(controlBefore, metric, baselineDurationWeeks) : 0; + const cAfter = hasControlGroup ? avgMetric(controlAfter, metric, treatmentDurationWeeks) : 0; let didEstimate: number; if (hasControlGroup) { @@ -296,12 +262,16 @@ export function computeDiD( ? splitPeriods(controlWeekly, startTs, endTs, beforeStartTs, beforeEndTs) : { before: [] as ComputedWeeklyMetrics[], after: [] as ComputedWeeklyMetrics[] }; + const treatmentDurationWeeks = Math.max(1, Math.round((endTs - startTs) / (7 * 86400000))); + const baselineDurationWeeks = Math.max(1, Math.round((beforeEndTs - beforeStartTs) / (7 * 86400000))); + const metrics: Record = {}; for (const metric of METRICS) { metrics[metric] = computeMetricDiD( tSplit.before, tSplit.after, cSplit.before, cSplit.after, - metric, hasControlGroup + metric, hasControlGroup, + treatmentDurationWeeks, baselineDurationWeeks ); } @@ -366,7 +336,8 @@ export function buildCounterfactualSeries( if (!hasControlGroup) { const { before: beforeData } = splitPeriods(treatmentWeekly, startTs, endTs, beforeStartTs, beforeEndTs); - const preAvg = avgMetric(beforeData, metric); + const baselineDurationWeeks = Math.max(1, Math.round((beforeEndTs - beforeStartTs) / (7 * 86400000))); + const preAvg = avgMetric(beforeData, metric, baselineDurationWeeks); return treatmentWeekly.map(w => ({ week: w.week, @@ -376,14 +347,18 @@ export function buildCounterfactualSeries( })); } + // Calculate variances for Bayesian update + const baselineDurationWeeks = Math.max(1, Math.round((beforeEndTs - beforeStartTs) / (7 * 86400000))); + const treatmentDurationWeeks = Math.max(1, Math.round((endTs - startTs) / (7 * 86400000))); + const controlAsinSet = getExperimentAsins(experiment.control_asins, salesData); const controlWeekly = aggregateWeeklyMetrics(controlAsinSet, salesData, experiment.marketplace); const { before: tBeforeData } = splitPeriods(treatmentWeekly, startTs, endTs, beforeStartTs, beforeEndTs); const { before: cBeforeData } = splitPeriods(controlWeekly, startTs, endTs, beforeStartTs, beforeEndTs); - const tPreAvg = avgMetric(tBeforeData, metric); - const cPreAvg = avgMetric(cBeforeData, metric); + const tPreAvg = avgMetric(tBeforeData, metric, baselineDurationWeeks); + const cPreAvg = avgMetric(cBeforeData, metric, baselineDurationWeeks); // Build a map of control weekly values const controlMap = new Map();