fix: replace volume-based similarity with per-dimension comparison for near-duplicate detection

Compares each sorted dimension individually (max relative diff ≤15% per axis)
instead of total cubic volume. Catches cases like 29x15x20 vs 26x16x19.5
that volume comparison misses.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Christian Vidal Wolf
2026-04-07 10:51:10 +02:00
co-authored by Claude Sonnet 4.6
parent f44e4314e1
commit 9753958761
+22 -12
View File
@@ -110,6 +110,16 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
}, [groups, showOnlyInconsistent]); }, [groups, showOnlyInconsistent]);
const nearDuplicateClusters = useMemo((): NearDuplicateCluster[] => { const nearDuplicateClusters = useMemo((): NearDuplicateCluster[] => {
// Two groups are "similar" if every sorted dimension pair differs by ≤15%
// (keys are already sorted ascending, e.g. "15x20x29")
const THRESHOLD = 0.15;
const maxRelativeDiff = (keyA: string, keyB: string): number => {
const a = keyA.split('x').map(Number);
const b = keyB.split('x').map(Number);
return Math.max(...a.map((v, i) => Math.abs(v - b[i]) / Math.max(v, b[i], 0.001)));
};
const validGroups = groups.filter(g => g.volume > 0); const validGroups = groups.filter(g => g.volume > 0);
const assignedKeys = new Set<string>(); const assignedKeys = new Set<string>();
const clusters: NearDuplicateCluster[] = []; const clusters: NearDuplicateCluster[] = [];
@@ -125,13 +135,9 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
const b = validGroups[j]; const b = validGroups[j];
if (assignedKeys.has(b.key)) continue; if (assignedKeys.has(b.key)) continue;
// Check if b is within 5% volume of every group already in the cluster // b must be similar to every group already in the cluster
const isNear = cluster.every(g => { const isSimilar = cluster.every(g => maxRelativeDiff(g.key, b.key) <= THRESHOLD);
const maxVol = Math.max(g.volume, b.volume); if (isSimilar) {
return Math.abs(g.volume - b.volume) / maxVol <= 0.05;
});
if (isNear) {
cluster.push(b); cluster.push(b);
assignedKeys.add(b.key); assignedKeys.add(b.key);
} }
@@ -139,9 +145,13 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
if (cluster.length >= 2) { if (cluster.length >= 2) {
const volumes = cluster.map(g => g.volume); const volumes = cluster.map(g => g.volume);
const maxVol = Math.max(...volumes); // Max dimension-wise diff across all pairs in the cluster
const minVol = Math.min(...volumes); let maxDiffPct = 0;
const maxDiffPct = ((maxVol - minVol) / maxVol) * 100; for (let x = 0; x < cluster.length; x++) {
for (let y = x + 1; y < cluster.length; y++) {
maxDiffPct = Math.max(maxDiffPct, maxRelativeDiff(cluster[x].key, cluster[y].key) * 100);
}
}
clusters.push({ groups: cluster, volumes, maxDiffPct }); clusters.push({ groups: cluster, volumes, maxDiffPct });
} }
} }
@@ -244,7 +254,7 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
<Link2 className="w-4 h-4 text-violet-400" /> <Link2 className="w-4 h-4 text-violet-400" />
<span className="text-sm font-semibold text-violet-300">Possible data entry errors</span> <span className="text-sm font-semibold text-violet-300">Possible data entry errors</span>
<span className="text-xs text-slate-500 bg-slate-900 px-2 py-0.5 rounded-full border border-slate-700"> <span className="text-xs text-slate-500 bg-slate-900 px-2 py-0.5 rounded-full border border-slate-700">
{nearDuplicateClusters.length} cluster{nearDuplicateClusters.length !== 1 ? 's' : ''} within 5% volume {nearDuplicateClusters.length} cluster{nearDuplicateClusters.length !== 1 ? 's' : ''} with similar dimensions (15% per axis)
</span> </span>
</div> </div>
{nearDuplicateClusters.map((cluster, ci) => ( {nearDuplicateClusters.map((cluster, ci) => (
@@ -266,7 +276,7 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
</span> </span>
))} ))}
<span className="text-xs text-violet-400/70"> <span className="text-xs text-violet-400/70">
diff. {cluster.maxDiffPct.toFixed(1)}% max axis diff {cluster.maxDiffPct.toFixed(1)}%
</span> </span>
</div> </div>
</button> </button>