mirror of
https://github.com/christianvidalwolf-prog/Craze-Data-check.git
synced 2026-08-03 22:15:23 +02:00
fix: replace volume-based similarity with per-dimension comparison for near-duplicate detection
Compares each sorted dimension individually (max relative diff ≤15% per axis) instead of total cubic volume. Catches cases like 29x15x20 vs 26x16x19.5 that volume comparison misses. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
f44e4314e1
commit
9753958761
@@ -110,6 +110,16 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
|
|||||||
}, [groups, showOnlyInconsistent]);
|
}, [groups, showOnlyInconsistent]);
|
||||||
|
|
||||||
const nearDuplicateClusters = useMemo((): NearDuplicateCluster[] => {
|
const nearDuplicateClusters = useMemo((): NearDuplicateCluster[] => {
|
||||||
|
// Two groups are "similar" if every sorted dimension pair differs by ≤15%
|
||||||
|
// (keys are already sorted ascending, e.g. "15x20x29")
|
||||||
|
const THRESHOLD = 0.15;
|
||||||
|
|
||||||
|
const maxRelativeDiff = (keyA: string, keyB: string): number => {
|
||||||
|
const a = keyA.split('x').map(Number);
|
||||||
|
const b = keyB.split('x').map(Number);
|
||||||
|
return Math.max(...a.map((v, i) => Math.abs(v - b[i]) / Math.max(v, b[i], 0.001)));
|
||||||
|
};
|
||||||
|
|
||||||
const validGroups = groups.filter(g => g.volume > 0);
|
const validGroups = groups.filter(g => g.volume > 0);
|
||||||
const assignedKeys = new Set<string>();
|
const assignedKeys = new Set<string>();
|
||||||
const clusters: NearDuplicateCluster[] = [];
|
const clusters: NearDuplicateCluster[] = [];
|
||||||
@@ -125,13 +135,9 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
|
|||||||
const b = validGroups[j];
|
const b = validGroups[j];
|
||||||
if (assignedKeys.has(b.key)) continue;
|
if (assignedKeys.has(b.key)) continue;
|
||||||
|
|
||||||
// Check if b is within 5% volume of every group already in the cluster
|
// b must be similar to every group already in the cluster
|
||||||
const isNear = cluster.every(g => {
|
const isSimilar = cluster.every(g => maxRelativeDiff(g.key, b.key) <= THRESHOLD);
|
||||||
const maxVol = Math.max(g.volume, b.volume);
|
if (isSimilar) {
|
||||||
return Math.abs(g.volume - b.volume) / maxVol <= 0.05;
|
|
||||||
});
|
|
||||||
|
|
||||||
if (isNear) {
|
|
||||||
cluster.push(b);
|
cluster.push(b);
|
||||||
assignedKeys.add(b.key);
|
assignedKeys.add(b.key);
|
||||||
}
|
}
|
||||||
@@ -139,9 +145,13 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
|
|||||||
|
|
||||||
if (cluster.length >= 2) {
|
if (cluster.length >= 2) {
|
||||||
const volumes = cluster.map(g => g.volume);
|
const volumes = cluster.map(g => g.volume);
|
||||||
const maxVol = Math.max(...volumes);
|
// Max dimension-wise diff across all pairs in the cluster
|
||||||
const minVol = Math.min(...volumes);
|
let maxDiffPct = 0;
|
||||||
const maxDiffPct = ((maxVol - minVol) / maxVol) * 100;
|
for (let x = 0; x < cluster.length; x++) {
|
||||||
|
for (let y = x + 1; y < cluster.length; y++) {
|
||||||
|
maxDiffPct = Math.max(maxDiffPct, maxRelativeDiff(cluster[x].key, cluster[y].key) * 100);
|
||||||
|
}
|
||||||
|
}
|
||||||
clusters.push({ groups: cluster, volumes, maxDiffPct });
|
clusters.push({ groups: cluster, volumes, maxDiffPct });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -244,7 +254,7 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
|
|||||||
<Link2 className="w-4 h-4 text-violet-400" />
|
<Link2 className="w-4 h-4 text-violet-400" />
|
||||||
<span className="text-sm font-semibold text-violet-300">Possible data entry errors</span>
|
<span className="text-sm font-semibold text-violet-300">Possible data entry errors</span>
|
||||||
<span className="text-xs text-slate-500 bg-slate-900 px-2 py-0.5 rounded-full border border-slate-700">
|
<span className="text-xs text-slate-500 bg-slate-900 px-2 py-0.5 rounded-full border border-slate-700">
|
||||||
{nearDuplicateClusters.length} cluster{nearDuplicateClusters.length !== 1 ? 's' : ''} within 5% volume
|
{nearDuplicateClusters.length} cluster{nearDuplicateClusters.length !== 1 ? 's' : ''} with similar dimensions (≤15% per axis)
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
{nearDuplicateClusters.map((cluster, ci) => (
|
{nearDuplicateClusters.map((cluster, ci) => (
|
||||||
@@ -266,7 +276,7 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
|
|||||||
</span>
|
</span>
|
||||||
))}
|
))}
|
||||||
<span className="text-xs text-violet-400/70">
|
<span className="text-xs text-violet-400/70">
|
||||||
— diff. {cluster.maxDiffPct.toFixed(1)}%
|
— max axis diff {cluster.maxDiffPct.toFixed(1)}%
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
</button>
|
</button>
|
||||||
|
|||||||
Reference in New Issue
Block a user