mirror of
https://github.com/christianvidalwolf-prog/Craze-Data-check.git
synced 2026-08-03 18:45:23 +02:00
fix: tighten near-duplicate detection to <1 cm absolute difference per axis
Replaces percentage-based threshold (15%) with strict absolute tolerance: two dimension groups are flagged only if all three sorted dimensions differ by less than 1 cm. Targets likely data entry typos. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
9753958761
commit
d48e2cb0ff
@@ -110,14 +110,14 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
|
|||||||
}, [groups, showOnlyInconsistent]);
|
}, [groups, showOnlyInconsistent]);
|
||||||
|
|
||||||
const nearDuplicateClusters = useMemo((): NearDuplicateCluster[] => {
|
const nearDuplicateClusters = useMemo((): NearDuplicateCluster[] => {
|
||||||
// Two groups are "similar" if every sorted dimension pair differs by ≤15%
|
// Two groups are "similar" if every sorted dimension pair differs by < 1 cm absolute
|
||||||
// (keys are already sorted ascending, e.g. "15x20x29")
|
// (keys are already sorted ascending, e.g. "15x20x29")
|
||||||
const THRESHOLD = 0.15;
|
const MAX_DIFF_CM = 1;
|
||||||
|
|
||||||
const maxRelativeDiff = (keyA: string, keyB: string): number => {
|
const maxAbsDiff = (keyA: string, keyB: string): number => {
|
||||||
const a = keyA.split('x').map(Number);
|
const a = keyA.split('x').map(Number);
|
||||||
const b = keyB.split('x').map(Number);
|
const b = keyB.split('x').map(Number);
|
||||||
return Math.max(...a.map((v, i) => Math.abs(v - b[i]) / Math.max(v, b[i], 0.001)));
|
return Math.max(...a.map((v, i) => Math.abs(v - b[i])));
|
||||||
};
|
};
|
||||||
|
|
||||||
const validGroups = groups.filter(g => g.volume > 0);
|
const validGroups = groups.filter(g => g.volume > 0);
|
||||||
@@ -135,8 +135,8 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
|
|||||||
const b = validGroups[j];
|
const b = validGroups[j];
|
||||||
if (assignedKeys.has(b.key)) continue;
|
if (assignedKeys.has(b.key)) continue;
|
||||||
|
|
||||||
// b must be similar to every group already in the cluster
|
// b must be within 1 cm on every axis of every group already in the cluster
|
||||||
const isSimilar = cluster.every(g => maxRelativeDiff(g.key, b.key) <= THRESHOLD);
|
const isSimilar = cluster.every(g => maxAbsDiff(g.key, b.key) < MAX_DIFF_CM);
|
||||||
if (isSimilar) {
|
if (isSimilar) {
|
||||||
cluster.push(b);
|
cluster.push(b);
|
||||||
assignedKeys.add(b.key);
|
assignedKeys.add(b.key);
|
||||||
@@ -145,11 +145,11 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
|
|||||||
|
|
||||||
if (cluster.length >= 2) {
|
if (cluster.length >= 2) {
|
||||||
const volumes = cluster.map(g => g.volume);
|
const volumes = cluster.map(g => g.volume);
|
||||||
// Max dimension-wise diff across all pairs in the cluster
|
// Max absolute diff (cm) across all pairs in the cluster
|
||||||
let maxDiffPct = 0;
|
let maxDiffPct = 0;
|
||||||
for (let x = 0; x < cluster.length; x++) {
|
for (let x = 0; x < cluster.length; x++) {
|
||||||
for (let y = x + 1; y < cluster.length; y++) {
|
for (let y = x + 1; y < cluster.length; y++) {
|
||||||
maxDiffPct = Math.max(maxDiffPct, maxRelativeDiff(cluster[x].key, cluster[y].key) * 100);
|
maxDiffPct = Math.max(maxDiffPct, maxAbsDiff(cluster[x].key, cluster[y].key));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
clusters.push({ groups: cluster, volumes, maxDiffPct });
|
clusters.push({ groups: cluster, volumes, maxDiffPct });
|
||||||
@@ -254,7 +254,7 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
|
|||||||
<Link2 className="w-4 h-4 text-violet-400" />
|
<Link2 className="w-4 h-4 text-violet-400" />
|
||||||
<span className="text-sm font-semibold text-violet-300">Possible data entry errors</span>
|
<span className="text-sm font-semibold text-violet-300">Possible data entry errors</span>
|
||||||
<span className="text-xs text-slate-500 bg-slate-900 px-2 py-0.5 rounded-full border border-slate-700">
|
<span className="text-xs text-slate-500 bg-slate-900 px-2 py-0.5 rounded-full border border-slate-700">
|
||||||
{nearDuplicateClusters.length} cluster{nearDuplicateClusters.length !== 1 ? 's' : ''} with similar dimensions (≤15% per axis)
|
{nearDuplicateClusters.length} cluster{nearDuplicateClusters.length !== 1 ? 's' : ''} with dimensions differing <1 cm per axis
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
{nearDuplicateClusters.map((cluster, ci) => (
|
{nearDuplicateClusters.map((cluster, ci) => (
|
||||||
@@ -276,7 +276,7 @@ export function DimensionsView({ data, headers, onEdit, onSaveRow, onCaptureStat
|
|||||||
</span>
|
</span>
|
||||||
))}
|
))}
|
||||||
<span className="text-xs text-violet-400/70">
|
<span className="text-xs text-violet-400/70">
|
||||||
— max axis diff {cluster.maxDiffPct.toFixed(1)}%
|
— max diff {cluster.maxDiffPct.toFixed(1)} cm
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
</button>
|
</button>
|
||||||
|
|||||||
Reference in New Issue
Block a user