christianV/CrazeAnalytix
1
0
Fork 0
mirror of https://github.com/christianvidalwolf-prog/CrazeAnalytix.git synced 2026-08-03 13:55:23 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: resolve vendor data upload error by implementing client-side parsing and sequential batching

Browse Source
This commit is contained in:
Christian Vidal Wolf
2026-02-20 13:32:26 +01:00
parent a8258398f7
commit 85c789bf52
5 changed files with 162 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
App.tsx
+27 -7
View File
@@ -5,7 +5,7 @@ import Dashboard from './components/Dashboard';
import FilterBar from './components/FilterBar';
import AIChat from './components/AIChat';
import CrazeLogo from './components/CrazeLogo';
import { processCSV, processExcel, filterData, aggregateData, processAdsCSV, processAdsExcel, mergeSalesAndAdsData, processTrafficExcel, processStockExcel, filterAdsData, calculateForecastViewData, processVendorStockExcel, processUKInventoryExcel, calculateVelocityMap, getUniqueValues, processForecastExcel, processBuyBoxExcel } from './services/dataProcessor';
import { processCSV, processExcel, filterData, aggregateData, processAdsCSV, processAdsExcel, mergeSalesAndAdsData, processTrafficExcel, processStockExcel, filterAdsData, calculateForecastViewData, processVendorStockExcel, processUKInventoryExcel, calculateVelocityMap, getUniqueValues, processForecastExcel, processBuyBoxExcel, processVendorCSV } from './services/dataProcessor';
import { SalesRecord, FilterState, AggregatedData, AdsRecord, TrafficRecord, ForecastRecord, ProductForecastData } from './types';
import { queryGemini } from './services/geminiService';
import { ChartIcon, TableIcon, UploadIcon, DownloadIcon, CloseIcon, TrendingIcon, MegaphoneIcon } from './components/Icons';
@@ -438,19 +438,39 @@ const App: React.FC = () => {
}
};
// Handle uploaded Vendor CSV (sends to Supabase via API)
// Handle uploaded Vendor CSV (sends to Supabase via API with batching)
const handleVendorUpload = async (file: File) => {
setSyncing(true);
try {
const text = await file.text();
console.log("[App] Parsing Vendor CSV on client...");
const rows = await processVendorCSV(file);
if (rows.length === 0) {
alert("No valid rows found in CSV.");
return;
}
console.log(`[App] Uploading ${rows.length} rows in batches...`);
const BATCH_SIZE = 500;
let totalUpserted = 0;
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
const batch = rows.slice(i, i + BATCH_SIZE);
const response = await fetch('/api/upload-vendor-data', {
method: 'POST',
headers: { 'Content-Type': 'text/csv' },
body: text,
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(batch),
});
const result = await response.json();
if (!response.ok) throw new Error(result.error);
alert(`Vendor data uploaded: ${result.rowsUpserted} rows processed.`);
if (!response.ok) throw new Error(result.error || `Batch ${i / BATCH_SIZE + 1} failed`);
totalUpserted += result.rowsUpserted;
console.log(`[App] Batch ${Math.floor(i / BATCH_SIZE) + 1} complete. Total: ${totalUpserted}`);
}
alert(`Vendor data uploaded: ${totalUpserted} rows processed successfully.`);
setIsDataModalOpen(false);
} catch (error: any) {
console.error("Failed to upload vendor data", error);
api/upload-vendor-data.ts
+11 -3
View File
@@ -1,6 +1,7 @@
import type { VercelRequest, VercelResponse } from '@vercel/node';
import { createClient } from '@supabase/supabase-js';
import Papa from 'papaparse';
import { VendorDailyRow } from '../types';
const supabase = createClient(
process.env.SUPABASE_URL || '',
@@ -34,6 +35,13 @@ export default async function handler(req: VercelRequest, res: VercelResponse) {
if (req.method !== 'POST') return res.status(405).json({ error: 'Method not allowed' });
try {
let rows: VendorDailyRow[] = [];
// Check if the body is already parsed JSON (array of rows)
if (Array.isArray(req.body)) {
console.log(`[upload-vendor-data] Received ${req.body.length} rows as JSON`);
rows = req.body;
} else {
const csvText = typeof req.body === 'string' ? req.body : req.body?.toString() || '';
if (!csvText.trim()) {
@@ -49,7 +57,7 @@ export default async function handler(req: VercelRequest, res: VercelResponse) {
console.error('[upload-vendor-data] Parse errors:', parsed.errors.slice(0, 5));
}
const rows = parsed.data
rows = parsed.data
.filter(row => row['Date'] && row['Market'] && row['ASIN'])
.map(row => ({
date: row['Date'],
@@ -68,9 +76,10 @@ export default async function handler(req: VercelRequest, res: VercelResponse) {
amazon_has_buybox: row['Amazon Has Buybox'] === '1',
glance_views: parseIntSafe(row['Glance Views']),
}));
}
if (rows.length === 0) {
return res.status(400).json({ error: 'No valid rows found in CSV' });
return res.status(400).json({ error: 'No valid rows found' });
}
// Upsert in batches of 500
@@ -94,7 +103,6 @@ export default async function handler(req: VercelRequest, res: VercelResponse) {
res.status(200).json({
success: true,
rowsParsed: parsed.data.length,
rowsUpserted: totalUpserted,
});
} catch (error: any) {
services/dataProcessor.ts
+51 -1
View File
@@ -1,7 +1,57 @@
import { SalesRecord, AdsRecord, TrafficRecord, CombinedKPIs, FilterState, AggregatedData, LineGrowthMetric, ItemGrowthMetric, SeasonalityPoint, YearlySplitData, PivotRow, YearlyData, TimeSeriesData, ComparisonTimeSeriesPoint, ForecastRecord, MonthlyForecastPoint, ProductForecastData } from '../types';
import { SalesRecord, AdsRecord, TrafficRecord, CombinedKPIs, FilterState, AggregatedData, LineGrowthMetric, ItemGrowthMetric, SeasonalityPoint, YearlySplitData, PivotRow, YearlyData, TimeSeriesData, ComparisonTimeSeriesPoint, ForecastRecord, MonthlyForecastPoint, ProductForecastData, VendorCSVRow, VendorDailyRow } from '../types';
import * as XLSX from 'xlsx';
import Papa from 'papaparse';
/**
* Parses Vendor CSV and returns mapped VendorDailyRow items.
*/
export const processVendorCSV = (fileOrContent: File | string): Promise<VendorDailyRow[]> => {
return new Promise((resolve, reject) => {
const config = {
header: true,
skipEmptyLines: true,
complete: (results: any) => {
const rows = results.data
.filter((row: any) => row['Date'] && row['Market'] && row['ASIN'])
.map((row: any) => ({
date: row['Date'],
market: row['Market'],
asin: row['ASIN'],
product_title: row['Product Title'] || null,
tags: row['Tags'] || null,
bsr_top_rank: parseIntSafe(row['Top Level Category (Rank)']),
bsr_top_category: row['Top Level Category (Name)'] || null,
bsr_detail_rank: parseIntSafe(row['Detail Level Category (Rank)']),
bsr_detail_category: row['Detail Level Category (Name)'] || null,
avg_rating: parseCurrency(row['Average Rating']), // Uses existing parseCurrency which handles EU/US
num_reviews: parseIntSafe(row['Number of Reviews']),
buybox_owner: row['Buybox Seller Name'] || null,
buybox_price: parseCurrency(row['Buybox Price']),
amazon_has_buybox: row['Amazon Has Buybox'] === '1',
glance_views: parseIntSafe(row['Glance Views']),
}));
resolve(rows);
},
error: (error: any) => {
reject(error);
}
};
if (typeof fileOrContent === 'string') {
Papa.parse(fileOrContent, config);
} else {
Papa.parse(fileOrContent, config);
}
});
};
function parseIntSafe(val: string | undefined | null): number | null {
if (!val || typeof val !== 'string' || val.trim() === '') return null;
const cleaned = val.replace(/\./g, '').replace(',', '.').replace(/[^0-9.]/g, '');
const num = parseInt(cleaned, 10);
return isNaN(num) ? null : num;
}
// Helper to parse currency values handling both EU (1.234,56) and US/Standard (1,234.56 or 1234.56) formats
const parseCurrency = (value: string): number => {
if (!value) return 0;
services/supabase.ts
+1 -19
View File
@@ -1,5 +1,6 @@
import { createClient, SupabaseClient } from '@supabase/supabase-js';
import { VendorDailyRow } from '../types';
const supabaseUrl = process.env.SUPABASE_URL || '';
const supabaseAnonKey = process.env.SUPABASE_ANON_KEY || '';
@@ -16,25 +17,6 @@ const getClient = (): SupabaseClient => {
return _client;
};
export interface VendorDailyRow {
id?: number;
date: string;
market: string;
asin: string;
product_title: string | null;
tags: string | null;
bsr_top_rank: number | null;
bsr_top_category: string | null;
bsr_detail_rank: number | null;
bsr_detail_category: string | null;
avg_rating: number | null;
num_reviews: number | null;
buybox_owner: string | null;
buybox_price: number | null;
amazon_has_buybox: boolean | null;
glance_views: number | null;
}
export interface VendorFilters {
markets?: string[];
tags?: string[];
types.ts
+37
View File
@@ -242,3 +242,40 @@ export interface BuyBoxLostData {
countries: string[]; // ['DE', 'FR', 'IT', 'UK', 'ES']
reasons: Record<string, string>; // { DE: 'Amazon', FR: 'Unknown' }
}
export interface VendorDailyRow {
id?: number;
date: string;
market: string;
asin: string;
product_title: string | null;
tags: string | null;
bsr_top_rank: number | null;
bsr_top_category: string | null;
bsr_detail_rank: number | null;
bsr_detail_category: string | null;
avg_rating: number | null;
num_reviews: number | null;
buybox_owner: string | null;
buybox_price: number | null;
amazon_has_buybox: boolean | null;
glance_views: number | null;
}
export interface VendorCSVRow {
Date: string;
Market: string;
ASIN: string;
'Product Title': string;
Tags: string;
'Top Level Category (Rank)': string;
'Top Level Category (Name)': string;
'Detail Level Category (Rank)': string;
'Detail Level Category (Name)': string;
'Average Rating': string;
'Number of Reviews': string;
'Buybox Seller Name': string;
'Buybox Price': string;
'Amazon Has Buybox': string;
'Glance Views': string;
}