feat: Automate Dropbox data loading with Vite proxy

This commit is contained in:
Christian Vidal Wolf
2026-01-16 10:20:22 +01:00
parent 135e0d78ed
commit 470db20458
9 changed files with 406 additions and 1664 deletions
+48 -95
View File
@@ -23,7 +23,7 @@ const RefreshIcon = ({ className }: { className?: string }) => (
// Hardcoded Permanent URL for Auto-Loading // Hardcoded Permanent URL for Auto-Loading
// Using the original share link to leverage Dropbox's redirect for robust fetching. // Using the original share link to leverage Dropbox's redirect for robust fetching.
const PERMANENT_DROPBOX_URL = "https://www.dropbox.com/scl/fi/b9zxn4z5i7sxwfakk5g5y/Amazon-Sell-Out-2023-2025.csv?rlkey=uoto6v0mm99py8nszy8ldtez8&dl=0"; const PERMANENT_DROPBOX_URL = "https://www.dropbox.com/scl/fi/b9zxn4z5i7sxwfakk5g5y/Amazon-Sell-Out-2023-2025.csv?rlkey=uoto6v0mm99py8nszy8ldtez8&st=7vk22iod&dl=0";
const App: React.FC = () => { const App: React.FC = () => {
const [rawData, setRawData] = useState<SalesRecord[]>([]); const [rawData, setRawData] = useState<SalesRecord[]>([]);
@@ -52,15 +52,7 @@ const App: React.FC = () => {
// Handle URL Fetch (Auto/Manual) // Handle URL Fetch (Auto/Manual)
const handleUrlFetch = useCallback(async (url: string) => { const handleUrlFetch = useCallback(async (url: string) => {
setSyncing(true); setSyncing(true);
setLoading(true);
// Reset previous errors on new attempt
// setLastUpdated(null);
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 15000); // 15s Timeout
try { try {
console.log(`[Sync] Starting fetch from: ${url}`);
let directUrl = url; let directUrl = url;
// Create a direct download link for Dropbox if it's a share link. // Create a direct download link for Dropbox if it's a share link.
if (url.includes('dropbox.com/') && !url.includes('dl.dropboxusercontent.com')) { if (url.includes('dropbox.com/') && !url.includes('dl.dropboxusercontent.com')) {
@@ -69,28 +61,28 @@ const App: React.FC = () => {
directUrl = urlObject.toString(); directUrl = urlObject.toString();
} }
// Use a CORS proxy to bypass browser's same-origin policy restrictions. let fetchUrl = directUrl;
const proxyUrl = `https://corsproxy.io/?${encodeURIComponent(directUrl)}`;
console.log(`[Sync] Fetching via proxy: ${proxyUrl}`); // Check if we are in a local environment to use the Vite proxy
const isLocal = window.location.hostname === 'localhost' || window.location.hostname === '127.0.0.1';
const response = await fetch(proxyUrl, { if (isLocal && url.includes('dropbox.com')) {
signal: controller.signal // Extract path and query from the direct URL
}); const urlObj = new URL(directUrl);
// Pass the necessary query params including st and dl=1
if (!response.ok) { const searchParams = urlObj.search;
throw new Error(`Failed to fetch CSV: ${response.status} ${response.statusText}`); fetchUrl = `/api/dropbox${urlObj.pathname}${searchParams}`;
} else {
// Production or non-local fallback: Use CORS proxy
// Using allorigins.win as a fallback if not local
fetchUrl = `https://api.allorigins.win/raw?url=${encodeURIComponent(directUrl)}`;
} }
const response = await fetch(fetchUrl);
if (!response.ok) throw new Error(`Failed to fetch CSV from URL: ${response.status} ${response.statusText}`);
const csvText = await response.text(); const csvText = await response.text();
console.log(`[Sync] Download complete. Bytes: ${csvText.length}`);
if (!csvText || csvText.trim().length === 0) {
throw new Error("Downloaded file is empty");
}
const data = await processCSV(csvText); const data = await processCSV(csvText);
console.log(`[Sync] Processing complete. Rows: ${data.length}`);
await saveSalesData(data); await saveSalesData(data);
@@ -101,20 +93,12 @@ const App: React.FC = () => {
localStorage.setItem('craze_last_updated', now); localStorage.setItem('craze_last_updated', now);
localStorage.setItem('craze_csv_url', url); localStorage.setItem('craze_csv_url', url);
setIsDataModalOpen(false); // Close modal on success setIsDataModalOpen(false); // Close modal on success
} catch (error: any) { } catch (error) {
console.error("[Sync] Error during data fetch:", error); console.error("Failed to fetch/parse CSV from URL", error);
let msg = "Failed to sync data."; // Don't alert on auto-fetch to avoid spamming the user on startup if offline
if (error.name === 'AbortError') { // alert("Error syncing data. Please check the URL.");
msg = "Connection timed out (15s). Proxy might be slow."; throw error; // re-throw to be caught by caller
} else if (error.message) {
msg = error.message;
}
alert(`Error: ${msg}\n\nSwitching to manual mode.`);
// IMPORTANT: If fetch fails, we MUST stop loading so the user can interact
setRawData([]);
setActiveUrl(null); // Clear active URL so valid manual upload is required or retry
} finally { } finally {
clearTimeout(timeoutId);
setSyncing(false); setSyncing(false);
setLoading(false); setLoading(false);
} }
@@ -137,31 +121,25 @@ const App: React.FC = () => {
useEffect(() => { useEffect(() => {
const initApp = async () => { const initApp = async () => {
setLoading(true); setLoading(true);
console.log("[Init] App starting...");
try { const currentStoredUrl = localStorage.getItem('craze_csv_url');
const currentStoredUrl = localStorage.getItem('craze_csv_url'); if (currentStoredUrl !== PERMANENT_DROPBOX_URL) {
if (currentStoredUrl !== PERMANENT_DROPBOX_URL) { localStorage.setItem('craze_csv_url', PERMANENT_DROPBOX_URL);
console.log("[Init] Updating stored URL to default permanent URL"); setActiveUrl(PERMANENT_DROPBOX_URL);
localStorage.setItem('craze_csv_url', PERMANENT_DROPBOX_URL); }
setActiveUrl(PERMANENT_DROPBOX_URL);
}
const { data, lastUpdated: date } = await loadSalesData(); const { data, lastUpdated: date } = await loadSalesData();
if (data && data.length > 0) { if (data && data.length > 0) {
console.log("[Init] Loaded data from cache:", data.length, "rows. Last updated:", date); console.log("Loaded data from cache:", data.length, "rows");
initializeData(data); initializeData(data);
setLastUpdated(date); setLastUpdated(date);
setLoading(false); setLoading(false);
} else { } else {
console.log("[Init] No cache found. Auto-fetching from Permanent URL..."); console.log("No cache found. Auto-fetching from Permanent URL...");
// Catch error here so initApp doesn't crash, handleUrlFetch handles UI handleUrlFetch(PERMANENT_DROPBOX_URL).catch(e => {
await handleUrlFetch(PERMANENT_DROPBOX_URL); console.error("Initial fetch failed.");
} });
} catch (e) {
console.error("[Init] Critical failure during initialization:", e);
setLoading(false); // Ensure we never get stuck in infinite load
} }
}; };
initApp(); initApp();
@@ -211,9 +189,6 @@ const App: React.FC = () => {
// Refresh if it's after 7 AM and we haven't refreshed today // Refresh if it's after 7 AM and we haven't refreshed today
if (now.getHours() >= 7 && lastRefreshDate !== today) { if (now.getHours() >= 7 && lastRefreshDate !== today) {
console.log("Triggering daily data refresh..."); console.log("Triggering daily data refresh...");
// Use a background version (no UI loading state) if possible,
// but re-using handleUrlFetch is fine for now but might show spinners.
// Ideally split the "silent sync" logic.
handleUrlFetch(PERMANENT_DROPBOX_URL).then(() => { handleUrlFetch(PERMANENT_DROPBOX_URL).then(() => {
localStorage.setItem('craze_last_refresh_date', today); localStorage.setItem('craze_last_refresh_date', today);
console.log("Daily refresh successful."); console.log("Daily refresh successful.");
@@ -224,11 +199,12 @@ const App: React.FC = () => {
}; };
// Check immediately on load in case the user opens the app after 7 AM // Check immediately on load in case the user opens the app after 7 AM
// checkAndRefresh(); // Disabled for now to prevent double-fetch on startup logic conflict checkAndRefresh();
// And then check periodically (e.g., every 15 minutes) in case app is left open across midnight // And then check periodically (e.g., every 15 minutes) in case app is left open across midnight
// const interval = setInterval(checkAndRefresh, 15 * 60 * 1000); const interval = setInterval(checkAndRefresh, 15 * 60 * 1000);
// return () => clearInterval(interval);
return () => clearInterval(interval);
}, [handleUrlFetch]); }, [handleUrlFetch]);
@@ -298,12 +274,6 @@ const App: React.FC = () => {
setRawData([]); setRawData([]);
}; };
const handleSkipLoading = () => {
console.log("User skipped loading.");
setLoading(false);
setSyncing(false);
};
return ( return (
<div className="min-h-screen flex flex-col bg-background text-slate-200"> <div className="min-h-screen flex flex-col bg-background text-slate-200">
@@ -390,29 +360,11 @@ const App: React.FC = () => {
{/* Main Content */} {/* Main Content */}
<main className="flex-1 relative"> <main className="flex-1 relative">
{loading ? ( {loading ? (
// Initial loading spinner with SKIP Option // Initial loading spinner
<div className="flex flex-col items-center justify-center h-[80vh] gap-6 text-center animate-fade-in"> <div className="flex flex-col items-center justify-center h-[80vh] gap-4">
<div className="relative"> <div className="w-16 h-16 border-4 border-indigo-500 border-t-transparent rounded-full animate-spin"></div>
<div className="w-20 h-20 border-4 border-indigo-500/30 border-t-indigo-500 rounded-full animate-spin"></div> <h2 className="text-xl font-bold text-slate-300">Loading Dashboard...</h2>
<div className="absolute inset-0 flex items-center justify-center"> <p className="text-sm text-slate-500">Syncing with Dropbox...</p>
<UploadIcon className="w-8 h-8 text-indigo-400 animate-pulse" />
</div>
</div>
<div>
<h2 className="text-2xl font-bold text-white mb-2">Syncing Data</h2>
<p className="text-slate-400 max-w-md mx-auto">
Connecting to Dropbox to fetch the latest analytics...
</p>
</div>
<div className="mt-4 flex flex-col gap-3">
<button
onClick={handleSkipLoading}
className="text-sm font-semibold text-slate-500 hover:text-white hover:underline transition-colors"
>
Taking too long? Skip Sync
</button>
</div>
</div> </div>
) : ( ) : (
<> <>
@@ -464,4 +416,5 @@ const App: React.FC = () => {
</div> </div>
); );
}; };
export default App; export default App;
+11
View File
@@ -0,0 +1,11 @@
{
"folders": [
{
"path": "."
},
{
"path": "../../Downloads"
}
],
"settings": {}
}
+16
View File
@@ -0,0 +1,16 @@
{
"hosting": {
"public": "dist",
"ignore": [
"firebase.json",
"**/.*",
"**/node_modules/**"
],
"rewrites": [
{
"source": "**",
"destination": "/index.html"
}
]
}
}
+3
View File
@@ -22,6 +22,8 @@
} }
} }
</script> </script>
<!-- PapaParse for CSV parsing -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/PapaParse/5.4.1/papaparse.min.js"></script>
<style> <style>
/* Custom Scrollbar */ /* Custom Scrollbar */
::-webkit-scrollbar { ::-webkit-scrollbar {
@@ -42,6 +44,7 @@
background: #475569; background: #475569;
} }
</style> </style>
</head> </head>
<body class="bg-background text-slate-200 antialiased overflow-y-auto"> <body class="bg-background text-slate-200 antialiased overflow-y-auto">
+1 -4
View File
@@ -1,7 +1,6 @@
import React from 'react'; import React from 'react';
import ReactDOM from 'react-dom/client'; import ReactDOM from 'react-dom/client';
import App from './App'; import App from './App';
import ErrorBoundary from './components/ErrorBoundary';
const rootElement = document.getElementById('root'); const rootElement = document.getElementById('root');
if (!rootElement) { if (!rootElement) {
@@ -11,8 +10,6 @@ if (!rootElement) {
const root = ReactDOM.createRoot(rootElement); const root = ReactDOM.createRoot(rootElement);
root.render( root.render(
<React.StrictMode> <React.StrictMode>
<ErrorBoundary> <App />
<App />
</ErrorBoundary>
</React.StrictMode> </React.StrictMode>
); );
+272 -1533
View File
File diff suppressed because it is too large Load Diff
+7 -16
View File
@@ -5,30 +5,21 @@
"type": "module", "type": "module",
"scripts": { "scripts": {
"dev": "vite", "dev": "vite",
"build": "tsc && vite build", "build": "vite build",
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
"preview": "vite preview" "preview": "vite preview"
}, },
"dependencies": { "dependencies": {
"react": "^18.3.1", "react": "^19.2.0",
"react-dom": "^18.3.1", "react-dom": "^19.2.0",
"@google/genai": "^1.30.0", "@google/genai": "^1.30.0",
"recharts": "^3.5.0", "recharts": "^3.5.0",
"xlsx": "^0.18.5", "xlsx": "^0.18.5",
"papaparse": "^5.5.3" "papaparse": "^5.5.3"
}, },
"devDependencies": { "devDependencies": {
"@types/node": "^22.5.0", "@types/node": "^22.14.0",
"@types/react": "^18.3.3", "@vitejs/plugin-react": "^5.0.0",
"@types/react-dom": "^18.3.0", "typescript": "~5.8.2",
"@types/papaparse": "^5.3.14", "vite": "^6.2.0"
"@vitejs/plugin-react": "^4.3.1",
"typescript": "^5.6.2",
"vite": "^5.4.2",
"eslint": "^9.9.1",
"eslint-plugin-react-hooks": "^5.1.0-rc.0",
"eslint-plugin-react-refresh": "^0.4.9",
"globals": "^15.9.0",
"typescript-eslint": "^8.0.0"
} }
} }
+24
View File
@@ -0,0 +1,24 @@
import { CombinedKPIs, FilterState } from '../types';
export const filterCombinedData = (data: CombinedKPIs[], filters: FilterState): CombinedKPIs[] => {
return data.filter(item => {
// 1. Month Logic
const recordMonth = item.month; // e.g. "Apr-23"
const pureMonth = recordMonth.split('-')[0]; // "Apr"
// 2. Filter Checks
const customerMatch = filters.customer.length === 0 || filters.customer.includes(item.marketplace);
const yearMatch = filters.year.length === 0 || filters.year.includes(item.year.toString());
// Exact or partial month match
const monthMatch = filters.month.length === 0 || filters.month.includes(pureMonth) || filters.month.includes(recordMonth);
const lineMatch = filters.line.length === 0 || filters.line.includes(item.line);
const asinMatch = filters.asin.length === 0 || filters.asin.includes(item.asin);
const skuMatch = filters.sku.length === 0 || filters.sku.includes(item.sku);
const titleMatch = filters.title.length === 0 || filters.title.includes(item.title);
return customerMatch && yearMatch && monthMatch && lineMatch && asinMatch && skuMatch && titleMatch;
});
};
+23 -15
View File
@@ -3,21 +3,29 @@ import { defineConfig, loadEnv } from 'vite';
import react from '@vitejs/plugin-react'; import react from '@vitejs/plugin-react';
export default defineConfig(({ mode }) => { export default defineConfig(({ mode }) => {
const env = loadEnv(mode, '.', ''); const env = loadEnv(mode, '.', '');
return { return {
server: { server: {
port: 3000, port: 3000,
host: '0.0.0.0', host: '0.0.0.0',
}, proxy: {
plugins: [react()], '/api/dropbox': {
define: { target: 'https://www.dropbox.com',
'process.env.API_KEY': JSON.stringify(env.GEMINI_API_KEY), changeOrigin: true,
'process.env.GEMINI_API_KEY': JSON.stringify(env.GEMINI_API_KEY) rewrite: (path) => path.replace(/^\/api\/dropbox/, ''),
}, followRedirects: true
resolve: {
alias: {
'@': path.resolve(__dirname, '.'),
} }
} }
}; },
plugins: [react()],
define: {
'process.env.API_KEY': JSON.stringify(env.GEMINI_API_KEY),
'process.env.GEMINI_API_KEY': JSON.stringify(env.GEMINI_API_KEY)
},
resolve: {
alias: {
'@': path.resolve(__dirname, '.'),
}
}
};
}); });