Implement ML-based card detection and classification

ML_SETUP_GUIDE.md

@@ -0,0 +1,81 @@
+# Jass Card Detection ML Setup Guide
+
+This document outlines the strategy and implementation details for transitioning the card detection system from basic color thresholding to a deep learning-based object recognition pipeline.
+
+## 1. Architecture Overview
+The system uses a **Hybrid Pipeline** to balance performance on mobile devices with high accuracy.
+
+**Workflow:**
+`Camera Feed` $\rightarrow$ `Image Processing (Localization)` $\rightarrow$ `Card Cropping` $\rightarrow$ `ML Classifier (Identity)` $\rightarrow$ `Game State Update`
+
+1.  **Localization**: Uses brightness and shape analysis to identify bounding boxes of cards (existing logic in `Detection.tsx`).
+2.  **Classification**: Crops each detected card and passes it through two specialized TensorFlow.js models to determine the **Suit** and the **Value**.
+
+## 2. Data Collection & Labeling
+Because Jass cards have unique iconography, we use a custom dataset created from internet samples.
+
+### Labeling Strategy: Folder-Based Annotation
+Instead of using bounding-box tools, we use the directory structure as labels.
+
+**Directory Structure:**
+```text
+dataset/
+├── suit_model/
+│   ├── Schellen/  (Bells)
+│   ├── Schilten/  (Shields)
+│   ├── Eicheln/   (Acorns)
+│   └── Rosen/     (Roses)
+└── value_model/
+    ├── 6/
+    ├── 7/
+    ...
+    └── 13/
+```
+
+### Process:
+1.  **Sourcing**: Collect 30-50 images per class from Swiss-German Jass deck galleries.
+2.  **Cropping**: Crop the center of the card for the Suit model and the corners for the Value model.
+3.  **Augmentation**: Use a script to generate variations:
+    *   Rotations ($\pm 15^\circ$)
+    *   Brightness/Contrast shifts
+    *   Gaussian noise to simulate mobile camera grain.
+
+## 3. Model Training (Python)
+We use **Transfer Learning** to minimize the required dataset size.
+
+*   **Base Model**: MobileNetV2 (pre-trained on ImageNet).
+*   **Modification**: Remove the final 1000-class layer and replace it with a Dense layer matching the number of labels (4 for suits, 8 for values).
+*   **Optimization**:
+    *   Loss: `categorical_crossentropy`
+    *   Optimizer: `Adam`
+    *   Regularization: Dropout (0.2) to prevent overfitting.
+*   **Target Size**: 64x64 pixels.
+
+## 4. Conversion & Deployment
+Once trained in Python, the models are converted to the TensorFlow.js format.
+
+**Conversion Command:**
+```bash
+tensorflowjs_converter --input_format=keras /path/to/model.h5 /public/models/suit_model
+```
+
+**Deployment Path:**
+The app expects the models in the public directory:
+*   `/public/models/suit_model/model.json`
+*   `/public/models/value_model/model.json`
+
+## 5. Frontend Integration
+The integration is handled by `src/services/CardModelService.ts`.
+
+*   **Initialization**: Models are loaded in parallel during `App.jsx` mount.
+*   **Inference**:
+    *   Tensors are created from the `canvas` crop.
+    *   Normalization is applied (`div(255.0)`).
+    *   `tf.tidy()` is used to wrap operations and prevent WebGL memory leaks.
+*   **Fallback**: If models are missing or fail to load, the app automatically reverts to the legacy color-analysis detection to ensure the app remains functional.
+
+## 6. Validation Metrics
+To verify the setup, the following metrics should be measured:
+1.  **Inference Latency**: Time from "Scan Table" click to result (Target: < 500ms).
+2.  **Classification Accuracy**: Percentage of correct identifications on a hold-out test set.
+3.  **Memory Footprint**: GPU memory usage during scan.

src/App.jsx

@@ -1,10 +1,24 @@
-import React from 'react';
+import React, { useEffect } from 'react';
 import { GameStateProvider, useGameStateContext } from './context/GameStateContext';
+import { cardModelService } from './services/CardModelService';
 import SetupScreen from './components/Setup/SetupScreen';
 import CameraScreen from './components/Camera/CameraScreen';
 import ResultsScreen from './components/Results/ResultsScreen';
 import HistoryScreen from './components/History/HistoryScreen';
 
+const App = () => {
+  useEffect(() => {
+    cardModelService.init().catch(err => console.warn('Model loading failed, using fallback detection:', err));
+  }, []);
+
+  return (
+    <GameStateProvider>
+      <AppContent />
+    </GameStateProvider>
+  );
+};
+
+
 const App = () => {
   return (
     <GameStateProvider>

src/components/Camera/CameraScreen.tsx

@@ -9,7 +9,8 @@ const CameraScreen: React.FC = () => {
     gameState, 
     setCurrentScreen, 
     setCameraStream, 
-    scanTable 
+    scanTable,
+    showResults
   } = useGameStateContext();
   
   const videoRef = useRef<HTMLVideoElement>(null);
@@ -47,16 +48,10 @@ const CameraScreen: React.FC = () => {
   };
 
   const scanTableHandler = async () => {
-    // Trigger detection via Detection component
+    if ((window as any).detectCards) {
-    if (videoRef.current && canvasRef.current) {
+      (window as any).detectCards();
-      // Call the detection component's detection method
-      // Since detection is done in the Detection component, we'll call it through the component
-      // For testing, we'll force a detection via the Detection child component's functionality
-      const detectionComponent = document.querySelector('Detection');
-      if (detectionComponent) {
-        // This will be handled by the actual Detection component
     } else {
-        // Fallback to simulated detection for demo purposes
+      // Fallback if the global detection function is not yet available
       const detectedCards = [
         {
           id: '1',
@@ -89,10 +84,8 @@ const CameraScreen: React.FC = () => {
           confidence: 0.91
         }
       ];
-        
       handleCardsDetected(detectedCards);
     }
-    }
   };
 
   return (

src/components/Detection/Detection.tsx

@@ -1,5 +1,6 @@
 import React, { useRef, useEffect } from 'react';
 import { Card } from '../../types';
+import { cardModelService } from '../../services/CardModelService';
 
 interface DetectionProps {
   videoRef: React.RefObject<HTMLVideoElement>;
@@ -42,45 +43,58 @@ const Detection: React.FC<DetectionProps> = ({ videoRef, canvasRef, onCardsDetec
   };
 
   // Enhanced card detection using image processing specialized for Jass cards
-  const processImageForCards = (canvas: HTMLCanvasElement, ctx: CanvasRenderingContext2D): Promise<Card[]> => {
+  const processImageForCards = async (canvas: HTMLCanvasElement, ctx: CanvasRenderingContext2D): Promise<Card[]> => {
-    return new Promise((resolve) => {
     // Card detection logic optimized for Swiss/German-style cards
-      // Jass cards have specific visual characteristics:
-      // - White or light-colored background
-      // - Specific suit symbols with distinctive colors (red, green, black, yellow)
-      // - Usually rectangular with clear edges
-      
-      // Create image data
     const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
-      const data = imageData.data;
-      
-      // Find potential card regions by looking for light-colored rectangles
     const cardRegions = findCardRegions(imageData, canvas.width, canvas.height);
-      
-      // Detect cards from identified regions
     const detectedCards: Card[] = [];
 
-      cardRegions.forEach((region, index) => {
+    for (let i = 0; i < cardRegions.length; i++) {
-        // For better accuracy, we also need to analyze the suit symbols
+      const region = cardRegions[i];
-        const suit = detectCardSuit(ctx, canvas, region);
+      const cardCrop = createCrop(ctx, canvas, region);
-        const value = detectCardValue(ctx, canvas, region);
+      
+      let suit: 'Schellen' | 'Schilten' | 'Eicheln' | 'Rosen';
+      let value: number;
+      let confidence = 0.85;
+
+      if (cardModelService.isReady()) {
+        const suitRes = await cardModelService.classifySuit(cardCrop);
+        const valRes = await cardModelService.classifyValue(cardCrop);
+        suit = suitRes.label as any;
+        value = parseInt(valRes.label);
+        confidence = (suitRes.confidence + valRes.confidence) / 2;
+      } else {
+        suit = detectCardSuit(ctx, canvas, region);
+        value = detectCardValue(ctx, canvas, region);
+      }
       
       detectedCards.push({
-          id: `card-${index}`,
+        id: `card-${i}`,
         suit,
         value,
         x: region.x,
         y: region.y,
         width: region.width,
         height: region.height,
-          confidence: 0.85
+        confidence
-        });
       });
+    }
     
-      resolve(detectedCards);
+    return detectedCards;
-    });
   };
 
+  const createCrop = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, region: {x: number, y: number, width: number, height: number}): HTMLCanvasElement => {
+    const cropCanvas = document.createElement('canvas');
+    cropCanvas.width = region.width;
+    cropCanvas.height = region.height;
+    const cropCtx = cropCanvas.getContext('2d');
+    if (cropCtx) {
+      cropCtx.drawImage(canvas, region.x, region.y, region.width, region.height, 0, 0, region.width, region.height);
+    }
+    return cropCanvas;
+  };
+
+
   // Enhanced card region detection specialized for Jass cards
   const findCardRegions = (imageData: ImageData, width: number, height: number): {x: number, y: number, width: number, height: number}[] => {
     const regions = [];

src/services/CardModelService.ts

@@ -0,0 +1,91 @@
+import * as tf from '@tensorflow/tfjs';
+
+export interface ClassificationResult {
+  label: string;
+  confidence: number;
+}
+
+class CardModelService {
+  private suitModel: tf.LayersModel | null = null;
+  private valueModel: tf.LayersModel | null = null;
+
+  private readonly SUIT_MODEL_PATH = '/models/suit_model/model.json';
+  private readonly VALUE_MODEL_PATH = '/models/value_model/model.json';
+
+  private readonly SUIT_LABELS = ['Schellen', 'Schilten', 'Eicheln', 'Rosen'];
+  private readonly VALUE_LABELS = ['6', '7', '8', '9', '10', '11', '12', '13'];
+
+  async init(): Promise<void> {
+    try {
+      // Load models in parallel
+      const [sModel, vModel] = await Promise.all([
+        tf.loadLayersModel(this.SUIT_MODEL_PATH),
+        tf.loadLayersModel(this.VALUE_MODEL_PATH),
+      ]);
+      this.suitModel = sModel;
+      this.valueModel = vModel;
+      console.log('Card detection models loaded successfully');
+    } catch (error) {
+      console.error('Failed to load card models:', error);
+      throw error;
+    }
+  }
+
+  async classifySuit(imageElement: HTMLCanvasElement | HTMLImageElement): Promise<ClassificationResult> {
+    if (!this.suitModel) throw new Error('Suit model not initialized');
+
+    return tf.tidy(() => {
+      const tensor = this.preprocess(imageElement, 64);
+      const prediction = this.suitModel!.predict(tensor) as tf.Tensor;
+      const { index, confidence } = this.getTopK(prediction);
+      
+      return {
+        label: this.SUIT_LABELS[index],
+        confidence,
+      };
+    });
+  }
+
+  async classifyValue(imageElement: HTMLCanvasElement | HTMLImageElement): Promise<ClassificationResult> {
+    if (!this.valueModel) throw new Error('Value model not initialized');
+
+    return tf.tidy(() => {
+      const tensor = this.preprocess(imageElement, 64);
+      const prediction = this.valueModel!.predict(tensor) as tf.Tensor;
+      const { index, confidence } = this.getTopK(prediction);
+      
+      return {
+        label: this.VALUE_LABELS[index],
+        confidence,
+      };
+    });
+  }
+
+  private preprocess(imageElement: HTMLCanvasElement | HTMLImageElement, size: number): tf.Tensor {
+    return tf.browser.fromPixels(imageElement)
+      .resizeBilinear([size, size])
+      .expandDims(0)
+      .div(255.0); // Normalize to [0, 1]
+  }
+
+  private getTopK(prediction: tf.Tensor): { index: number; confidence: number } {
+    const data = prediction.dataSync();
+    let maxIndex = 0;
+    let maxVal = -1;
+
+    for (let i = 0; i < data.length; i++) {
+      if (data[i] > maxVal) {
+        maxVal = data[i];
+        maxIndex = i;
+      }
+    }
+
+    return { index: maxIndex, confidence: maxVal };
+  }
+
+  isReady(): boolean {
+    return this.suitModel !== null && this.valueModel !== null;
+  }
+}
+
+export const cardModelService = new CardModelService();