diff --git a/ML_SETUP_GUIDE.md b/ML_SETUP_GUIDE.md new file mode 100644 index 0000000..436bb61 --- /dev/null +++ b/ML_SETUP_GUIDE.md @@ -0,0 +1,81 @@ +# Jass Card Detection ML Setup Guide + +This document outlines the strategy and implementation details for transitioning the card detection system from basic color thresholding to a deep learning-based object recognition pipeline. + +## 1. Architecture Overview +The system uses a **Hybrid Pipeline** to balance performance on mobile devices with high accuracy. + +**Workflow:** +`Camera Feed` $\rightarrow$ `Image Processing (Localization)` $\rightarrow$ `Card Cropping` $\rightarrow$ `ML Classifier (Identity)` $\rightarrow$ `Game State Update` + +1. **Localization**: Uses brightness and shape analysis to identify bounding boxes of cards (existing logic in `Detection.tsx`). +2. **Classification**: Crops each detected card and passes it through two specialized TensorFlow.js models to determine the **Suit** and the **Value**. + +## 2. Data Collection & Labeling +Because Jass cards have unique iconography, we use a custom dataset created from internet samples. + +### Labeling Strategy: Folder-Based Annotation +Instead of using bounding-box tools, we use the directory structure as labels. + +**Directory Structure:** +```text +dataset/ +├── suit_model/ +│ ├── Schellen/ (Bells) +│ ├── Schilten/ (Shields) +│ ├── Eicheln/ (Acorns) +│ └── Rosen/ (Roses) +└── value_model/ + ├── 6/ + ├── 7/ + ... + └── 13/ +``` + +### Process: +1. **Sourcing**: Collect 30-50 images per class from Swiss-German Jass deck galleries. +2. **Cropping**: Crop the center of the card for the Suit model and the corners for the Value model. +3. **Augmentation**: Use a script to generate variations: + * Rotations ($\pm 15^\circ$) + * Brightness/Contrast shifts + * Gaussian noise to simulate mobile camera grain. + +## 3. Model Training (Python) +We use **Transfer Learning** to minimize the required dataset size. + +* **Base Model**: MobileNetV2 (pre-trained on ImageNet). +* **Modification**: Remove the final 1000-class layer and replace it with a Dense layer matching the number of labels (4 for suits, 8 for values). +* **Optimization**: + * Loss: `categorical_crossentropy` + * Optimizer: `Adam` + * Regularization: Dropout (0.2) to prevent overfitting. +* **Target Size**: 64x64 pixels. + +## 4. Conversion & Deployment +Once trained in Python, the models are converted to the TensorFlow.js format. + +**Conversion Command:** +```bash +tensorflowjs_converter --input_format=keras /path/to/model.h5 /public/models/suit_model +``` + +**Deployment Path:** +The app expects the models in the public directory: +* `/public/models/suit_model/model.json` +* `/public/models/value_model/model.json` + +## 5. Frontend Integration +The integration is handled by `src/services/CardModelService.ts`. + +* **Initialization**: Models are loaded in parallel during `App.jsx` mount. +* **Inference**: + * Tensors are created from the `canvas` crop. + * Normalization is applied (`div(255.0)`). + * `tf.tidy()` is used to wrap operations and prevent WebGL memory leaks. +* **Fallback**: If models are missing or fail to load, the app automatically reverts to the legacy color-analysis detection to ensure the app remains functional. + +## 6. Validation Metrics +To verify the setup, the following metrics should be measured: +1. **Inference Latency**: Time from "Scan Table" click to result (Target: < 500ms). +2. **Classification Accuracy**: Percentage of correct identifications on a hold-out test set. +3. **Memory Footprint**: GPU memory usage during scan. diff --git a/src/App.jsx b/src/App.jsx index bcfa937..2b430a2 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -1,10 +1,24 @@ -import React from 'react'; +import React, { useEffect } from 'react'; import { GameStateProvider, useGameStateContext } from './context/GameStateContext'; +import { cardModelService } from './services/CardModelService'; import SetupScreen from './components/Setup/SetupScreen'; import CameraScreen from './components/Camera/CameraScreen'; import ResultsScreen from './components/Results/ResultsScreen'; import HistoryScreen from './components/History/HistoryScreen'; +const App = () => { + useEffect(() => { + cardModelService.init().catch(err => console.warn('Model loading failed, using fallback detection:', err)); + }, []); + + return ( + + + + ); +}; + + const App = () => { return ( diff --git a/src/components/Camera/CameraScreen.tsx b/src/components/Camera/CameraScreen.tsx index 76e3387..e441c1f 100644 --- a/src/components/Camera/CameraScreen.tsx +++ b/src/components/Camera/CameraScreen.tsx @@ -9,7 +9,8 @@ const CameraScreen: React.FC = () => { gameState, setCurrentScreen, setCameraStream, - scanTable + scanTable, + showResults } = useGameStateContext(); const videoRef = useRef(null); @@ -47,51 +48,43 @@ const CameraScreen: React.FC = () => { }; const scanTableHandler = async () => { - // Trigger detection via Detection component - if (videoRef.current && canvasRef.current) { - // Call the detection component's detection method - // Since detection is done in the Detection component, we'll call it through the component - // For testing, we'll force a detection via the Detection child component's functionality - const detectionComponent = document.querySelector('Detection'); - if (detectionComponent) { - // This will be handled by the actual Detection component - } else { - // Fallback to simulated detection for demo purposes - const detectedCards = [ - { - id: '1', - suit: 'Schellen', - value: 11, - x: 100, - y: 100, - width: 40, - height: 55, - confidence: 0.85 - }, - { - id: '2', - suit: 'Schilten', - value: 12, - x: 200, - y: 150, - width: 40, - height: 55, - confidence: 0.78 - }, - { - id: '3', - suit: 'Eicheln', - value: 10, - x: 300, - y: 200, - width: 40, - height: 55, - confidence: 0.91 - } - ]; - - handleCardsDetected(detectedCards); - } + if ((window as any).detectCards) { + (window as any).detectCards(); + } else { + // Fallback if the global detection function is not yet available + const detectedCards = [ + { + id: '1', + suit: 'Schellen', + value: 11, + x: 100, + y: 100, + width: 40, + height: 55, + confidence: 0.85 + }, + { + id: '2', + suit: 'Schilten', + value: 12, + x: 200, + y: 150, + width: 40, + height: 55, + confidence: 0.78 + }, + { + id: '3', + suit: 'Eicheln', + value: 10, + x: 300, + y: 200, + width: 40, + height: 55, + confidence: 0.91 + } + ]; + handleCardsDetected(detectedCards); } }; diff --git a/src/components/Detection/Detection.tsx b/src/components/Detection/Detection.tsx index 349c102..0df89c2 100644 --- a/src/components/Detection/Detection.tsx +++ b/src/components/Detection/Detection.tsx @@ -1,5 +1,6 @@ import React, { useRef, useEffect } from 'react'; import { Card } from '../../types'; +import { cardModelService } from '../../services/CardModelService'; interface DetectionProps { videoRef: React.RefObject; @@ -42,45 +43,58 @@ const Detection: React.FC = ({ videoRef, canvasRef, onCardsDetec }; // Enhanced card detection using image processing specialized for Jass cards - const processImageForCards = (canvas: HTMLCanvasElement, ctx: CanvasRenderingContext2D): Promise => { - return new Promise((resolve) => { - // Card detection logic optimized for Swiss/German-style cards - // Jass cards have specific visual characteristics: - // - White or light-colored background - // - Specific suit symbols with distinctive colors (red, green, black, yellow) - // - Usually rectangular with clear edges + const processImageForCards = async (canvas: HTMLCanvasElement, ctx: CanvasRenderingContext2D): Promise => { + // Card detection logic optimized for Swiss/German-style cards + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const cardRegions = findCardRegions(imageData, canvas.width, canvas.height); + const detectedCards: Card[] = []; + + for (let i = 0; i < cardRegions.length; i++) { + const region = cardRegions[i]; + const cardCrop = createCrop(ctx, canvas, region); - // Create image data - const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); - const data = imageData.data; + let suit: 'Schellen' | 'Schilten' | 'Eicheln' | 'Rosen'; + let value: number; + let confidence = 0.85; + + if (cardModelService.isReady()) { + const suitRes = await cardModelService.classifySuit(cardCrop); + const valRes = await cardModelService.classifyValue(cardCrop); + suit = suitRes.label as any; + value = parseInt(valRes.label); + confidence = (suitRes.confidence + valRes.confidence) / 2; + } else { + suit = detectCardSuit(ctx, canvas, region); + value = detectCardValue(ctx, canvas, region); + } - // Find potential card regions by looking for light-colored rectangles - const cardRegions = findCardRegions(imageData, canvas.width, canvas.height); - - // Detect cards from identified regions - const detectedCards: Card[] = []; - - cardRegions.forEach((region, index) => { - // For better accuracy, we also need to analyze the suit symbols - const suit = detectCardSuit(ctx, canvas, region); - const value = detectCardValue(ctx, canvas, region); - - detectedCards.push({ - id: `card-${index}`, - suit, - value, - x: region.x, - y: region.y, - width: region.width, - height: region.height, - confidence: 0.85 - }); + detectedCards.push({ + id: `card-${i}`, + suit, + value, + x: region.x, + y: region.y, + width: region.width, + height: region.height, + confidence }); - - resolve(detectedCards); - }); + } + + return detectedCards; }; + const createCrop = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, region: {x: number, y: number, width: number, height: number}): HTMLCanvasElement => { + const cropCanvas = document.createElement('canvas'); + cropCanvas.width = region.width; + cropCanvas.height = region.height; + const cropCtx = cropCanvas.getContext('2d'); + if (cropCtx) { + cropCtx.drawImage(canvas, region.x, region.y, region.width, region.height, 0, 0, region.width, region.height); + } + return cropCanvas; + }; + + // Enhanced card region detection specialized for Jass cards const findCardRegions = (imageData: ImageData, width: number, height: number): {x: number, y: number, width: number, height: number}[] => { const regions = []; diff --git a/src/services/CardModelService.ts b/src/services/CardModelService.ts new file mode 100644 index 0000000..860070b --- /dev/null +++ b/src/services/CardModelService.ts @@ -0,0 +1,91 @@ +import * as tf from '@tensorflow/tfjs'; + +export interface ClassificationResult { + label: string; + confidence: number; +} + +class CardModelService { + private suitModel: tf.LayersModel | null = null; + private valueModel: tf.LayersModel | null = null; + + private readonly SUIT_MODEL_PATH = '/models/suit_model/model.json'; + private readonly VALUE_MODEL_PATH = '/models/value_model/model.json'; + + private readonly SUIT_LABELS = ['Schellen', 'Schilten', 'Eicheln', 'Rosen']; + private readonly VALUE_LABELS = ['6', '7', '8', '9', '10', '11', '12', '13']; + + async init(): Promise { + try { + // Load models in parallel + const [sModel, vModel] = await Promise.all([ + tf.loadLayersModel(this.SUIT_MODEL_PATH), + tf.loadLayersModel(this.VALUE_MODEL_PATH), + ]); + this.suitModel = sModel; + this.valueModel = vModel; + console.log('Card detection models loaded successfully'); + } catch (error) { + console.error('Failed to load card models:', error); + throw error; + } + } + + async classifySuit(imageElement: HTMLCanvasElement | HTMLImageElement): Promise { + if (!this.suitModel) throw new Error('Suit model not initialized'); + + return tf.tidy(() => { + const tensor = this.preprocess(imageElement, 64); + const prediction = this.suitModel!.predict(tensor) as tf.Tensor; + const { index, confidence } = this.getTopK(prediction); + + return { + label: this.SUIT_LABELS[index], + confidence, + }; + }); + } + + async classifyValue(imageElement: HTMLCanvasElement | HTMLImageElement): Promise { + if (!this.valueModel) throw new Error('Value model not initialized'); + + return tf.tidy(() => { + const tensor = this.preprocess(imageElement, 64); + const prediction = this.valueModel!.predict(tensor) as tf.Tensor; + const { index, confidence } = this.getTopK(prediction); + + return { + label: this.VALUE_LABELS[index], + confidence, + }; + }); + } + + private preprocess(imageElement: HTMLCanvasElement | HTMLImageElement, size: number): tf.Tensor { + return tf.browser.fromPixels(imageElement) + .resizeBilinear([size, size]) + .expandDims(0) + .div(255.0); // Normalize to [0, 1] + } + + private getTopK(prediction: tf.Tensor): { index: number; confidence: number } { + const data = prediction.dataSync(); + let maxIndex = 0; + let maxVal = -1; + + for (let i = 0; i < data.length; i++) { + if (data[i] > maxVal) { + maxVal = data[i]; + maxIndex = i; + } + } + + return { index: maxIndex, confidence: maxVal }; + } + + isReady(): boolean { + return this.suitModel !== null && this.valueModel !== null; + } +} + +export const cardModelService = new CardModelService();