diff --git a/.github/workflows/dart.yml b/.github/workflows/dart.yml index 54f1f7dc..98956a6d 100644 --- a/.github/workflows/dart.yml +++ b/.github/workflows/dart.yml @@ -37,12 +37,14 @@ jobs: - name: Download golden images uses: actions/download-artifact@v4 with: - name: golden-images-f0cf20d904f67e03c7ae247d18287b7f8a053cd2 - path: ./golden-downloads + name: golden-images-${{ github.sha }} + path: ./thermion_dart/test/golden-downloads - name: Install Python dependencies run: | python -m pip install --upgrade pip pip install Pillow numpy + - name: Compare golden images + run: cd thermion_dart/test && python compare_goldens.py - name: Upload logs if: failure() || steps.build.outcome == 'failure' uses: actions/upload-artifact@v4 diff --git a/thermion_dart/test/compare_goldens.py b/thermion_dart/test/compare_goldens.py new file mode 100644 index 00000000..4dccf34e --- /dev/null +++ b/thermion_dart/test/compare_goldens.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +""" +Compare PNG files between golden reference images and test output images. +""" + +import os +import sys +from pathlib import Path +from PIL import Image, ImageChops +import numpy as np + +def calculate_image_difference(img1_path, img2_path): + """ + Calculate the difference between two images. + Returns a tuple of (mean_squared_error, max_difference, are_identical) + """ + try: + with Image.open(img1_path) as img1, Image.open(img2_path) as img2: + # Convert to same mode if different + if img1.mode != img2.mode: + img1 = img1.convert('RGBA') + img2 = img2.convert('RGBA') + + # Check if dimensions match + if img1.size != img2.size: + print(f"Size mismatch: {img1_path} {img1.size} vs {img2_path} {img2.size}") + return None, None, False + + # Calculate difference + diff = ImageChops.difference(img1, img2) + + # Convert to numpy for calculations + diff_array = np.array(diff) + + # Calculate metrics + mse = np.mean(diff_array ** 2) + max_diff = np.max(diff_array) + are_identical = mse == 0 + + return mse, max_diff, are_identical + + except Exception as e: + print(f"Error comparing {img1_path} and {img2_path}: {e}") + return None, None, False + +def find_png_files(directory): + """Find all PNG files in a directory recursively.""" + png_files = [] + for root, dirs, files in os.walk(directory): + for file in files: + if file.lower().endswith('.png'): + png_files.append(os.path.relpath(os.path.join(root, file), directory)) + return sorted(png_files) + +def main(): + # Define paths + golden_dir = Path("golden-downloads/thermion_dart/test/output") + output_dir = Path("output") + + # Check if directories exist + if not golden_dir.exists(): + print(f"Error: Golden directory not found: {golden_dir}") + sys.exit(1) + + if not output_dir.exists(): + print(f"Error: Output directory not found: {output_dir}") + sys.exit(1) + + # Find PNG files in both directories + golden_files = find_png_files(golden_dir) + output_files = find_png_files(output_dir) + + print(f"Found {len(golden_files)} golden files and {len(output_files)} output files") + + # Track comparison results + identical_count = 0 + different_count = 0 + missing_count = 0 + error_count = 0 + + # Compare each golden file with its corresponding output file + for golden_file in golden_files: + golden_path = golden_dir / golden_file + output_path = output_dir / golden_file + + if not output_path.exists(): + print(f"❌ MISSING: {golden_file} (exists in golden but not in output)") + missing_count += 1 + continue + + mse, max_diff, are_identical = calculate_image_difference(golden_path, output_path) + + if mse is None: + print(f"❌ ERROR: {golden_file} (failed to compare)") + error_count += 1 + continue + + if are_identical: + print(f"✅ IDENTICAL: {golden_file}") + identical_count += 1 + else: + print(f"⚠️ DIFFERENT: {golden_file} (MSE: {mse:.2f}, Max diff: {max_diff})") + different_count += 1 + + # Check for files that exist in output but not in golden + extra_files = set(output_files) - set(golden_files) + for extra_file in extra_files: + print(f"ℹ️ EXTRA: {extra_file} (exists in output but not in golden)") + + # Print summary + print("\n" + "="*50) + print("COMPARISON SUMMARY:") + print(f"✅ Identical files: {identical_count}") + print(f"⚠️ Different files: {different_count}") + print(f"❌ Missing files: {missing_count}") + print(f"❌ Error files: {error_count}") + print(f"ℹ️ Extra files: {len(extra_files)}") + print(f"📊 Total golden files: {len(golden_files)}") + + # Exit with appropriate code + if different_count > 0 or missing_count > 0 or error_count > 0: + print("\n❌ COMPARISON FAILED") + sys.exit(1) + else: + print("\n✅ ALL COMPARISONS PASSED") + sys.exit(0) + +if __name__ == "__main__": + main()