ConsistentlyInconsistentYT--Pixeltovoxelprojector/tests/benchmarks/run_all_benchmarks.py

#!/usr/bin/env python3
"""
Comprehensive Benchmark Runner

Executes all benchmark suites and generates unified reports:
- Main benchmark suite
- Camera benchmarks
- CUDA voxel benchmarks (if available)
- Network benchmarks

Generates:
- Combined JSON results
- HTML summary report
- Performance comparison graphs
"""

import os
import sys
import json
import time
import subprocess
from pathlib import Path
from datetime import datetime
from typing import Dict, List


class BenchmarkRunner:
    """Orchestrate all benchmark suites"""

    def __init__(self):
        self.results = {}
        self.start_time = None
        self.end_time = None
        self.benchmark_dir = Path(__file__).parent
        self.output_dir = self.benchmark_dir / "benchmark_results"
        self.output_dir.mkdir(parents=True, exist_ok=True)

    def run_all(self):
        """Run all benchmarks"""
        print("="*70)
        print(" PixelToVoxel Comprehensive Benchmark Suite")
        print("="*70)

        self.start_time = datetime.now()
        print(f"\nStarted: {self.start_time.strftime('%Y-%m-%d %H:%M:%S')}\n")

        # Check environment
        self._check_environment()

        # Run benchmarks
        self._run_main_benchmark()
        self._run_camera_benchmark()
        self._run_cuda_benchmark()
        self._run_network_benchmark()

        self.end_time = datetime.now()

        # Generate reports
        self._save_results()
        self._generate_summary()

        print("\n" + "="*70)
        print(" Benchmark Suite Completed")
        print("="*70)
        duration = (self.end_time - self.start_time).total_seconds()
        print(f"\nTotal Duration: {duration:.1f} seconds")
        print(f"Results saved to: {self.output_dir}")

    def _check_environment(self):
        """Check required dependencies"""
        print("Checking environment...")

        # Check Python version
        python_version = sys.version_info
        print(f"  Python: {python_version.major}.{python_version.minor}.{python_version.micro}")

        # Check dependencies
        dependencies = {
            'numpy': False,
            'cv2': False,
            'matplotlib': False,
            'psutil': False,
        }

        for module in dependencies:
            try:
                __import__(module)
                dependencies[module] = True
                print(f"  {module}: OK")
            except ImportError:
                print(f"  {module}: MISSING")

        # Check CUDA
        cuda_available = self._check_cuda()
        print(f"  CUDA: {'OK' if cuda_available else 'NOT AVAILABLE'}")

        print()

    def _check_cuda(self) -> bool:
        """Check if CUDA is available"""
        try:
            result = subprocess.run(
                ['nvcc', '--version'],
                capture_output=True,
                timeout=5
            )
            return result.returncode == 0
        except Exception:
            return False

    def _run_main_benchmark(self):
        """Run main benchmark suite"""
        print("\n" + "="*70)
        print(" Running Main Benchmark Suite")
        print("="*70)

        try:
            # Import and run
            sys.path.insert(0, str(self.benchmark_dir))
            from benchmark_suite import BenchmarkSuite

            suite = BenchmarkSuite(output_dir=str(self.output_dir))

            # Run core benchmarks
            from benchmark_suite import (
                benchmark_voxel_ray_casting,
                benchmark_motion_detection,
                benchmark_voxel_update
            )

            import cv2
            HAS_CV2 = True
        except ImportError:
            HAS_CV2 = False

            # Ray casting
            suite.run_benchmark(
                "Voxel Ray Casting (500^3)",
                benchmark_voxel_ray_casting,
                iterations=50,
                warmup=5,
                grid_size=500,
                num_rays=1000
            )

            # Motion detection (if OpenCV available)
            if HAS_CV2:
                suite.run_benchmark(
                    "Motion Detection (8K)",
                    benchmark_motion_detection,
                    iterations=50,
                    warmup=5,
                    width=7680,
                    height=4320
                )

            # Voxel updates
            suite.run_benchmark(
                "Voxel Grid Updates",
                benchmark_voxel_update,
                iterations=100,
                warmup=10,
                grid_size=500,
                num_updates=10000
            )

            suite.save_results()
            suite.generate_report()

            # Store results
            self.results['main_suite'] = {
                'completed': True,
                'num_benchmarks': len(suite.results),
                'results': [
                    {
                        'name': r.name,
                        'throughput_fps': r.throughput_fps,
                        'latency_p99_ms': r.latency_p99_ms,
                    }
                    for r in suite.results
                ]
            }

            print("\n✓ Main benchmark suite completed")

        except Exception as e:
            print(f"\n✗ Main benchmark suite failed: {e}")
            self.results['main_suite'] = {'completed': False, 'error': str(e)}

    def _run_camera_benchmark(self):
        """Run camera benchmark suite"""
        print("\n" + "="*70)
        print(" Running Camera Benchmark Suite")
        print("="*70)

        try:
            sys.path.insert(0, str(self.benchmark_dir))
            from camera_benchmark import CameraBenchmark

            benchmark = CameraBenchmark(
                output_dir=str(self.output_dir / "camera")
            )

            results = benchmark.run_all_benchmarks()

            self.results['camera_suite'] = {
                'completed': True,
                'benchmarks': list(results.keys()),
            }

            print("\n✓ Camera benchmark suite completed")

        except Exception as e:
            print(f"\n✗ Camera benchmark suite failed: {e}")
            self.results['camera_suite'] = {'completed': False, 'error': str(e)}

    def _run_cuda_benchmark(self):
        """Run CUDA voxel benchmarks"""
        print("\n" + "="*70)
        print(" Running CUDA Voxel Benchmarks")
        print("="*70)

        cuda_binary = self.benchmark_dir / "voxel_benchmark"

        # Try to compile if not exists
        if not cuda_binary.exists():
            print("CUDA benchmark not compiled. Attempting to compile...")
            try:
                result = subprocess.run(
                    ['make', 'voxel_benchmark'],
                    cwd=self.benchmark_dir,
                    capture_output=True,
                    timeout=60
                )
                if result.returncode != 0:
                    print(f"Compilation failed: {result.stderr.decode()}")
                    self.results['cuda_suite'] = {
                        'completed': False,
                        'error': 'Compilation failed'
                    }
                    return
            except Exception as e:
                print(f"Cannot compile CUDA benchmark: {e}")
                self.results['cuda_suite'] = {
                    'completed': False,
                    'error': f'Cannot compile: {e}'
                }
                return

        # Run CUDA benchmark
        if cuda_binary.exists():
            try:
                print("\nRunning CUDA benchmarks...")
                result = subprocess.run(
                    [str(cuda_binary)],
                    cwd=self.benchmark_dir,
                    capture_output=True,
                    timeout=300
                )

                output = result.stdout.decode()
                print(output)

                self.results['cuda_suite'] = {
                    'completed': True,
                    'output': output
                }

                print("\n✓ CUDA benchmark suite completed")

            except subprocess.TimeoutExpired:
                print("\n✗ CUDA benchmark timed out")
                self.results['cuda_suite'] = {
                    'completed': False,
                    'error': 'Timeout'
                }
            except Exception as e:
                print(f"\n✗ CUDA benchmark failed: {e}")
                self.results['cuda_suite'] = {
                    'completed': False,
                    'error': str(e)
                }
        else:
            print("\n✗ CUDA benchmark binary not found")
            print("  To enable: cd to benchmarks dir and run 'make'")
            self.results['cuda_suite'] = {
                'completed': False,
                'error': 'Binary not found'
            }

    def _run_network_benchmark(self):
        """Run network benchmark suite"""
        print("\n" + "="*70)
        print(" Running Network Benchmark Suite")
        print("="*70)

        try:
            sys.path.insert(0, str(self.benchmark_dir))
            from network_benchmark import NetworkBenchmark

            benchmark = NetworkBenchmark(
                output_dir=str(self.output_dir / "network")
            )

            results = benchmark.run_all_benchmarks()

            self.results['network_suite'] = {
                'completed': True,
                'benchmarks': list(results.keys()),
            }

            print("\n✓ Network benchmark suite completed")

        except Exception as e:
            print(f"\n✗ Network benchmark suite failed: {e}")
            self.results['network_suite'] = {
                'completed': False,
                'error': str(e)
            }

    def _save_results(self):
        """Save combined results"""
        timestamp = self.start_time.strftime("%Y%m%d_%H%M%S")
        results_file = self.output_dir / f"combined_results_{timestamp}.json"

        combined = {
            'timestamp': self.start_time.isoformat(),
            'duration_sec': (self.end_time - self.start_time).total_seconds(),
            'suites': self.results,
        }

        with open(results_file, 'w') as f:
            json.dump(combined, f, indent=2)

        print(f"\nCombined results saved to: {results_file}")

    def _generate_summary(self):
        """Generate summary report"""
        timestamp = self.start_time.strftime("%Y%m%d_%H%M%S")
        summary_file = self.output_dir / f"summary_{timestamp}.txt"

        with open(summary_file, 'w') as f:
            f.write("="*70 + "\n")
            f.write(" PixelToVoxel Benchmark Summary\n")
            f.write("="*70 + "\n\n")

            f.write(f"Started:  {self.start_time.strftime('%Y-%m-%d %H:%M:%S')}\n")
            f.write(f"Finished: {self.end_time.strftime('%Y-%m-%d %H:%M:%S')}\n")
            duration = (self.end_time - self.start_time).total_seconds()
            f.write(f"Duration: {duration:.1f} seconds\n\n")

            f.write("="*70 + "\n")
            f.write(" Suite Status\n")
            f.write("="*70 + "\n\n")

            for suite_name, suite_data in self.results.items():
                status = "✓ PASS" if suite_data.get('completed', False) else "✗ FAIL"
                f.write(f"{suite_name:20s} {status}\n")

                if not suite_data.get('completed', False):
                    error = suite_data.get('error', 'Unknown error')
                    f.write(f"  Error: {error}\n")

            f.write("\n")

            # Main suite details
            if 'main_suite' in self.results and self.results['main_suite'].get('completed'):
                f.write("="*70 + "\n")
                f.write(" Main Suite Results\n")
                f.write("="*70 + "\n\n")

                for result in self.results['main_suite'].get('results', []):
                    f.write(f"{result['name']}\n")
                    f.write(f"  Throughput: {result['throughput_fps']:.2f} FPS\n")
                    f.write(f"  p99 Latency: {result['latency_p99_ms']:.2f} ms\n\n")

            f.write("="*70 + "\n")
            f.write(" End of Summary\n")
            f.write("="*70 + "\n")

        print(f"Summary saved to: {summary_file}")


def main():
    """Main entry point"""
    runner = BenchmarkRunner()
    runner.run_all()


if __name__ == "__main__":
    main()