当不确定是不是无意中更改了数据集时，可以做一下数据集对比。
import os import json import hashlib import numpy as np from PIL import Image from pathlib import Path import pandas as pd from collections import defaultdict class DatasetComparator: def __init__(self, original_dataset_path, your_dataset_path): """ 数据集对比器 - 针对统一目录结构 Args: original_dataset_path: 原始数据集路径 your_dataset_path: 你的数据集路径 """ self.original_path = Path(original_dataset_path) self.your_path = Path(your_dataset_path) # 验证路径是否存在 self._validate_paths() # 存储对比结果 self.comparison_results = {} def _validate_paths(self): """验证路径和目录结构""" print("验证数据集路径:") print(f"原始数据集路径: {self.original_path}") print(f" 是否存在: {self.original_path.exists()}") print(f"你的数据集路径: {self.your_path}") print(f" 是否存在: {self.your_path.exists()}") if not self.original_path.exists(): raise FileNotFoundError(f"原始数据集路径不存在: {self.original_path}") if not self.your_path.exists(): raise FileNotFoundError(f"你的数据集路径不存在: {self.your_path}") # 检查数据集结构 self._check_structure(self.original_path, "原始数据集") self._check_structure(self.your_path, "你的数据集") def _check_structure(self, dataset_path, dataset_name): """检查数据集目录结构""" print(f"\n检查{dataset_name}结构:") # 检查是否有images和labels目录 images_dir = dataset_path / 'images' labels_dir = dataset_path / 'labels' print(f" images目录: {images_dir.exists()}") print(f" labels目录: {labels_dir.exists()}") if images_dir.exists(): splits = self._detect_splits(images_dir) print(f" images下的划分: {splits}") if labels_dir.exists(): splits = self._detect_splits(labels_dir) print(f" labels下的划分: {splits}") def _detect_splits(self, dir_path): """检测目录下的划分""" splits = [] for item in dir_path.iterdir(): if item.is_dir(): splits.append(item.name) return splits def calculate_file_hash(self, file_path): """计算文件的MD5哈希值""" hash_md5 = hashlib.md5() try: with open(file_path, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): hash_md5.update(chunk) return hash_md5.hexdigest() except Exception as e: print(f"计算文件哈希值时出错 {file_path}: {e}") return None def get_common_splits(self): """获取两个数据集共有的划分""" # 获取原始数据集的划分 original_img_dir = self.original_path / 'images' original_splits = set() if original_img_dir.exists(): for item in original_img_dir.iterdir(): if item.is_dir(): original_splits.add(item.name) # 获取你的数据集的划分 your_img_dir = self.your_path / 'images' your_splits = set() if your_img_dir.exists(): for item in your_img_dir.iterdir(): if item.is_dir(): your_splits.add(item.name) # 返回共同划分 common_splits = original_splits.intersection(your_splits) print(f"\n划分检测:") print(f"原始数据集划分: {sorted(original_splits)}") print(f"你的数据集划分: {sorted(your_splits)}") print(f"共同划分: {sorted(common_splits)}") return sorted(common_splits) def compare_split(self, split_name): """ 对比一个数据划分 Args: split_name: 划分名称 (如 train, val, test) """ print(f"\n{'='*60}") print(f"对比划分: {split_name}") print('='*60) # 构建路径 original_img_dir = self.original_path / 'images' / split_name original_label_dir = self.original_path / 'labels' / split_name your_img_dir = self.your_path / 'images' / split_name your_label_dir = self.your_path / 'labels' / split_name # 检查目录是否存在 dirs_exist = { 'original_images': original_img_dir.exists(), 'original_labels': original_label_dir.exists(), 'your_images': your_img_dir.exists(), 'your_labels': your_label_dir.exists() } for name, exists in dirs_exist.items(): print(f"{name}: {'✅ 存在' if exists else '❌ 不存在'}") result = { 'split': split_name, 'dirs_exist': dirs_exist, 'images': None, 'labels': None } # 对比图片 if dirs_exist['original_images'] and dirs_exist['your_images']: result['images'] = self._compare_files(original_img_dir, your_img_dir, 'images') # 对比标签 if dirs_exist['original_labels'] and dirs_exist['your_labels']: result['labels'] = self._compare_files(original_label_dir, your_label_dir, 'labels') self.comparison_results[split_name] = result return result def _compare_files(self, original_dir, your_dir, file_type='images'): """ 对比两个目录中的文件 Args: original_dir: 原始数据集目录 your_dir: 你的数据集目录 file_type: 文件类型 ('images' 或 'labels') """ # 获取所有文件 if file_type == 'images': # 图片文件扩展名 extensions = ['.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif'] else: # 标签文件扩展名 extensions = ['.txt', '.json', '.xml', '.csv'] original_files = [] for ext in extensions: original_files.extend(list(original_dir.glob(f'*{ext}'))) original_files.extend(list(original_dir.glob(f'*{ext.upper()}'))) your_files = [] for ext in extensions: your_files.extend(list(your_dir.glob(f'*{ext}'))) your_files.extend(list(your_dir.glob(f'*{ext.upper()}'))) # 提取文件名（不含扩展名） original_names = set() original_name_to_path = {} for file in original_files: stem = file.stem original_names.add(stem) original_name_to_path[stem] = file your_names = set() your_name_to_path = {} for file in your_files: stem = file.stem your_names.add(stem) your_name_to_path[stem] = file # 找出共同文件和独有文件 common_names = original_names.intersection(your_names) only_in_original = original_names - your_names only_in_your = your_names - original_names # 对比共同文件 common_comparison = [] for name in list(common_names)[:200]: # 限制数量，避免太多 orig_path = original_name_to_path[name] your_path = your_name_to_path[name] comparison = { 'name': name, 'original_path': str(orig_path), 'your_path': str(your_path), 'original_size': orig_path.stat().st_size, 'your_size': your_path.stat().st_size, 'size_same': False, 'hash_same': False, 'content_same': False } # 对比文件大小 comparison['size_same'] = (comparison['original_size'] == comparison['your_size']) # 计算哈希值 orig_hash = self.calculate_file_hash(orig_path) your_hash = self.calculate_file_hash(your_path) if orig_hash and your_hash: comparison['hash_same'] = (orig_hash == your_hash) # 对于图片，进行更详细的对比 if file_type == 'images' and orig_hash and your_hash and orig_hash == your_hash: comparison['content_same'] = True elif file_type == 'labels': # 对比标签内容 try: with open(orig_path, 'r', encoding='utf-8', errors='ignore') as f: orig_content = f.read().strip() with open(your_path, 'r', encoding='utf-8', errors='ignore') as f: your_content = f.read().strip() comparison['content_same'] = (orig_content == your_content) except Exception as e: comparison['content_error'] = str(e) common_comparison.append(comparison) return { 'common_files': common_comparison, 'only_in_original': list(only_in_original), 'only_in_your': list(only_in_your), 'original_count': len(original_files), 'your_count': len(your_files), 'common_count': len(common_names), 'identical_count': sum(1 for f in common_comparison if f.get('hash_same', False) or f.get('content_same', False)) } def compare_all(self): """对比所有数据划分""" common_splits = self.get_common_splits() if not common_splits: print("\n警告: 没有找到共同的划分!") return for split in common_splits: self.compare_split(split) # 生成汇总报告 self.generate_summary_report() def generate_summary_report(self): """生成汇总报告""" print(f"\n{'='*80}") print("数据集对比汇总报告") print('='*80) summary_data = [] for split, result in self.comparison_results.items(): # 图片对比 if result['images']: img_data = result['images'] summary_data.append({ '划分': split, '类型': '图片', '原始数量': img_data['original_count'], '你的数量': img_data['your_count'], '共同文件': img_data['common_count'], '只在你数据集': len(img_data['only_in_your']), '只在原始数据集': len(img_data['only_in_original']), '完全相同': img_data['identical_count'] }) # 标签对比 if result['labels']: label_data = result['labels'] summary_data.append({ '划分': split, '类型': '标签', '原始数量': label_data['original_count'], '你的数量': label_data['your_count'], '共同文件': label_data['common_count'], '只在你数据集': len(label_data['only_in_your']), '只在原始数据集': len(label_data['only_in_original']), '完全相同': label_data['identical_count'] }) # 创建DataFrame显示汇总信息 if summary_data: df = pd.DataFrame(summary_data) print("\n汇总统计:") print(df.to_string(index=False)) # 输出详细差异 print(f"\n{'='*80}") print("详细差异分析") print('='*80) for split, result in self.comparison_results.items(): print(f"\n{split.upper()} 划分详细差异:") # 图片差异 if result['images']: img_data = result['images'] print(f"\n图片对比:") print(f" 原始数据集: {img_data['original_count']} 张图片") print(f" 你的数据集: {img_data['your_count']} 张图片") print(f" 共同文件: {img_data['common_count']} 个") if img_data['only_in_original']: print(f" 只在原始数据集: {len(img_data['only_in_original'])} 个") if len(img_data['only_in_original']) <= 10: for name in img_data['only_in_original']: print(f" - {name}") else: print(f" 示例: {img_data['only_in_original'][:5]}") if img_data['only_in_your']: print(f" 只在你数据集: {len(img_data['only_in_your'])} 个") if len(img_data['only_in_your']) <= 10: for name in img_data['only_in_your']: print(f" - {name}") else: print(f" 示例: {img_data['only_in_your'][:5]}") print(f" 完全相同的图片: {img_data['identical_count']}/{img_data['common_count']}") # 标签差异 if result['labels']: label_data = result['labels'] print(f"\n标签对比:") print(f" 原始数据集: {label_data['original_count']} 个标签") print(f" 你的数据集: {label_data['your_count']} 个标签") print(f" 共同文件: {label_data['common_count']} 个") if label_data['only_in_original']: print(f" 只在原始数据集: {len(label_data['only_in_original'])} 个") if len(label_data['only_in_original']) <= 10: for name in label_data['only_in_original']: print(f" - {name}") else: print(f" 示例: {label_data['only_in_original'][:5]}") if label_data['only_in_your']: print(f" 只在你数据集: {len(label_data['only_in_your'])} 个") if len(label_data['only_in_your']) <= 10: for name in label_data['only_in_your']: print(f" - {name}") else: print(f" 示例: {label_data['only_in_your'][:5]}") print(f" 完全相同的标签: {label_data['identical_count']}/{label_data['common_count']}") def save_report_to_file(self, output_file=None): """将报告保存到文件""" if output_file is None: output_file = 'dataset_comparison_report.txt' output_path = Path(output_file) with open(output_path, 'w', encoding='utf-8') as f: import sys original_stdout = sys.stdout sys.stdout = f print("数据集对比报告") print(f"原始数据集: {self.original_path}") print(f"你的数据集: {self.your_path}") print(f"生成时间: {pd.Timestamp.now()}") print('='*80) self.generate_summary_report() # 添加更详细的信息 print(f"\n{'='*80}") print("详细文件对比") print('='*80) for split, result in self.comparison_results.items(): print(f"\n{split.upper()} 划分详细对比:") # 输出只在原始数据集中的文件 if result.get('images') and result['images']['only_in_original']: print(f"\n只在原始数据集的图片 ({len(result['images']['only_in_original'])}个):") for name in result['images']['only_in_original'][:50]: print(f" {name}") if len(result['images']['only_in_original']) > 50: print(f" ... 还有 {len(result['images']['only_in_original']) - 50} 个") # 输出只在你数据集中的文件 if result.get('images') and result['images']['only_in_your']: print(f"\n只在你数据集的图片 ({len(result['images']['only_in_your'])}个):") for name in result['images']['only_in_your'][:50]: print(f" {name}") if len(result['images']['only_in_your']) > 50: print(f" ... 还有 {len(result['images']['only_in_your']) - 50} 个") sys.stdout = original_stdout print(f"\n报告已保存到: {output_path}") # 主程序 if __name__ == "__main__": # 设置数据集路径 # 注意：根据你的实际情况调整这些路径 original_dataset_path = r" " # y原始数据集路径 your_dataset_path = r" "#你的数据集路径 # 或者如果你的数据集结构是这样的： # original_dataset_path = r"你的原始数据集路径" # your_dataset_path = r"你的数据集路径" print("开始数据集对比...") print(f"原始数据集: {original_dataset_path}") print(f"你的数据集: {your_dataset_path}") try: # 创建比较器 comparator = DatasetComparator(original_dataset_path, your_dataset_path) # 执行对比 comparator.compare_all() # 保存报告到文件 desktop_path = Path.home() / 'Desktop' / 'dataset_comparison_report.txt' comparator.save_report_to_file(str(desktop_path)) print("\n对比完成!") print(f"报告已保存到: {desktop_path}") except Exception as e: print(f"发生错误: {e}") import traceback traceback.print_exc()
