""" cksgen – Generate and compare MD5 checksum lists with Python Copyright (c) 2023 Helmut Kaczmarek This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . General Usage: cksgen [-h] [-c CONFIG] [-e] If you haven't compiled the cksgen.py file into an executable file (e.g. cksgen.exe on a Windows system; you can do so with "pyinstaller --onefile cksgen.py")) and haven't registered it in the system environment variables, then: python /path/to/cksgen.py [-h] [-c CONFIG] [-e] Help: cksgen -h See example configuration file: cksgen -e Example usage with a configuration file "photos.conf": cksgen -c photos """ import os import hashlib import glob import datetime import configparser import argparse # Example configuration that users can see with "-e" option EXAMPLE_CONFIG = r""" # Configuration file for cksgen.py # # Save the configuration files in the same # folder where the program is located. [USER_SETTINGS] # The configuration's name. # A subfolder with this name will be created. conf_name = Example # File types that cksgen will look for. # Multiple file types are separated by commas without spaces. allowed_extensions = jpg,jpeg,dng,cr2 # The actual directory where cksgen looks for files. data_directory = C:\Users\Username\Path\To\Example # The directory where lists of MD5 checksums are placed. lists_directory = conf_name\Lists # The directory in which the log files are stored. logs_directory = conf_name\Logs # Specifies how many checksum files and log files cksgen should keep. files_to_keep = 10 """ # Parse command line arguments parser = argparse.ArgumentParser(description="Generate MD5 checksums for files in a directory.") parser.add_argument("-c", "-conf", "--config", help="Specify the configuration file to use.") parser.add_argument("-e", "-ex", "--exampleconf", action="store_true", help="Display example configuration.") args = parser.parse_args() # Display example configuration and exit if "-e" option is used if args.exampleconf: print(EXAMPLE_CONFIG) exit(0) # Process configuration if "-c" option is used if args.config: if args.config == 'example': config_content = EXAMPLE_CONFIG else: print("Starting cksgen...") config_filename = args.config + ".conf" with open(config_filename, 'r') as config_file: config_content = config_file.read() # Parse the configuration content config = configparser.ConfigParser() config.read_string(config_content) # Extract user settings from the configuration allowed_extensions = [ext.strip() for ext in config.get('USER_SETTINGS', 'allowed_extensions').split(';')] data_directory = config.get('USER_SETTINGS', 'data_directory') lists_directory = os.path.join(os.path.dirname(__file__), config.get('USER_SETTINGS', 'lists_directory')) logs_directory = os.path.join(os.path.dirname(__file__), config.get('USER_SETTINGS', 'logs_directory')) files_to_keep = int(config.get('USER_SETTINGS', 'files_to_keep')) # Function to load configuration from a file def load_config(config_path): config = configparser.ConfigParser() config.read(config_path) return config['USER_SETTINGS'] # Function to delete old files based on number to keep def delete_old_files(directory, files_to_keep): all_files = glob.glob(os.path.join(directory, '*.*')) all_files.sort(key=os.path.getmtime, reverse=True) files_to_delete = all_files[files_to_keep:] for file in files_to_delete: os.remove(file) # Function to compare checksum files for differences def compare_checksum_files(file1, file2): with open(file1, 'r') as f1, open(file2, 'r') as f2: checksums1 = f1.read().splitlines()[1:] # Skip the first line checksums2 = f2.read().splitlines()[1:] # Skip the second line different_files = [] if not checksums2: return [""] for line1 in checksums1: checksum1, file1 = line1.split('\t') for line2 in checksums2: checksum2, file2 = line2.split('\t') if file1 == file2 and checksum1 != checksum2: different_files.append(file1) break return different_files # Main execution block if __name__ == "__main__": # Parse command line arguments parser = argparse.ArgumentParser(description="Generate checksums for files.") parser.add_argument("-conf", "--config", type=str, help="Configuration file name without extension") args = parser.parse_args() # If a configuration file is specified if args.config: config_filename = args.config + ".conf" config = load_config(config_filename) # Extract settings from the configuration conf_name = config.get('conf_name') allowed_extensions = config.get('allowed_extensions').split(',') data_directory = config.get('data_directory') files_to_keep = int(config.get('files_to_keep')) # Get the script's directory script_directory = os.path.dirname(os.path.abspath(__file__)) # Create paths for lists and logs directories lists_directory = os.path.join(script_directory, conf_name, 'Lists') logs_directory = os.path.join(script_directory, conf_name, 'Logs') # Create the directories if they don't exist os.makedirs(lists_directory, exist_ok=True) os.makedirs(logs_directory, exist_ok=True) # Generate timestamp and datetime strings current_timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') current_datetime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # Generate filenames for checksum and log files checksum_filename = os.path.join(lists_directory, f'{current_timestamp}_checksum.txt') log_filename = os.path.join(logs_directory, f'{current_timestamp}_log.txt') # Function to scan a directory and generate checksums def scan_directory(directory): with open(checksum_filename, 'a') as f: f.write(f"MD5 checksums on: {current_datetime}\n") # Walk through the directory tree for root, _, files in os.walk(directory): for file in files: file_path = os.path.join(root, file) extension = file_path.split('.')[-1].lower() # Calculate checksum for allowed file types if extension in allowed_extensions: md5_checksum = hashlib.md5(open(file_path, 'rb').read()).hexdigest() # Write checksum and file path to the checksum file with open(checksum_filename, 'a') as f: f.write(f'{md5_checksum}\t{file_path}\n') # Display progress print(f'Processing {file_path}') # Scan the specified data directory scan_directory(data_directory) # Create a log entry for successful checksum generation log_entry = f'MD5 checksums have been created and stored in {checksum_filename}.\n' with open(log_filename, 'a') as f: f.write(log_entry) # Delete old files based on the specified number to keep delete_old_files(lists_directory, files_to_keep) delete_old_files(logs_directory, files_to_keep) # Compare the last two checksum files for differences last_checksum_files = glob.glob(os.path.join(lists_directory, '*_checksum.txt')) if len(last_checksum_files) >= 2: # Sort the list of files in reverse order (newest first) last_checksum_files.sort(reverse=True) # Get the paths of the two most recent checksum files last_checksum_file1 = last_checksum_files[0] last_checksum_file2 = last_checksum_files[1] # Compare the checksums in the two files different_files = compare_checksum_files(last_checksum_file1, last_checksum_file2) log_message = '' # If there are different checksums, log and print the differences if different_files: print('ATTENTION: Different MD5 checksums found! See log file in', log_filename) log_message += 'ATTENTION: The following files have different checksums:\n' for file in different_files: log_message += file + '\n' else: # If no differences, log and print a message log_message += 'INFO: No different MD5 checksums found.\n' print('INFO: No different MD5 checksums found.') # Write the log message to the log file with open(log_filename, 'a') as f: f.write(log_message) # If there's only one checksum file, log and print a message elif len(last_checksum_files) == 1: with open(log_filename, 'a') as f: f.write("INFO: Checksums could not be compared because there is currently only one checksum file.\n") print("INFO: Checksums could not be compared because there is currently only one checksum file.") print(log_entry)