cksgen/cksgen.py

"""
cksgen – Generate and compare MD5 checksum lists with Python

Copyright (c) 2023 Helmut Kaczmarek <code@helmutkaczmarek.de>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.

General Usage:
cksgen [-h] [-c CONFIG] [-e]

If you haven't compiled the cksgen.py file into an executable file (e.g. 
cksgen.exe on a Windows system; you can do so with "pyinstaller 
--onefile cksgen.py")) and haven't registered it in the system 
environment variables, then:

python /path/to/cksgen.py [-h] [-c CONFIG] [-e]

Help:
cksgen -h

See example configuration file:
cksgen -e

Example usage with a configuration file "photos.conf":
cksgen -c photos
"""

import os
import hashlib
import glob
import datetime
import configparser
import argparse

# Example configuration that users can see with "-e" option
EXAMPLE_CONFIG = r"""
# Configuration file for cksgen.py
#
# Save the configuration files in the same
# folder where the program is located.
[USER_SETTINGS]
# The configuration's name.
# A subfolder with this name will be created.
conf_name = Example

# File types that cksgen will look for.
# Multiple file types are separated by commas without spaces.
allowed_extensions = jpg,jpeg,dng,cr2

# The actual directory where cksgen looks for files.
data_directory = C:\Users\Username\Path\To\Example

# The directory where lists of MD5 checksums are placed.
lists_directory = conf_name\Lists

# The directory in which the log files are stored.
logs_directory = conf_name\Logs

# Specifies how many checksum files and log files cksgen should keep.
files_to_keep = 10
"""

# Parse command line arguments
parser = argparse.ArgumentParser(description="Generate MD5 checksums for files in a directory.")
parser.add_argument("-c", "-conf", "--config", help="Specify the configuration file to use.")
parser.add_argument("-e", "-ex", "--exampleconf", action="store_true", help="Display example configuration.")
args = parser.parse_args()

# Display example configuration and exit if "-e" option is used
if args.exampleconf:
    print(EXAMPLE_CONFIG)
    exit(0)

# Process configuration if "-c" option is used
if args.config:
    if args.config == 'example':
        config_content = EXAMPLE_CONFIG
    else:
        print("Starting cksgen...")
        config_filename = args.config + ".conf"
        with open(config_filename, 'r') as config_file:
            config_content = config_file.read()

    # Parse the configuration content
    config = configparser.ConfigParser()
    config.read_string(config_content)

    # Extract user settings from the configuration
    allowed_extensions = [ext.strip() for ext in config.get('USER_SETTINGS', 'allowed_extensions').split(';')]
    data_directory = config.get('USER_SETTINGS', 'data_directory')
    lists_directory = os.path.join(os.path.dirname(__file__), config.get('USER_SETTINGS', 'lists_directory'))
    logs_directory = os.path.join(os.path.dirname(__file__), config.get('USER_SETTINGS', 'logs_directory'))
    files_to_keep = int(config.get('USER_SETTINGS', 'files_to_keep'))

    # Function to load configuration from a file
    def load_config(config_path):
        config = configparser.ConfigParser()
        config.read(config_path)
        return config['USER_SETTINGS']

    # Function to delete old files based on number to keep
    def delete_old_files(directory, files_to_keep):
        all_files = glob.glob(os.path.join(directory, '*.*'))
        all_files.sort(key=os.path.getmtime, reverse=True)

        files_to_delete = all_files[files_to_keep:]
        for file in files_to_delete:
            os.remove(file)

    # Function to compare checksum files for differences
    def compare_checksum_files(file1, file2):
        with open(file1, 'r') as f1, open(file2, 'r') as f2:
            checksums1 = f1.read().splitlines()[1:]  # Skip the first line
            checksums2 = f2.read().splitlines()[1:]  # Skip the second line

        different_files = []

        if not checksums2:
            return ["<single_file>"]

        for line1 in checksums1:
            checksum1, file1 = line1.split('\t')
            for line2 in checksums2:
                checksum2, file2 = line2.split('\t')
                if file1 == file2 and checksum1 != checksum2:
                    different_files.append(file1)
                    break

        return different_files

    # Main execution block
    if __name__ == "__main__":
        # Parse command line arguments
        parser = argparse.ArgumentParser(description="Generate checksums for files.")
        parser.add_argument("-conf", "--config", type=str, help="Configuration file name without extension")

        args = parser.parse_args()

        # If a configuration file is specified
        if args.config:
            config_filename = args.config + ".conf"
            config = load_config(config_filename)

            # Extract settings from the configuration
            conf_name = config.get('conf_name')
            allowed_extensions = config.get('allowed_extensions').split(',')
            data_directory = config.get('data_directory')
            files_to_keep = int(config.get('files_to_keep'))

            # Get the script's directory
            script_directory = os.path.dirname(os.path.abspath(__file__))

            # Create paths for lists and logs directories
            lists_directory = os.path.join(script_directory, conf_name, 'Lists')
            logs_directory = os.path.join(script_directory, conf_name, 'Logs')

            # Create the directories if they don't exist
            os.makedirs(lists_directory, exist_ok=True)
            os.makedirs(logs_directory, exist_ok=True)

            # Generate timestamp and datetime strings
            current_timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
            current_datetime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')

            # Generate filenames for checksum and log files
            checksum_filename = os.path.join(lists_directory, f'{current_timestamp}_checksum.txt')
            log_filename = os.path.join(logs_directory, f'{current_timestamp}_log.txt')

            # Function to scan a directory and generate checksums
            def scan_directory(directory):
                with open(checksum_filename, 'a') as f:
                    f.write(f"MD5 checksums on: {current_datetime}\n")

                # Walk through the directory tree
                for root, _, files in os.walk(directory):
                    for file in files:
                        file_path = os.path.join(root, file)
                        extension = file_path.split('.')[-1].lower()

                        # Calculate checksum for allowed file types
                        if extension in allowed_extensions:
                            md5_checksum = hashlib.md5(open(file_path, 'rb').read()).hexdigest()

                            # Write checksum and file path to the checksum file
                            with open(checksum_filename, 'a') as f:
                                f.write(f'{md5_checksum}\t{file_path}\n')

                            # Display progress
                            print(f'Processing {file_path}')

            # Scan the specified data directory
            scan_directory(data_directory)

            # Create a log entry for successful checksum generation
            log_entry = f'MD5 checksums have been created and stored in {checksum_filename}.\n'
            with open(log_filename, 'a') as f:
                f.write(log_entry)

            # Delete old files based on the specified number to keep
            delete_old_files(lists_directory, files_to_keep)
            delete_old_files(logs_directory, files_to_keep)

            # Compare the last two checksum files for differences
            last_checksum_files = glob.glob(os.path.join(lists_directory, '*_checksum.txt'))
            if len(last_checksum_files) >= 2:
                # Sort the list of files in reverse order (newest first)
                last_checksum_files.sort(reverse=True)

                # Get the paths of the two most recent checksum files
                last_checksum_file1 = last_checksum_files[0]
                last_checksum_file2 = last_checksum_files[1]

                # Compare the checksums in the two files
                different_files = compare_checksum_files(last_checksum_file1, last_checksum_file2)

                log_message = ''

                # If there are different checksums, log and print the differences
                if different_files:
                    print('ATTENTION: Different MD5 checksums found! See log file in', log_filename)
                    log_message += 'ATTENTION: The following files have different checksums:\n'
                    for file in different_files:
                        log_message += file + '\n'
                else:
                     # If no differences, log and print a message
                    log_message += 'INFO: No different MD5 checksums found.\n'
                    print('INFO: No different MD5 checksums found.')

                # Write the log message to the log file
                with open(log_filename, 'a') as f:
                    f.write(log_message)

            # If there's only one checksum file, log and print a message
            elif len(last_checksum_files) == 1:
                with open(log_filename, 'a') as f:
                    f.write("INFO: Checksums could not be compared because there is currently only one checksum file.\n")

                print("INFO: Checksums could not be compared because there is currently only one checksum file.")

            print(log_entry)