import os import datetime import subprocess import logging import sys import boto3 from botocore.exceptions import ClientError from email_utils import send_email from gzip_util import gzip_file import json # For reading JSON config # Initialize a message list email_body_parts = [] BACKUP_DIR = "/var/www/apps/db-backup-script/backups" # Example default MYSQLDUMP_PATH = "/usr/bin/mysqldump" # Example default LOG_DIR = "/var/www/apps/db-backup-script/logs" LOG_FILE = "/var/www/apps/db-backup-script/logs/mysql_backup.log" # Assuming your JSON file is named 'config.json' and is in the same directory CONFIG_FILE_PATH = 'config.json' def load_config(file_path): with open(file_path, 'r') as f: return json.load(f) def set_platform_specific_paths(config): # Determine OS and set paths accordingly if sys.platform.startswith('win'): # Paths for Windows # You can also load these from .env if you prefer fine-grained control BACKUP_DIR = config['WINDOWS_PATHS']['BACKUP_DIR'] # Default if not in .env MYSQLDUMP_PATH = config['WINDOWS_PATHS']['MYSQLDUMP_PATH'] LOG_DIR = config['WINDOWS_PATHS']['LOG_DIR'] LOG_FILE = config['WINDOWS_PATHS']['LOG_FILE'] else: if 'LINUX_PATHS' in config: # Paths for Ubuntu/Linux BACKUP_DIR = config['LINUX_PATHS']['BACKUP_DIR'] MYSQLDUMP_PATH = config['LINUX_PATHS']['MYSQLDUMP_PATH'] LOG_DIR = config['LINUX_PATHS']['LOG_DIR'] LOG_FILE = config['LINUX_PATHS']['LOG_FILE'] else: print("Error: Linux paths not configured in JSON.") # Handle this error appropriately, e.g., exit or use default Linux paths. BACKUP_DIR = "/var/www/apps/db-backup-script/backups" # Example default MYSQLDUMP_PATH = "/usr/bin/mysqldump" # Example default LOG_DIR = "/var/www/apps/db-backup-script/logs" LOG_FILE = "/var/www/apps/db-backup-script/logs/mysql_backup.log" try: os.makedirs(BACKUP_DIR, exist_ok=True) logging.info(f"Ensured backup directory exists: {BACKUP_DIR}") os.makedirs(LOG_DIR, exist_ok=True) logging.info(f"Ensured log directory exists: {LOG_DIR}") except Exception as ex: logging.critical(f"---Unable to create required folders. Database Backup Process Failed: {ex} ---") exit(1) # --- Logging Setup --- # Configure logging to write messages to a file and to the console logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.FileHandler(LOG_FILE), logging.StreamHandler() ] ) # --- Helper Functions (unchanged from previous version) --- def build_path(database_name): timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') backup_filename = f"{timestamp}-{database_name}-database-backup.sql" backup_path = os.path.join(BACKUP_DIR, backup_filename) return backup_path def build_mysqldump_command(db_conf): command = [ MYSQLDUMP_PATH, f"-h{db_conf["DB_HOST"]}", f"-u{db_conf["DB_USER"]}", f"-p{db_conf["DB_PASSWORD"]}", db_conf["DB_NAME"] ] return command def clean_backup_folder(): folder_path = BACKUP_DIR for filename in os.listdir(folder_path): file_path = os.path.join(folder_path, filename) if os.path.isfile(file_path): os.remove(file_path) def start_backup(db_conf): backup_file_path = build_path(db_conf["DB_NAME"]) command = build_mysqldump_command(db_conf) try: logging.info(f"Starting backup for database: {db_conf["DB_NAME"]} to {backup_file_path}") with open(backup_file_path, "w", encoding="utf-8") as out_file: subprocess.run(command, stdout=out_file, check=True, text=True) logging.info(f"Successfully backed up {db_conf["DB_NAME"]}.") # Compress and delete original gzipped_path = gzip_file(backup_file_path, delete_original=True) email_body_parts.append(f"Successfully backed up {db_conf["DB_NAME"]}.") #return backup_file_path return gzipped_path except subprocess.CalledProcessError as e: logging.error(f"MySQL dump failed for {db_conf["DB_NAME"]}: {e}") logging.error(f"Command: {' '.join(e.cmd)}") email_body_parts.append(f"MySQL dump failed for {db_conf["DB_NAME"]}: {e}") if e.stderr: logging.error(f"Stderr: {e.stderr}") return None except FileNotFoundError: logging.error(f"Error: mysqldump not found at '{MYSQLDUMP_PATH}'. Please verify the path.") if sys.platform.startswith('win'): logging.error(r"On Windows, ensure MySQL is installed and 'mysqldump.exe' path is correct (e.g., C:\Program Files\MySQL\MySQL Server 8.0\bin\mysqldump.exe)") else: logging.error("On Ubuntu/Linux, you might need to install 'mysql-client' package: sudo apt install mysql-client") return None except Exception as e: logging.error(f"An unexpected error occurred during backup of {db_conf["DB_NAME"]}: {e}") return None def upload_to_git(): original_cwd = os.getcwd() GIT_EXECUTABLE = "git" try: logging.info(f"Changing directory to {BACKUP_DIR} for Git operations.") os.chdir(BACKUP_DIR) logging.info("Staging all changes in Git repository...") subprocess.run([GIT_EXECUTABLE, "add", "."], check=True, text=True) commit_message = f"Automated database backup commit on {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" logging.info(f"Committing changes with message: '{commit_message}'") subprocess.run([GIT_EXECUTABLE, "commit", "-m", commit_message], check=True, text=True) logging.info("Pushing changes to remote Git repository...") subprocess.run([GIT_EXECUTABLE, "push"], check=True, text=True) logging.info("Backup files successfully pushed to Git repository.") except subprocess.CalledProcessError as e: logging.error(f"Git upload failed: {e}") logging.error(f"Command: {' '.join(e.cmd)}") if e.stderr: logging.error(f"Stderr: {e.stderr}") raise except FileNotFoundError: logging.error(f"Error: Git executable ('{GIT_EXECUTABLE}') not found. Ensure Git is installed and in your system's PATH.") raise except Exception as e: logging.error(f"An unexpected error occurred during Git upload: {e}") raise finally: os.chdir(original_cwd) def remove_before_first_dash(input_string): """ Removes the part of a string before the first dash. Args: input_string: The string to process. Returns: The part of the string after the first dash, or the original string if no dash is found. """ parts = input_string.split('-', 1) # Split only at the first dash if len(parts) > 1: return parts[1] else: return input_string def upload_to_s3(aws_s3_configuration, file_paths): if not file_paths: logging.info("No backup files to upload to S3.") return # Basic validation for S3 configuration if not all([aws_s3_configuration["S3_BUCKET_NAME"], aws_s3_configuration["S3_REGION"], aws_s3_configuration["ACCESS_KEY"], aws_s3_configuration["SECRET_KEY"]]): logging.error("S3_BUCKET_NAME, S3_REGION, ACCESS_KEY or SECRET_KEY is not set. Cannot upload to S3. Please check your .env file.") email_body_parts.append(f"S3_BUCKET_NAME, S3_REGION, ACCESS_KEY or SECRET_KEY is not set. Cannot upload to S3. Please check your .env file.") return try: logging.info(f"Attempting to connect to AWS S3 bucket: {aws_s3_configuration["S3_BUCKET_NAME"]} in region: {aws_s3_configuration["S3_REGION"]}") s3_client = boto3.client('s3', region_name=aws_s3_configuration["S3_REGION"], aws_access_key_id=aws_s3_configuration["ACCESS_KEY"], aws_secret_access_key=aws_s3_configuration["SECRET_KEY"]) for file_path in file_paths: if not os.path.exists(file_path): logging.warning(f"File not found, skipping S3 upload: {file_path}") continue s3_object_key = os.path.basename(file_path) s3_object_key = remove_before_first_dash(s3_object_key) try: logging.info(f"Uploading {s3_object_key} to s3://{aws_s3_configuration["S3_BUCKET_NAME"]}/{s3_object_key}") s3_client.upload_file( file_path, aws_s3_configuration["S3_BUCKET_NAME"], s3_object_key, ExtraArgs={ 'StorageClass': 'GLACIER_IR', # Glacier Instant Retrieval 'ServerSideEncryption': 'AES256' }) logging.info(f"Uploaded to {aws_s3_configuration["S3_BUCKET_NAME"]}/{s3_object_key} with Glacier Instant Retrieval") email_body_parts.append(f"Successfully uploaded {s3_object_key} to S3.") except ClientError as ce: logging.error(f"Failed to upload {s3_object_key} to S3: {ce}") email_body_parts.append("Failed to upload {s3_object_key} to S3: {ce}") if ce.response['Error']['Code'] == 'AccessDenied': logging.error("S3 upload access denied. Check your AWS credentials and bucket policy.") email_body_parts.append("S3 upload access denied. Check your AWS credentials and bucket policy.") elif ce.response['Error']['Code'] == 'NoSuchBucket': logging.error(f"S3 bucket '{aws_s3_configuration["S3_BUCKET_NAME"]}' does not exist or you don't have access.") email_body_parts.append(f"S3 bucket '{aws_s3_configuration["S3_BUCKET_NAME"]}' does not exist or you don't have access.") else: logging.error(f"S3 ClientError details: {ce.response['Error']['Code']} - {ce.response['Error']['Message']}") email_body_parts.append(f"S3 ClientError details: {ce.response['Error']['Code']} - {ce.response['Error']['Message']}") except Exception as e: logging.error(f"An unexpected error occurred during S3 upload of {s3_object_key}: {e}") logging.info("All S3 uploads attempted.") except Exception as e: logging.critical(f"Failed to initialize S3 client or a critical S3 error occurred: {e}") email_body_parts.append(f"Failed to initialize S3 client or a critical S3 error occurred: {e}") # --- Main Execution (unchanged from previous version) --- if __name__ == "__main__": logging.info("--- Database Backup Process Started ---") backup_files_created = [] config = load_config(CONFIG_FILE_PATH) set_platform_specific_paths(config) clean_backup_folder() # You would then loop through db_configs and perform backups as in the previous example db_configs = config['DATABASE_CONFIGS'] email_body_parts.append(f"Starting backup for database") email_body_parts.append(f"-------------------------------------------------------------") for db_conf in db_configs: print( db_conf) # Basic validation that essential DB connection variables are loaded if not all([db_conf["DB_HOST"], db_conf["DB_USER"], db_conf["DB_PASSWORD"]]): logging.critical("Missing essential database connection variables. Please check your .env file.") continue email_body_parts.append(f"Starting backup for database: {db_conf["DB_NAME"]}") file_path = start_backup(db_conf) if file_path: backup_files_created.append(file_path) email_body_parts.append(f"") try: email_body_parts.append(f"Starting Git upload process...") email_body_parts.append(f"-------------------------------------------------------------") do_upload_to_git = config['ACTIONS']['DO_UPLOAD_TO_GIT'] if do_upload_to_git: logging.info("Starting Git upload process...") upload_to_git() else: logging.info("Disabled Git upload process...") email_body_parts.append(f"Disabled Git upload process...") email_body_parts.append(f"
Starting S3 upload process... ") email_body_parts.append(f"-------------------------------------------------------------") do_upload_to_s3 = config['ACTIONS']['DO_UPLOAD_TO_S3'] if do_upload_to_s3: logging.info("Starting S3 upload process...") aws_s3_configuration = config['AWS_S3_CONFIGURATION'] upload_to_s3(aws_s3_configuration, backup_files_created) else: logging.info("Disabled S3 upload process...") email_body_parts.append(f"Disabled S3 upload process...") # Send HTML email to multiple recipients # Final stage: send the mail do_send_email = config['ACTIONS']['DO_SEND_EMAIL'] if do_send_email : email_body_parts.append(f"

Backup completed for folloing files:") email_body_parts.append(f"-------------------------------------------------------------") for file_name in backup_files_created: email_body_parts.append(f"{file_name}") email_body_parts.append(f"


Starting sending mail") email_body ="
".join(email_body_parts) # for plain text email_config = config['EMAIL_CONFIGURATION'] if email_config["EMAIL_RECEIVERS"]: # Split the string by the comma delimiter email_receivers_array = [item.strip() for item in email_config["EMAIL_RECEIVERS"].split(',')] if email_config["EMAIL_SUBJECT"] and email_config["EMAIL_SUBJECT"].strip(): # Check if not None and not just whitespace email_subject = email_config["EMAIL_SUBJECT"].strip() else: email_subject = "Database backup process" send_email( email_config["EMAIL_HOST"], email_config["EMAIL_PORT"], email_config["EMAIL_USER"], email_config["EMAIL_PASS"], subject=email_subject, body=email_body, to_emails=email_receivers_array, html=True ) logging.info(f"Send Mail to: {email_receivers_array}") else: logging.info(f"database list array (DB_LIST_TO_BACKUP) not found or is empty.") else: logging.info("Disabled EMAIL send process...") email_body_parts.append(f"Disabled EMAIL send process...") logging.info("--- Database Backup Process Completed Successfully ---") exit(0) except Exception as ex: logging.critical(f"--- Database Backup Process Failed: {ex} ---") exit(1)