release-scripts/backup/database_backup-script.py

import os
import datetime
import subprocess
import logging
import sys
import boto3
from botocore.exceptions import ClientError
from dotenv import load_dotenv # Import load_dotenv
from email_utils import send_email

# Load environment variables from .env file
load_dotenv()

# --- Configuration ---
# Variables are now loaded from the .env file using os.getenv()
DB_HOST = os.getenv('DB_HOST')
DB_USER = os.getenv('DB_USER')
DB_PASSWORD = os.getenv('DB_PASSWORD')

ACCESS_KEY = os.getenv('ACCESS_KEY')
SECRET_KEY = os.getenv('SECRET_KEY')

# --- AWS S3 Configuration ---
S3_BUCKET_NAME = os.getenv('S3_BUCKET_NAME')
S3_REGION = os.getenv('S3_REGION')

# Initialize a message list
email_body_parts = []

# --- Platform-Specific Paths ---
# Determine OS and set paths accordingly
if sys.platform.startswith('win'):
    # Paths for Windows
    # You can also load these from .env if you prefer fine-grained control
    BACKUP_DIR = os.getenv('BACKUP_DIR', "C:/gita/database/backup") # Default if not in .env
    MYSQLDUMP_PATH = os.getenv('MYSQLDUMP_PATH', r'C:\Program Files\MySQL\MySQL Server 8.0\bin\mysqldump.exe')
    LOG_DIR = os.getenv('BACKUP_DIR', "C:/gita/database/backup") # Default if not in .env
    LOG_FILE = os.getenv('LOG_FILE', r'C:\gita\backup_log.txt')
    GIT_EXECUTABLE = "git" # Assuming git is in PATH on Windows
else:
    # Paths for Ubuntu/Linux
    BACKUP_DIR = os.getenv('BACKUP_DIR', "/var/lib/mysql-backups") # Default if not in .env
    MYSQLDUMP_PATH = os.getenv('MYSQLDUMP_PATH', "/usr/bin/mysqldump")
    LOG_FILE = os.getenv('LOG_FILE', "/var/logs/mysql-backup/mysql_backup.log")
    LOG_DIR = os.getenv('LOG_DIR', "/var/logs/mysql-backup")
    GIT_EXECUTABLE = "git" # Assuming git is in PATH on Linux

# --- Logging Setup ---
# Configure logging to write messages to a file and to the console
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler(LOG_FILE),
        logging.StreamHandler()
    ]
)

# --- Helper Functions (unchanged from previous version) ---

def build_path(database_name):
    timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
    backup_filename = f"{timestamp}-{database_name}-database-backup.sql"
    backup_path = os.path.join(BACKUP_DIR, backup_filename)
    return backup_path

def build_mysqldump_command(database_name):
    command = [
        MYSQLDUMP_PATH,
        f"-h{DB_HOST}",
        f"-u{DB_USER}",
        f"-p{DB_PASSWORD}",
        database_name
    ]
    return command
def clean_backup_folder():
    folder_path = BACKUP_DIR
    for filename in os.listdir(folder_path):
        file_path = os.path.join(folder_path, filename)
        if os.path.isfile(file_path):
            os.remove(file_path)

def start_backup(database):
    backup_file_path = build_path(database)
    command = build_mysqldump_command(database)
    try:
        logging.info(f"Starting backup for database: {database} to {backup_file_path}")
        with open(backup_file_path, "w", encoding="utf-8") as out_file:
            subprocess.run(command, stdout=out_file, check=True, text=True)
        logging.info(f"Successfully backed up {database}.")
        email_body_parts.append(f"Successfully backed up {database}.")
        return backup_file_path
    except subprocess.CalledProcessError as e:
        logging.error(f"MySQL dump failed for {database}: {e}")
        logging.error(f"Command: {' '.join(e.cmd)}")
        email_body_parts.append(f"MySQL dump failed for {database}: {e}")
        if e.stderr:
            logging.error(f"Stderr: {e.stderr}")
        return None
    except FileNotFoundError:
        logging.error(f"Error: mysqldump not found at '{MYSQLDUMP_PATH}'. Please verify the path.")
        if sys.platform.startswith('win'):
            logging.error(r"On Windows, ensure MySQL is installed and 'mysqldump.exe' path is correct (e.g., C:\Program Files\MySQL\MySQL Server 8.0\bin\mysqldump.exe)")
        else:
            logging.error("On Ubuntu/Linux, you might need to install 'mysql-client' package: sudo apt install mysql-client")
        return None
    except Exception as e:
        logging.error(f"An unexpected error occurred during backup of {database}: {e}")
        return None

def upload_to_git():
    original_cwd = os.getcwd()
    try:
        logging.info(f"Changing directory to {BACKUP_DIR} for Git operations.")
        os.chdir(BACKUP_DIR)

        logging.info("Staging all changes in Git repository...")
        subprocess.run([GIT_EXECUTABLE, "add", "."], check=True, text=True)
        commit_message = f"Automated database backup commit on {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
        logging.info(f"Committing changes with message: '{commit_message}'")
        subprocess.run([GIT_EXECUTABLE, "commit", "-m", commit_message], check=True, text=True)
        logging.info("Pushing changes to remote Git repository...")
        subprocess.run([GIT_EXECUTABLE, "push"], check=True, text=True)

        logging.info("Backup files successfully pushed to Git repository.")
    except subprocess.CalledProcessError as e:
        logging.error(f"Git upload failed: {e}")
        logging.error(f"Command: {' '.join(e.cmd)}")
        if e.stderr:
            logging.error(f"Stderr: {e.stderr}")
        raise
    except FileNotFoundError:
        logging.error(f"Error: Git executable ('{GIT_EXECUTABLE}') not found. Ensure Git is installed and in your system's PATH.")
        raise
    except Exception as e:
        logging.error(f"An unexpected error occurred during Git upload: {e}")
        raise
    finally:
        os.chdir(original_cwd)

def remove_before_first_dash(input_string):
    """
    Removes the part of a string before the first dash.

    Args:
        input_string: The string to process.

    Returns:
        The part of the string after the first dash, or the original string
        if no dash is found.
    """
    parts = input_string.split('-', 1)  # Split only at the first dash
    if len(parts) > 1:
        return parts[1]
    else:
        return input_string
    
def upload_to_s3(file_paths):
    if not file_paths:
        logging.info("No backup files to upload to S3.")
        return

    # Basic validation for S3 configuration
    if not all([S3_BUCKET_NAME, S3_REGION, ACCESS_KEY, SECRET_KEY]):
        logging.error("S3_BUCKET_NAME, S3_REGION, ACCESS_KEY or SECRET_KEY is not set. Cannot upload to S3. Please check your .env file.")
        email_body_parts.append(f"S3_BUCKET_NAME, S3_REGION, ACCESS_KEY or SECRET_KEY is not set. Cannot upload to S3. Please check your .env file.")

        return

    try:
        logging.info(f"Attempting to connect to AWS S3 bucket: {S3_BUCKET_NAME} in region: {S3_REGION}")
        s3_client = boto3.client('s3',
                                  region_name=S3_REGION,
                                  aws_access_key_id=ACCESS_KEY,
                                  aws_secret_access_key=SECRET_KEY)

        for file_path in file_paths:
            if not os.path.exists(file_path):
                logging.warning(f"File not found, skipping S3 upload: {file_path}")
                continue

            s3_object_key = os.path.basename(file_path)
            s3_object_key = remove_before_first_dash(s3_object_key)
            try:
                logging.info(f"Uploading {s3_object_key} to s3://{S3_BUCKET_NAME}/{s3_object_key}")
                s3_client.upload_file(file_path, S3_BUCKET_NAME, s3_object_key)
                logging.info(f"Successfully uploaded {s3_object_key} to S3.")
                email_body_parts.append(f"Successfully uploaded {s3_object_key} to S3.")

            except ClientError as ce:
                logging.error(f"Failed to upload {s3_object_key} to S3: {ce}")
                email_body_parts.append("Failed to upload {s3_object_key} to S3: {ce}")

                if ce.response['Error']['Code'] == 'AccessDenied':
                    logging.error("S3 upload access denied. Check your AWS credentials and bucket policy.")
                    email_body_parts.append("S3 upload access denied. Check your AWS credentials and bucket policy.")
                elif ce.response['Error']['Code'] == 'NoSuchBucket':
                    logging.error(f"S3 bucket '{S3_BUCKET_NAME}' does not exist or you don't have access.")
                    email_body_parts.append(f"S3 bucket '{S3_BUCKET_NAME}' does not exist or you don't have access.")

                else:
                    logging.error(f"S3 ClientError details: {ce.response['Error']['Code']} - {ce.response['Error']['Message']}")
                    email_body_parts.append(f"S3 ClientError details: {ce.response['Error']['Code']} - {ce.response['Error']['Message']}")

            except Exception as e:
                logging.error(f"An unexpected error occurred during S3 upload of {s3_object_key}: {e}")
        logging.info("All S3 uploads attempted.")
    except Exception as e:
        logging.critical(f"Failed to initialize S3 client or a critical S3 error occurred: {e}")
        email_body_parts.append(f"Failed to initialize S3 client or a critical S3 error occurred: {e}")



# --- Main Execution (unchanged from previous version) ---
if __name__ == "__main__":
    logging.info("--- Database Backup Process Started ---")
    backup_files_created = []

    # Basic validation that essential DB connection variables are loaded
    if not all([DB_HOST, DB_USER, DB_PASSWORD]):
        logging.critical("Missing essential database connection variables. Please check your .env file.")
        exit(1)

    try:
        os.makedirs(BACKUP_DIR, exist_ok=True)
        logging.info(f"Ensured backup directory exists: {BACKUP_DIR}")
        os.makedirs(LOG_DIR, exist_ok=True)
        logging.info(f"Ensured log directory exists: {LOG_DIR}")

        # Get the database list array from the environment variable
        DB_LIST_TO_BACKUP = os.getenv('DB_LIST_TO_BACKUP')

        # Check if the variable exists and is not empty before splitting
        if DB_LIST_TO_BACKUP:
            # Split the string by the comma delimiter
            databases_array_to_backup = [item.strip() for item in DB_LIST_TO_BACKUP.split(',')]
            logging.info(f"Backup databases for:  {databases_array_to_backup}")
        else:
            logging.error(f"database list array (DB_LIST_TO_BACKUP) not found or is empty.")

        clean_backup_folder()
        email_body_parts.append(f"Starting backup for database: {databases_array_to_backup}")
        email_body_parts.append(f"-------------------------------------------------------------")
        for db_name in databases_array_to_backup:
            file_path = start_backup(db_name)
            if file_path:
                backup_files_created.append(file_path)
        email_body_parts.append(f"")
        
        email_body_parts.append(f"Starting Git upload process...")
        email_body_parts.append(f"-------------------------------------------------------------")

        if os.getenv('UPLOAD_TO_GIT', 'false').lower() == 'true':
            logging.info("Starting Git upload process...")            
            upload_to_git()
        else:
            logging.info("Disabled Git upload process...")
            email_body_parts.append(f"Disabled Git upload process...")

        email_body_parts.append(f"<br>Starting S3 upload process... ")
        email_body_parts.append(f"-------------------------------------------------------------")


        if os.getenv('UPLOAD_TO_S3', 'false').lower() == 'true':
            logging.info("Starting S3 upload process...")
            upload_to_s3(backup_files_created)
        else:
            logging.info("Disabled S3 upload process...")
            email_body_parts.append(f"Disabled S3 upload process...")


        # Send HTML email to multiple recipients
        # Final stage: send the mail
        email_body_parts.append(f"<br><br><br>Starting sending mail")
        email_body ="<br>".join(email_body_parts)  # for plain text
        EMAIL_RECEIVERS = os.getenv('EMAIL_RECEIVERS')
        if EMAIL_RECEIVERS:
            # Split the string by the comma delimiter
            email_receivers_array = [item.strip() for item in EMAIL_RECEIVERS.split(',')]
            send_email(
                subject=os.getenv("EMAIL_SUBJECT", "Database backup process"),
                body=email_body,
                to_emails=email_receivers_array,
                html=True                
            )
            logging.info(f"Send Mail to:  {email_receivers_array}")
        else:
            logging.info(f"database list array (DB_LIST_TO_BACKUP) not found or is empty.")
        

        logging.info("--- Database Backup Process Completed Successfully ---")


        exit(0)

    except Exception as ex:
        logging.critical(f"--- Database Backup Process Failed: {ex} ---")
        exit(1)