From 0bd733ecb8ebc9901a819135124ff9c0211c85c4 Mon Sep 17 00:00:00 2001 From: Vikas Nale Date: Sat, 5 Jul 2025 11:23:27 +0530 Subject: [PATCH] - Handle GZIP in database backup process, so that all the backups are zipped before uploaded to s3 - Add fag to check if emails should be send on process complete like did for GIT and S3 --- {backup => database-backup}/.env | 22 ++++++++ ...nical documentation for system admins.docx | Bin .../database_backup-script.py | 47 ++++++++++++------ {backup => database-backup}/email_utils.py | 0 database-backup/gzip_util.py | 30 +++++++++++ 5 files changed, 83 insertions(+), 16 deletions(-) rename {backup => database-backup}/.env (64%) rename {backup => database-backup}/Technical documentation for system admins.docx (100%) rename {backup => database-backup}/database_backup-script.py (88%) rename {backup => database-backup}/email_utils.py (100%) create mode 100644 database-backup/gzip_util.py diff --git a/backup/.env b/database-backup/.env similarity index 64% rename from backup/.env rename to database-backup/.env index c530031..6865c23 100644 --- a/backup/.env +++ b/database-backup/.env @@ -5,8 +5,30 @@ DB_PASSWORD=xxxxxxx DB_LIST_TO_BACKUP=MarcoBMSProd,MarcoBMSStage,gitea,mediawiki,redmine +DATABASE_CONFIGS = [ + { + 'DB_HOST': '147.93.98.152', + 'DB_USER': 'devuser', + 'DB_PASSWORD': 'AppUser@123$', + 'DB_NAME': 'gitea,mediawiki,redmine', + 'DB_TYPE': 'mysql' # Add database type if you have mixed databases (mysql, postgres, etc.) + }, + { + 'DB_HOST': '147.93.98.152', + 'DB_USER': 'devuser', + 'DB_PASSWORD': 'AppUser@123$', + 'DB_NAME': 'MarcoBMSProd', + 'DB_TYPE': 'mysql' + }, +] + + + +## Actionalble Blocks UPLOAD_TO_S3=true UPLOAD_TO_GIT=false +SEND_EMAIL=false + ## AWS S3 Configuration ACCESS_KEY=xxxxxxx diff --git a/backup/Technical documentation for system admins.docx b/database-backup/Technical documentation for system admins.docx similarity index 100% rename from backup/Technical documentation for system admins.docx rename to database-backup/Technical documentation for system admins.docx diff --git a/backup/database_backup-script.py b/database-backup/database_backup-script.py similarity index 88% rename from backup/database_backup-script.py rename to database-backup/database_backup-script.py index f17298b..2ae8ad5 100644 --- a/backup/database_backup-script.py +++ b/database-backup/database_backup-script.py @@ -7,6 +7,7 @@ import boto3 from botocore.exceptions import ClientError from dotenv import load_dotenv # Import load_dotenv from email_utils import send_email +from gzip_util import gzip_file # Load environment variables from .env file load_dotenv() @@ -88,8 +89,13 @@ def start_backup(database): with open(backup_file_path, "w", encoding="utf-8") as out_file: subprocess.run(command, stdout=out_file, check=True, text=True) logging.info(f"Successfully backed up {database}.") + + # Compress and delete original + gzipped_path = gzip_file(backup_file_path, delete_original=True) + email_body_parts.append(f"Successfully backed up {database}.") - return backup_file_path + #return backup_file_path + return gzipped_path except subprocess.CalledProcessError as e: logging.error(f"MySQL dump failed for {database}: {e}") logging.error(f"Command: {' '.join(e.cmd)}") @@ -271,22 +277,31 @@ if __name__ == "__main__": # Send HTML email to multiple recipients # Final stage: send the mail - email_body_parts.append(f"


Starting sending mail") - email_body ="
".join(email_body_parts) # for plain text - EMAIL_RECEIVERS = os.getenv('EMAIL_RECEIVERS') - if EMAIL_RECEIVERS: - # Split the string by the comma delimiter - email_receivers_array = [item.strip() for item in EMAIL_RECEIVERS.split(',')] - send_email( - subject=os.getenv("EMAIL_SUBJECT", "Database backup process"), - body=email_body, - to_emails=email_receivers_array, - html=True - ) - logging.info(f"Send Mail to: {email_receivers_array}") + if os.getenv('SEND_EMAIL', 'false').lower() == 'true': + email_body_parts.append(f"

Backup completed for folloing files:") + email_body_parts.append(f"-------------------------------------------------------------") + + for file_name in backup_files_created: + email_body_parts.append(f"{file_name}") + + email_body_parts.append(f"


Starting sending mail") + email_body ="
".join(email_body_parts) # for plain text + EMAIL_RECEIVERS = os.getenv('EMAIL_RECEIVERS') + if EMAIL_RECEIVERS: + # Split the string by the comma delimiter + email_receivers_array = [item.strip() for item in EMAIL_RECEIVERS.split(',')] + send_email( + subject=os.getenv("EMAIL_SUBJECT", "Database backup process"), + body=email_body, + to_emails=email_receivers_array, + html=True + ) + logging.info(f"Send Mail to: {email_receivers_array}") + else: + logging.info(f"database list array (DB_LIST_TO_BACKUP) not found or is empty.") else: - logging.info(f"database list array (DB_LIST_TO_BACKUP) not found or is empty.") - + logging.info("Disabled EMAIL send process...") + email_body_parts.append(f"Disabled EMAIL send process...") logging.info("--- Database Backup Process Completed Successfully ---") diff --git a/backup/email_utils.py b/database-backup/email_utils.py similarity index 100% rename from backup/email_utils.py rename to database-backup/email_utils.py diff --git a/database-backup/gzip_util.py b/database-backup/gzip_util.py new file mode 100644 index 0000000..ee4049e --- /dev/null +++ b/database-backup/gzip_util.py @@ -0,0 +1,30 @@ +import gzip +import shutil +import os + +def gzip_file(source_file, delete_original=False): + """ + Compress any file using gzip. + + Args: + source_file (str): Full path to the file to compress. + delete_original (bool): Whether to delete the original file after compression. + + Returns: + str: Path to the gzipped file. + """ + if not os.path.isfile(source_file): + raise FileNotFoundError(f"File not found: {source_file}") + + gzipped_file = source_file + '.gz' + + with open(source_file, 'rb') as f_in: + with gzip.open(gzipped_file, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + if delete_original: + os.remove(source_file) + print(f"Original file deleted: {source_file}") + + print(f"Gzipped file created: {gzipped_file}") + return gzipped_file