236 lines
9.6 KiB
Python
236 lines
9.6 KiB
Python
import mysql.connector
|
|
import os
|
|
import base64
|
|
import logging # Import logging
|
|
from datetime import datetime
|
|
from dotenv import load_dotenv # Import load_dotenv
|
|
|
|
# Import the utility functions
|
|
from s3_utils import get_s3_client, download_image_from_s3, upload_object_to_s3, generate_thumbnail_s3_key
|
|
from image_processing_utils import create_thumbnail_from_bytes, create_thumbnail_from_base64
|
|
|
|
# --- Logging Configuration ---
|
|
# Create a logger object
|
|
logger = logging.getLogger(__name__)
|
|
logger.setLevel(logging.INFO) # Set the default logging level
|
|
|
|
# Create handlers
|
|
# Console handler
|
|
console_handler = logging.StreamHandler()
|
|
console_handler.setLevel(logging.INFO) # Console shows INFO and above
|
|
|
|
# File handler
|
|
log_file_name = f"image_processor_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
|
|
file_handler = logging.FileHandler(log_file_name)
|
|
file_handler.setLevel(logging.DEBUG) # File captures all DEBUG messages and above
|
|
|
|
# Create formatters and add them to handlers
|
|
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
console_handler.setFormatter(formatter)
|
|
file_handler.setFormatter(formatter)
|
|
|
|
# Add handlers to the logger
|
|
logger.addHandler(console_handler)
|
|
logger.addHandler(file_handler)
|
|
|
|
logger.info(f"Logging initialized. Full logs available in '{log_file_name}'")
|
|
|
|
|
|
# --- Configuration ---
|
|
DB_CONFIG = {
|
|
'host': 'your_mysql_host',
|
|
'user': 'your_mysql_user',
|
|
'password': 'your_mysql_password',
|
|
'database': 'your_mysql_database'
|
|
}
|
|
|
|
AWS_CONFIG = {
|
|
'aws_access_key_id': 'YOUR_AWS_ACCESS_KEY_ID',
|
|
'aws_secret_access_key': 'YOUR_AWS_SECRET_ACCESS_KEY',
|
|
'region_name': 'your_aws_region' # e.g., 'us-east-1'
|
|
}
|
|
|
|
S3_BUCKET_NAME = 'your-s3-bucket-name'
|
|
THUMBNAIL_PREFIX = 'thumbnails/' # Folder where thumbnails will be stored in S3
|
|
THUMBNAIL_SIZE = (128, 128) # Width, Height for thumbnails
|
|
THUMBNAIL_OUTPUT_FORMAT = 'JPEG' # Output format for thumbnails
|
|
|
|
# --- Helper to connect to DB and S3 ---
|
|
def get_db_connection():
|
|
try:
|
|
mydb = mysql.connector.connect(**DB_CONFIG)
|
|
logger.info("Successfully connected to MySQL database.")
|
|
return mydb
|
|
except mysql.connector.Error as err:
|
|
logger.error(f"Error connecting to MySQL: {err}")
|
|
return None
|
|
|
|
def get_s3_connection():
|
|
s3_client = get_s3_client(**AWS_CONFIG)
|
|
if s3_client:
|
|
logger.info("Successfully obtained S3 client.")
|
|
else:
|
|
logger.error("Failed to obtain S3 client.")
|
|
return s3_client
|
|
|
|
# --- Core Processing Logic ---
|
|
def process_single_image(mycursor, s3_client, image_id, original_image_source_key, base64_image_data=None):
|
|
"""
|
|
Handles the creation and upload of a thumbnail for a single image,
|
|
and updates the database.
|
|
"""
|
|
logger.info(f"Processing image ID: {image_id}")
|
|
image_data_bytes = None
|
|
source_type = "unknown"
|
|
|
|
if original_image_source_key:
|
|
source_type = "S3"
|
|
logger.debug(f"Attempting to download image from S3: '{original_image_source_key}'")
|
|
image_data_bytes = download_image_from_s3(s3_client, S3_BUCKET_NAME, original_image_source_key)
|
|
elif base64_image_data:
|
|
source_type = "Base64"
|
|
logger.debug(f"Attempting to decode base64 image data for image ID: {image_id}")
|
|
image_data_bytes = base64.b64decode(base64_image_data) # We decode here, then pass bytes to create_thumbnail_from_bytes
|
|
else:
|
|
logger.warning(f"No valid image source (S3 link or base64) for image ID {image_id}. Skipping.")
|
|
return
|
|
|
|
if image_data_bytes:
|
|
logger.debug(f"Image data ({source_type}) obtained for ID: {image_id}. Creating thumbnail...")
|
|
thumbnail_bytes = create_thumbnail_from_bytes(
|
|
image_data_bytes,
|
|
size=THUMBNAIL_SIZE,
|
|
output_format=THUMBNAIL_OUTPUT_FORMAT
|
|
)
|
|
|
|
if thumbnail_bytes:
|
|
# Determine the key for the thumbnail
|
|
if original_image_source_key:
|
|
thumbnail_s3_key = generate_thumbnail_s3_key(original_image_source_key, THUMBNAIL_PREFIX)
|
|
else:
|
|
# If only base64 data, we need a way to derive a unique key.
|
|
# A simple approach: use the ID and a timestamp or hash.
|
|
# In a real scenario, you might want to store the original filename
|
|
# or a hash of the base64 data to avoid collisions.
|
|
thumbnail_s3_key = f"{THUMBNAIL_PREFIX}base64_image_{image_id}.{THUMBNAIL_OUTPUT_FORMAT.lower()}"
|
|
logger.debug(f"Generated base64 thumbnail S3 key: '{thumbnail_s3_key}' for ID: {image_id}")
|
|
|
|
content_type = f'image/{THUMBNAIL_OUTPUT_FORMAT.lower()}' # e.g., image/jpeg
|
|
if upload_object_to_s3(s3_client, S3_BUCKET_NAME, thumbnail_s3_key, thumbnail_bytes, content_type=content_type):
|
|
# Update database with new thumbnailLink
|
|
update_sql = "UPDATE images SET thumbnailLink = %s WHERE id = %s"
|
|
try:
|
|
mycursor.execute(update_sql, (thumbnail_s3_key, image_id))
|
|
mydb.commit()
|
|
logger.info(f"Successfully updated database for image ID {image_id} with thumbnail: '{thumbnail_s3_key}'")
|
|
except mysql.connector.Error as err:
|
|
logger.error(f"Failed to update database for image ID {image_id}: {err}")
|
|
mydb.rollback() # Rollback if update fails
|
|
else:
|
|
logger.error(f"Failed to upload thumbnail for image ID {image_id} to S3.")
|
|
else:
|
|
logger.error(f"Failed to create thumbnail bytes for image ID {image_id}.")
|
|
else:
|
|
logger.error(f"No image data retrieved from {source_type} for image ID {image_id}. Skipping thumbnail creation.")
|
|
|
|
def process_images_from_s3_link():
|
|
"""
|
|
Processes images where `imageLink` is present and `thumbnailLink` is NULL.
|
|
"""
|
|
logger.info("Starting processing of images from S3 links...")
|
|
mydb = get_db_connection()
|
|
if not mydb:
|
|
logger.critical("Could not establish database connection. Aborting S3 link processing.")
|
|
return
|
|
|
|
s3 = get_s3_connection()
|
|
if not s3:
|
|
logger.critical("Could not establish S3 connection. Aborting S3 link processing.")
|
|
mydb.close()
|
|
return
|
|
|
|
mycursor = mydb.cursor()
|
|
|
|
try:
|
|
sql = "SELECT id, imageLink FROM images WHERE imageLink IS NOT NULL AND thumbnailLink IS NULL"
|
|
mycursor.execute(sql)
|
|
records = mycursor.fetchall()
|
|
|
|
if not records:
|
|
logger.info("No images found with S3 link and null thumbnailLink. Skipping this step.")
|
|
return
|
|
|
|
logger.info(f"Found {len(records)} images with S3 link and null thumbnailLink. Beginning processing...")
|
|
|
|
for image_id, image_link in records:
|
|
logger.info(f"Initiating processing for image ID: {image_id}, S3 Link: '{image_link}'")
|
|
process_single_image(mycursor, s3, image_id, image_link)
|
|
|
|
except mysql.connector.Error as err:
|
|
logger.error(f"MySQL Error during S3 link processing: {err}")
|
|
except Exception as e:
|
|
logger.error(f"An unexpected error occurred during S3 link processing: {e}", exc_info=True) # exc_info=True to log traceback
|
|
finally:
|
|
if 'mycursor' in locals() and mycursor:
|
|
mycursor.close()
|
|
if mydb.is_connected():
|
|
mydb.close()
|
|
logger.info("MySQL connection closed after S3 link processing.")
|
|
|
|
def process_images_from_base64_string():
|
|
"""
|
|
Processes images where `imageDataBase64` is present and `thumbnailLink` is NULL.
|
|
"""
|
|
logger.info("Starting processing of images from Base64 strings...")
|
|
mydb = get_db_connection()
|
|
if not mydb:
|
|
logger.critical("Could not establish database connection. Aborting Base64 processing.")
|
|
return
|
|
|
|
s3 = get_s3_connection()
|
|
if not s3:
|
|
logger.critical("Could not establish S3 connection. Aborting Base64 processing.")
|
|
mydb.close()
|
|
return
|
|
|
|
mycursor = mydb.cursor()
|
|
|
|
try:
|
|
sql = "SELECT id, imageLink, imageDataBase64 FROM images WHERE imageDataBase64 IS NOT NULL AND thumbnailLink IS NULL"
|
|
mycursor.execute(sql)
|
|
records = mycursor.fetchall()
|
|
|
|
if not records:
|
|
logger.info("No images found with base64 data and null thumbnailLink. Skipping this step.")
|
|
return
|
|
|
|
logger.info(f"Found {len(records)} images with base64 data and null thumbnailLink. Beginning processing...")
|
|
|
|
for image_id, image_link, base64_data in records:
|
|
logger.info(f"Initiating processing for image ID: {image_id} from Base64 data.")
|
|
if base64_data:
|
|
process_single_image(mycursor, s3, image_id, None, base64_data)
|
|
else:
|
|
logger.warning(f"Base64 data for image ID {image_id} is unexpectedly NULL after query. Skipping.")
|
|
|
|
except mysql.connector.Error as err:
|
|
logger.error(f"MySQL Error during Base64 processing: {err}")
|
|
except Exception as e:
|
|
logger.error(f"An unexpected error occurred during Base64 processing: {e}", exc_info=True) # exc_info=True to log traceback
|
|
finally:
|
|
if 'mycursor' in locals() and mycursor:
|
|
mycursor.close()
|
|
if mydb.is_connected():
|
|
mydb.close()
|
|
logger.info("MySQL connection closed after Base64 processing.")
|
|
|
|
if __name__ == "__main__":
|
|
logger.info("Application started.")
|
|
# Process images from S3 links first
|
|
process_images_from_s3_link()
|
|
logger.info("--- Finished processing images from S3 links ---")
|
|
|
|
# Then process images from base64 strings
|
|
process_images_from_base64_string()
|
|
logger.info("--- Finished processing images from base64 strings ---")
|
|
logger.info("All image thumbnail processing complete. Application exiting.") |