import logging
import os
import re
import datetime
import sys
import argparse
from bigquery.bigquery_edit_order_report_gateway import BigQueryEditOrderReportGateway
from exceptions.configuration_exception import HandlerException, ConfigurationException
from exceptions.task_exception import EnvironmentVariableDidNotMatchRegexError, NumberFormatError, \
NumberFormatWrongLengthError
from pymorri.monitoring.metric_monitoring_manager import MetricManager
from report.customer_notifcation_report import CustomerNotificationReport
from util.application_dependencies import ApplicationDependencies
from bigquery.bigquery_customer_notification_gateway import BigQueryCustomerNotificationGateway
from pymorri.api.morrisons.marketing import MarketingEmail, MarketingSMS
from pymorri.api.morrisons.customer import Customer
from pymorri.exceptions import MoriConfigurationException, MoriException, MoriSSMException
from util.customer_api_data_formatter import CustomerDataFormatter
from util.customer_data_validator import CustomerDataValidator
"""
Application should be run from the commandline below is usage instructions
"""
parser = argparse.ArgumentParser(description='Ecs Task to process email notifications',
epilog='Example: python main.py dev')
parser.add_argument('app_env', metavar='Application Environment', type=str, nargs='+',
help='The application environment where the application is currently running e.g dev or sit')
"""
Logging configuration
"""
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("storepick-edit-order-notification app")
[docs]def run_application():
"""
The main method to run the application.
Initialises dependencies class to retrieve google credentials for big query and application config.
Retrieves order and customer id's for edit order cutoff tomorrow orders from big query.
Uses pymorri Customer API to retrieve each customer's details using id, then formats and validates the data.
The Marketing API then attempts to send an email and sms to each customer found.
The success or failure of these attempts is recorded and sent as a report to a big query table.
"""
try:
metric_monitoring = None
env = get_application_env()
logger.info("Retrieving application dependencies from SSM an Config service...")
dependencies = ApplicationDependencies()
dependencies.execute(env, "storepick-edit-order-notification.json")
monitoring = dependencies.get_monitoring_logger()
metric_monitoring = MetricManager(env, monitoring, "edit-order", cut_off_threshold=100)
logger.info("Metric Monitoring Logs Initialised")
config = dependencies.application_configuration
ssm = dependencies.application_credentials
logger.info("Application dependencies set up with credentials and configurations from SSM")
big_query_client = dependencies.get_google_big_query_client()
logger.info("BigQuery client initialised with credentials")
big_query_manager = BigQueryCustomerNotificationGateway(big_query_client, dependencies)
logger.info("Retrieving BigQuery view for Edit Order Notifications...")
result = big_query_manager.get_all_customers_that_have_deliveries_tomorrow()
logger.info("BigQuery view retrieved successfully")
customer = Customer(config, ssm)
email = MarketingEmail(config, ssm)
email_template_id = dependencies.get_email_template_id()
sms_message = dependencies.get_sms_message()
sms = MarketingSMS(config, ssm)
data_validator = CustomerDataValidator(dependencies)
data_formatter = CustomerDataFormatter(dependencies)
logger.info("Marketing and Customer APIs Initialised")
report_data_list = []
total_num_report_data_sent = 0
metric_monitoring.add_metrics([
("email_invalid", "count"),
("email_sent", "count"),
("email_not_sent", "count"),
("sms_number_invalid", "count"),
("sms_sent", "count"),
("sms_not_sent", "count"),
("app_health", "check"),
])
initial_total = len(result)
metric_monitoring.send_single_metric("initial_total", "count", initial_total)
for res in result:
try:
order_id = res.get("orderId")
customer_id = res.get("retailerCustomerId")
try:
customer_response = customer.request(customer_id)
except Exception as e:
logger.error("The following error occurred while attempting to find customer with retailerCustomerId: "
"{customerId} Error: {error}".format(customerId=customer_id, error=e))
continue
customer_email = customer_response.get("email")
customer_phone = customer_response.get("phoneNumber")
customer_mobile = customer_response.get("mobilePhone")
email_format_ok = data_validator.validate_email(customer_email)
if email_format_ok:
email_payload = {
"email": customer_email,
"context": {}
}
try:
email_response = email.request(email_payload, email_template_id)
logger.info(email_response)
metric_monitoring.increment_metric("email_sent")
email_sent = True
except Exception as e:
logger.error("Email failed to sent: {error}".format(error=e))
metric_monitoring.increment_metric("email_not_sent")
email_sent = False
else:
metric_monitoring.increment_metric("email_invalid")
email_sent = False
if customer_mobile:
try:
customer_mobile = data_formatter.remove_any_non_digit_characters_from_phone_number(customer_mobile)
mobile_valid = data_validator.validate_uk_mobile_number(customer_mobile)
except NumberFormatError as e:
logger.error(e)
mobile_valid = False
except NumberFormatWrongLengthError as e:
logger.error(e)
mobile_valid = False
else:
mobile_valid = False
if customer_phone is not None and mobile_valid is False:
try:
customer_phone = data_formatter.remove_any_non_digit_characters_from_phone_number(customer_phone)
phone_valid = data_validator.validate_uk_mobile_number(customer_phone)
except NumberFormatError as e:
logger.error(e)
phone_valid = False
except NumberFormatWrongLengthError as e:
logger.error(e)
phone_valid = False
else:
phone_valid = False
if mobile_valid:
sms_params_mobile = [{"phoneNumber": customer_mobile, "content": sms_message}]
try:
phone_format_ok = True
sms.request(sms_params_mobile)
metric_monitoring.increment_metric("sms_sent")
sms_sent = True
except Exception as e:
logger.error("Sms failed to send: {error}".format(error=e))
metric_monitoring.increment_metric("sms_not_sent")
sms_sent = False
elif phone_valid:
sms_params_phone = [{"phoneNumber": customer_phone, "content": sms_message}]
try:
phone_format_ok = True
sms.request(sms_params_phone)
metric_monitoring.increment_metric("sms_sent")
sms_sent = True
except Exception as e:
logger.error("Sms failed to send: {error}".format(error=e))
metric_monitoring.increment_metric("sms_not_sent")
sms_sent = False
else:
metric_monitoring.increment_metric("sms_number_invalid")
phone_format_ok = False
sms_sent = False
timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
report_obj = CustomerNotificationReport(order_id, customer_id, phone_format_ok, email_format_ok,
sms_sent, email_sent, timestamp)
report_data = report_obj.get_object_for_report_data()
report_data_list.append(report_data)
metric_monitoring.increment_counter()
if len(report_data_list) % 5000 == 0:
try:
big_query_report = BigQueryEditOrderReportGateway(big_query_client, dependencies)
big_query_report.post_report_of_notifications(report_data_list)
total_num_report_data_sent = total_num_report_data_sent + len(report_data_list)
report_data_list = []
except Exception as e:
logger.error("Report data with 5000 rows failed to send to big query due to following error: ", e)
metric_monitoring.send_single_metric("app_health", "check", 2)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
line = exc_tb.tb_lineno
logger.error(
f"Row/Customer not processed due to an error: {e} (exc_type: {exc_type}, occurred in: {fname}:{line})")
error_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
error_report_data = {"OrderId": "FAILED PROCESS", "RetailerCustomerId": "FAILED PROCESS", "Sms": False,
"Email": False, "NotificationStatus": "FAILED PROCESS",
"Timestamp": error_timestamp}
report_data_list.append(error_report_data)
big_query_report = BigQueryEditOrderReportGateway(big_query_client, dependencies)
big_query_report.post_report_of_notifications(report_data_list)
total_num_report_data_sent = total_num_report_data_sent + len(report_data_list)
metric_monitoring.send_all_added_metrics()
final_total = total_num_report_data_sent
metric_monitoring.send_single_metric("final_total", "count", final_total)
if initial_total == final_total:
logger.info(f"Initial total and final total added to big query report were equal: {initial_total}")
metric_monitoring.send_single_metric("app_health", "check", 0)
else:
logger.error(f"Initial total and final total added to big query report were not equal. "
f"initial: {initial_total} Final: {final_total}")
metric_monitoring.send_single_metric("app_health", "check", 2)
logger.info(f"Process complete with {initial_total}/{final_total} rows of report data "
f"successfully sent")
except EnvironmentVariableDidNotMatchRegexError as e:
logger.error("(Code: {code}) The following error occurred fetching the application environment: {error}"
.format(error=e.detail, code=e.code))
exit(e.code)
except ConfigurationException as e:
logger.error("(Code: {code}) The following configuration error occurred during application execution: {detail}"
.format(detail=e.detail, code=e.code))
if metric_monitoring:
metric_monitoring.send_single_metric("app_health", "check", 2)
exit(e.code)
except HandlerException as e:
logger.error("(Code: {code}) The following handler error occurred during application execution: {detail}"
.format(detail=e.detail, code=e.code))
if metric_monitoring:
metric_monitoring.send_single_metric("app_health", "check", 2)
exit(e.code)
except MoriSSMException as e:
logger.error("(Code: {code}) The following MoriSSMException error occurred during "
"application execution: {detail}".format(detail=e.detail, code=e.code))
if metric_monitoring:
metric_monitoring.send_single_metric("app_health", "check", 2)
exit(e.code)
except MoriConfigurationException as e:
logger.error("(Code: {code}) The following MoriConfigurationException error occurred during "
"application execution: {detail}".format(detail=e.detail, code=e.code))
if metric_monitoring:
metric_monitoring.send_single_metric("app_health", "check", 2)
exit(e.code)
except MoriException as e:
logger.error("(Code: {code}) The following MoriException error occurred during application execution: {detail}"
.format(detail=e.detail, code=e.code))
if metric_monitoring:
metric_monitoring.send_single_metric("app_health", "check", 2)
exit(e.code)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
line = exc_tb.tb_lineno
logger.error(f"Application failed to finish the process due to an error: {e} (exc_type: {exc_type}, occured in: {fname}:{line})")
if metric_monitoring:
metric_monitoring.send_single_metric("app_health", "check", 2)
exit(160)
exit(0)
[docs]def get_application_env():
"""
Method to fetch the application environment from the command line
Returns:
str: A 3 letter environment description.
Examples:
"[0-9]{3,4}" - Building feature branch with number (relates to ticket reference in Jira)
"dev" - Building develop branch
"prd" - production environment
"""
args = parser.parse_args()
env = args.app_env[0]
if not re.match('dev|sit|prd|cpt|uat|cit|[0-9]{4}|[0-9]{3}', env):
raise EnvironmentVariableDidNotMatchRegexError
return env
if __name__ == '__main__':
run_application()