refactor: change submission rejection to file rejection in the error reports (#119)

* refactor: change submission rejection to file rejection in the error reports

Author: georgeRobertson

src/dve/pipeline/pipeline.py

@@ -42,6 +42,7 @@
 from dve.parser.file_handling.implementations.file import LocalFilesystemImplementation
 from dve.parser.file_handling.service import _get_implementation
 from dve.pipeline.utils import SubmissionStatus, deadletter_file, load_config, load_reader
+from dve.reporting.constants import ErrorReportStatus
 from dve.reporting.error_report import ERROR_SCHEMA, calculate_aggregates
 
 PERMISSIBLE_EXCEPTIONS: tuple[type[Exception]] = (
@@ -764,8 +765,8 @@ def _get_error_dataframes(self, submission_id: str):
                     pl.when(pl.col("Status") == pl.lit("informational"))
                     .then(pl.lit("Warning"))
                     .when(pl.col("FailureType") == pl.lit("submission"))  # type: ignore
-                    .then(pl.lit("Submission Failure"))  # type: ignore
-                    .otherwise(pl.lit("Record Rejection"))  # type: ignore
+                    .then(pl.lit(ErrorReportStatus.FILE_REJECTION.reporting_name))  # type: ignore
+                    .otherwise(pl.lit(ErrorReportStatus.RECORD_REJECTION.reporting_name))  # type: ignore
                     .alias("error_type")  # type: ignore
                 )
                 df = df.select(
@@ -828,9 +829,13 @@ def error_report(
             sub_stats = SubmissionStatisticsRecord(
                 submission_id=submission_info.submission_id,
                 record_count=submission_status.number_of_records,
-                number_submission_rejections=err_types.get("Submission Failure", 0),
-                number_record_rejections=err_types.get("Record Rejection", 0),
-                number_warnings=err_types.get("Warning", 0),
+                number_submission_rejections=err_types.get(
+                    ErrorReportStatus.FILE_REJECTION.reporting_name, 0
+                ),
+                number_record_rejections=err_types.get(
+                    ErrorReportStatus.RECORD_REJECTION.reporting_name, 0
+                ),
+                number_warnings=err_types.get(ErrorReportStatus.WARNING.reporting_name, 0),
             )
 
         summary_dict = {
@@ -841,7 +846,7 @@ def error_report(
         summary_items = er.SummaryItems(
             submission_status=submission_status,
             summary_dict=summary_dict,
-            row_headings=["Submission Failure", "Record Rejection", "Warning"],
+            row_headings=[e.reporting_name for e in ErrorReportStatus],
         )
 
         workbook = er.ExcelFormat(

src/dve/reporting/constants.py

@@ -0,0 +1,22 @@
+"""
+Constants used within the error reports
+"""
+
+from enum import Enum
+
+
+class ErrorReportStatus(Enum):
+    """
+    Constant to centrally hold error report status.
+    """
+
+    FILE_REJECTION = 1, "File Rejection"
+    RECORD_REJECTION = 2, "Record Rejection"
+    WARNING = 3, "Warning"
+
+    @property
+    def reporting_name(self):
+        """
+        The error report 'friendly' name.
+        """
+        return self.value[1]

src/dve/reporting/error_report.py

@@ -11,6 +11,7 @@
 from dve.common.error_utils import conditional_cast
 from dve.core_engine.message import FeedbackMessage
 from dve.parser.file_handling.service import open_stream
+from dve.reporting.constants import ErrorReportStatus
 
 ERROR_SCHEMA = {
     "Table": Utf8(),
@@ -85,7 +86,7 @@ def create_error_dataframe(errors: deque[FeedbackMessage], key_fields):
 
     df = df.with_columns(  # type: ignore
         pl.when(pl.col("Status") == pl.lit("error"))  # type: ignore
-        .then(pl.lit("Submission Failure"))  # type: ignore
+        .then(pl.lit(ErrorReportStatus.FILE_REJECTION.reporting_name))  # type: ignore
         .otherwise(pl.lit("Warning"))  # type: ignore
         .alias("error_type")
     )

src/dve/reporting/excel_report.py

@@ -17,6 +17,7 @@
 from polars.exceptions import ColumnNotFoundError
 
 from dve.pipeline.utils import SubmissionStatus
+from dve.reporting.constants import ErrorReportStatus
 
 
 @dataclass
@@ -97,9 +98,9 @@ def get_submission_status(self, aggregates: DataFrame) -> str:
         if aggregates.is_empty():
             return "File has been accepted, no issues to report"
         failures = aggregates["Type"].unique()
-        if "Submission Failure" in failures:
+        if ErrorReportStatus.FILE_REJECTION.reporting_name in failures:
             status = "File has been rejected"
-        elif "Warning" in failures:
+        elif ErrorReportStatus.WARNING.reporting_name in failures:
             status = "File has been accepted, all records accepted with warnings"
         else:
             status = "File has been accepted, no issues to report"

tests/test_pipeline/test_spark_pipeline.py

@@ -440,7 +440,7 @@ def test_error_report_where_report_is_expected(  # pylint: disable=redefined-out
         ("File Name", "doesnotmatter"),
         ("File Extension", "json"),
         ("Total Number of Records Processed", "9"),
-        ("Submission Failure", "0"),
+        ("File Rejection", "0"),
         ("Record Rejection", "2"),
         ("Warning", "0"),
     ]