multimodal_report

multimodal_report

Classes:

Name	Description
MultimodalReport	Multi-modal evaluation report combining quality and privacy components.

MultimodalReport pydantic-model

Bases: EvaluationReport

Multi-modal evaluation report combining quality and privacy components.

Assembles all evaluation components (SQS sub-metrics for tabular and/or text columns, privacy scores, PII replay, dataset statistics) from paired reference/output dataframes and renders them into an HTML report via Jinja2 templates.

Use from_dataframes to construct a fully populated report.

Fields:

• evaluation_dataset (EvaluationDataset)
• components (list[Component])
• config (SafeSynthesizerParameters | None)

config = None pydantic-field

Pipeline configuration parameters used for this evaluation.

jinja_context cached property

Template context with tooltips, flags, and per-column distribution figures.

from_dataframes(reference, output, test=None, column_statistics=None, config=None) staticmethod

Build a complete multi-modal evaluation report from dataframes.

Constructs an EvaluationDataset, runs all enabled evaluation components (quality and privacy), and assembles them into a report.

Parameters:

Name	Type	Description	Default
reference	DataFrame	Training (reference) dataframe.	required
output	DataFrame	Synthetic (output) dataframe.	required
test	DataFrame | None	Optional holdout dataframe for privacy metrics.	None
column_statistics	dict[str, ColumnStatistics] | None	Per-column PII entity metadata.	None
config	SafeSynthesizerParameters | None	Pipeline configuration controlling which metrics are enabled.	None

Returns:

Type	Description
MultimodalReport	A fully populated MultimodalReport ready for rendering.

Source code in src/nemo_safe_synthesizer/evaluation/reports/multimodal/multimodal_report.py

@staticmethod
def from_dataframes(
    reference: pd.DataFrame,
    output: pd.DataFrame,
    test: pd.DataFrame | None = None,
    column_statistics: dict[str, ColumnStatistics] | None = None,
    config: SafeSynthesizerParameters | None = None,
) -> MultimodalReport:
    """Build a complete multi-modal evaluation report from dataframes.

    Constructs an ``EvaluationDataset``, runs all enabled evaluation
    components (quality and privacy), and assembles them into a report.

    Args:
        reference: Training (reference) dataframe.
        output: Synthetic (output) dataframe.
        test: Optional holdout dataframe for privacy metrics.
        column_statistics: Per-column PII entity metadata.
        config: Pipeline configuration controlling which metrics are enabled.

    Returns:
        A fully populated ``MultimodalReport`` ready for rendering.
    """
    evaluation_dataset = EvaluationDataset.from_dataframes(
        reference=reference,
        output=output,
        test=test,
        column_statistics=column_statistics,
        rows=MultimodalReport._get_config_value("sqs_rows", DEFAULT_RECORD_COUNT, config),
        cols=MultimodalReport._get_config_value("sqs_columns", DEFAULT_SQS_REPORT_COLUMNS, config),
        mandatory_columns=MultimodalReport._get_config_value("mandatory_columns", [], config),
    )

    components = []

    attribute_inference_protection = AttributeInferenceProtection(
        score=EvaluationScore(grade=PrivacyGrade.UNAVAILABLE)
    )
    if config and config.get("aia_enabled"):
        attribute_inference_protection = AttributeInferenceProtection.from_evaluation_dataset(
            evaluation_dataset, config
        )
    components.append(attribute_inference_protection)

    membership_inference_protection = MembershipInferenceProtection(
        score=EvaluationScore(grade=PrivacyGrade.UNAVAILABLE)
    )
    if config and config.get("mia_enabled"):
        membership_inference_protection = MembershipInferenceProtection.from_evaluation_dataset(evaluation_dataset)
    components.append(membership_inference_protection)

    data_privacy_score = DataPrivacyScore(score=EvaluationScore(grade=PrivacyGrade.UNAVAILABLE))
    if attribute_inference_protection or membership_inference_protection:
        data_privacy_score = DataPrivacyScore.from_components(
            [attribute_inference_protection, membership_inference_protection]
        )
    components.append(data_privacy_score)

    pii_replay = PIIReplay()
    if column_statistics and config is not None and config.get("pii_replay_enabled"):
        # PII Replay requires full df's. Make a one-off dataset for that call.
        pii_replay = PIIReplay.from_evaluation_dataset(
            EvaluationDataset.from_dataframes(
                reference=reference,
                output=output,
                test=test,
                column_statistics=column_statistics,
                rows=MultimodalReport._get_config_value("sqs_rows", DEFAULT_RECORD_COUNT, config),
                cols=MultimodalReport._get_config_value("sqs_columns", DEFAULT_SQS_REPORT_COLUMNS, config),
                mandatory_columns=MultimodalReport._get_config_value("mandatory_columns", [], config),
                enable_sampling=False,
            ),
            config=config,
        )
    # drop columns with assigned type "text" from pii_replay.
    # TODO: This will be removed once text entities is added to the PII Replay section of the evaluation report.
    pii_replay.pii_replay_data = [
        datum for datum in pii_replay.pii_replay_data if datum.column_assigned_type != "text"
    ]
    components.append(pii_replay)

    dataset_statistics = DatasetStatistics.from_evaluation_dataset(evaluation_dataset)

    column_distribution = ColumnDistribution.from_evaluation_dataset(evaluation_dataset)
    correlation = Correlation.from_evaluation_dataset(evaluation_dataset)
    deep_structure = DeepStructure.from_evaluation_dataset(evaluation_dataset)
    text_semantic_similarity = TextSemanticSimilarity.from_evaluation_dataset(evaluation_dataset)
    text_structure_similarity = TextStructureSimilarity.from_evaluation_dataset(evaluation_dataset)
    sqs_score = SQSScore.from_components(
        [column_distribution, correlation, deep_structure, text_semantic_similarity, text_structure_similarity]
    )

    components += [
        dataset_statistics,
        column_distribution,
        correlation,
        deep_structure,
        text_semantic_similarity,
        text_structure_similarity,
        sqs_score,
    ]

    report = MultimodalReport(config=config, evaluation_dataset=evaluation_dataset, components=components)
    report.evaluation_dataset = evaluation_dataset
    return report