Skip to content

pii_replay

pii_replay

Classes:

Name Description
PIIReplayData

Per-column PII data listed in the PII Replay section of the SQS report.

PIIReplay

PII Replay metric -- counts PII values from the training data appearing in the synthetic data.

PIIReplayData pydantic-model

Bases: BaseModel

Per-column PII data listed in the PII Replay section of the SQS report.

Fields:

column_name pydantic-field

The name of the column with PII data.

column_assigned_type pydantic-field

The assigned type for the column (text, unique identifier, date, email, etc.).

pii_type = UNKNOWN_ENTITY pydantic-field

Type of the PII data in the column. For non-text fields, same as column_assigned_type. For text fields, the PII entities detected within the text (race, SSN, address, etc.).

total_training_data = 0 pydantic-field

Total rows in the training data that contain PII values.

unique_training_data = 0 pydantic-field

Count of distinct PII values for this entity in the training column.

total_synthetic_data = 0 pydantic-field

Number of synthetic rows whose column value matches a training PII value.

unique_synthetic_data = 0 pydantic-field

Count of distinct training PII values that appear in the synthetic column.

unique_synthetic_data_percentage = 0 pydantic-field

Percentage of distinct training PII values replayed in the synthetic data (unique_synthetic_data / unique_training_data * 100).

PIIReplay pydantic-model

Bases: Component

PII Replay metric -- counts PII values from the training data appearing in the synthetic data.

For each classified PII entity, reports total and unique replay counts. This component does not produce a numeric score; it surfaces PII leakage details for the HTML report.

Fields:

training_total_records = 0 pydantic-field

Total rows in the training data.

synthetic_total_records = 0 pydantic-field

Total rows in the synthetic data.

pii_replay_data = list() pydantic-field

Per-column / per-entity replay statistics.

skipped_reason = None pydantic-field

Why PII replay could not run, such as a failed PII detection, or None.

jinja_context cached property

Template context with PII replay statistics and entity type list.

from_evaluation_datasets(evaluation_datasets, config=None) staticmethod

Compute PII replay counts from classified entity metadata.

Source code in src/nemo_safe_synthesizer/evaluation/components/pii_replay.py
@staticmethod
def from_evaluation_datasets(
    evaluation_datasets: EvaluationDatasets, config: SafeSynthesizerParameters | None = None
) -> PIIReplay:
    """Compute PII replay counts from classified entity metadata."""
    pii_replay_entities = config.get("pii_replay_entities") if config else None
    pii_replay_columns = config.get("pii_replay_columns") if config else None
    locale = config.get("replace_pii.locale") if config else None
    region = phone_region(locale) if isinstance(locale, str) else None
    statistics = evaluation_datasets.column_statistics or {}
    if not statistics and not pii_replay_columns:
        logger.warning("No classified entities, skipping PII Replay.")
        return PIIReplay(score=EvaluationScore())

    # Build up a list of keys of tuple(col_name, entity_type)
    classified_entities: list[tuple[str, str, str]] = []
    for col, column_statistics in statistics.items():
        entity_names = column_statistics.detected_entity_counts.keys()
        entity_assigned_type = column_statistics.assigned_type or UNKNOWN_ENTITY
        # Scope down to user supplied set of entities if there is one
        if pii_replay_entities:
            entity_names = set(entity_names).intersection(set(pii_replay_entities))
        classified_entities += [(col, entity_name, entity_assigned_type) for entity_name in entity_names]
    # Add user-specified columns without detected PII, comparing the whole column. Columns missing from
    # either dataset are skipped.
    if pii_replay_columns:
        for user_specified_col in sorted(set(pii_replay_columns).difference(statistics.keys())):
            if (
                user_specified_col in evaluation_datasets.training.columns
                and user_specified_col in evaluation_datasets.synthetic.columns
            ):
                classified_entities.append((user_specified_col, UNKNOWN_ENTITY, UNKNOWN_ENTITY))

    pii_replay_data = []
    for col, entity_name, entity_assigned_type in classified_entities:
        # UNKNOWN_ENTITY case, use the entire column
        training_entity_count = len(evaluation_datasets.training[col])
        training_entity_unique_values = evaluation_datasets.training[col].unique()

        if entity_name != UNKNOWN_ENTITY:
            # Ideal case, use the count of detected entities in training for that (col, entity_type).
            # Also get the set of unique values tagged with that entity type.
            training_entity_count = statistics[col].detected_entity_counts[entity_name]
            training_entity_unique_values = statistics[col].detected_entity_values[entity_name]

        substring_match = entity_name != UNKNOWN_ENTITY and statistics[col].assigned_entity == EntityType.FREE_TEXT
        # Values that differ only in letter case, or in how a number is written, count once. So does one
        # phone number written in several ways, in a column of whole phone numbers.
        key_of = (
            _phone_replay_key(region)
            if entity_name == EntityType.PHONE_NUMBER.value and not substring_match
            else _replay_key
        )
        training_keys = {key_of(value) for value in training_entity_unique_values if not _is_missing(value)}
        training_entity_unique_count = len(training_keys)
        synthetic_col = evaluation_datasets.synthetic[col]
        synthetic_entity_count, synthetic_entity_unique_count = _synthetic_replay_counts(
            synthetic_col,
            training_keys,
            substring_match=substring_match,
            key_of=key_of,
        )
        replay_percentage = (
            synthetic_entity_unique_count / training_entity_unique_count * 100
            if training_entity_unique_count
            else 0
        )
        replay_percentage = math.ceil(replay_percentage * 10) / 10

        pii_replay_data.append(
            PIIReplayData(
                column_name=col,
                column_assigned_type=entity_assigned_type,
                pii_type=entity_name,
                total_training_data=training_entity_count,
                unique_training_data=training_entity_unique_count,
                total_synthetic_data=synthetic_entity_count,
                unique_synthetic_data=synthetic_entity_unique_count,
                unique_synthetic_data_percentage=replay_percentage,
            )
        )

    return PIIReplay(
        score=EvaluationScore(),
        training_total_records=evaluation_datasets.training.shape[0],
        synthetic_total_records=evaluation_datasets.synthetic.shape[0],
        pii_replay_data=pii_replay_data,
    )