Skip to content

executor

executor

Plan-driven structured replacement execution.

Classes:

Name Description
ReplacementExecutionResult

Internal structured-execution result returned to the public facade.

ReplacementExecutor

Execute a replacement plan's structured and free-text columns in dependency order on a dataframe copy.

ReplacementExecutionResult(dataframe, column_statistics, generation_statistics, dependency_drifts, replacement_map) dataclass

Internal structured-execution result returned to the public facade.

Attributes:

Name Type Description
dataframe DataFrame

Copy of the input with every structured target replaced.

column_statistics dict[str, ColumnStatistics]

Per-column replacement metadata, keyed by column name.

generation_statistics ReplacementGenerationStatistics

Aggregate generation counts and timing.

dependency_drifts tuple[GroupDependencyDrift, ...]

Group mappings reused although their dependency values changed.

replacement_map ReplacementMap | None

Per-occurrence original and replacement values when capture was requested, otherwise None.

dataframe instance-attribute

Copy of the input with every structured target replaced.

column_statistics instance-attribute

Per-column replacement metadata, keyed by column name.

generation_statistics instance-attribute

Aggregate generation counts and timing.

dependency_drifts instance-attribute

Group mappings reused although their dependency values changed.

replacement_map instance-attribute

Per-occurrence original and replacement values when capture was requested, otherwise None.

ReplacementExecutor(plan, generator, *, group_column, base_seed, free_text_detector=None, capture_replacement_map=False, find_and_replace=False, locale='en_US')

Execute a replacement plan's structured and free-text columns in dependency order on a dataframe copy.

Methods:

Name Description
execute

Return a copy with every structured plan target replaced.

detect

Return the column statistics execute reports, without generating or writing any replacement.

Source code in src/nemo_safe_synthesizer/pii_replacer/replacement/executor.py
def __init__(
    self,
    plan: PiiReplacementPlan,
    generator: ReplacementGenerator,
    *,
    group_column: str | None,
    base_seed: int,
    free_text_detector: FreeTextDetector | None = None,
    capture_replacement_map: bool = False,
    find_and_replace: bool = False,
    locale: str = "en_US",
) -> None:
    self._plan = plan
    self._generator = generator
    self._group_column = group_column
    self._base_seed = base_seed
    self._locale = locale
    self._phone_region = phone_region(locale)
    self._data_to_sampler_value_mapping = _compile_data_to_sampler_value_mapping(plan.data_to_sampler_value_mapping)
    self._full_name_patterns = _full_name_patterns(plan)
    self._capture_replacement_map = capture_replacement_map
    self._cache: dict[StructuredMappingKey, _CachedReplacement] = {}
    self._detector = free_text_detector
    self._address_parts: dict[str, dict[str, AddressParts]] = {}
    self._free_text = FreeTextReplacementExecutor(
        generator,
        free_text_detector,
        base_seed=base_seed,
        capture_replacement_map=capture_replacement_map,
        find_and_replace=find_and_replace,
        locale=locale,
    )
    self._replacement_records: list[StructuredReplacementRecord] = []
    self._shared: dict[_SharedMappingKey, str] = {}
    # The original each replacement went to in every identifier column, so two originals never share one.
    self._owners_by_column: dict[str, ReplacementOwners] = {}
    # Identifier columns whose originals are all distinct, such as key columns; numeric ranges widen for them.
    self._distinct_columns: set[str] = set()
    self._group_replacements: set[str] = set()
    # The same for every column of a format that an identifier or grouping column has. New values in
    # that column avoid other originals' values, so a value another column shares into it later is not a
    # repeat.
    self._owners_by_format: dict[_FormatKey, ReplacementOwners] = {}
    self._repeated_by_target: dict[str, int] = {}
    self._kept_name_parts_by_target: dict[str, int] = {}
    self._identifier_allocators: dict[str, IdentifierAllocator] = {}
    self._dependency_conflicts: dict[tuple[str, frozenset[EntityType]], int] = {}
    self._generation_elapsed = 0.0
    self._generated_count = 0

execute(dataframe)

Return a copy with every structured plan target replaced.

Source code in src/nemo_safe_synthesizer/pii_replacer/replacement/executor.py
def execute(self, dataframe: pd.DataFrame) -> ReplacementExecutionResult:
    """Return a copy with every structured plan target replaced."""
    self._reset_execution_state()
    free_text_specs = [spec for spec in self._plan.columns_to_replace if spec.entity_type is EntityType.FREE_TEXT]

    working = dataframe.copy(deep=True)
    original_group_identities = self._snapshot_group_identities(dataframe)
    self._seed_reserved_values(dataframe)
    self._prepare_identifier_allocators(dataframe)
    for spec in self._plan.columns_to_replace:
        if spec.column_name in self._owners_by_column or spec.column_name == self._group_column:
            self._owners_by_format.setdefault(self._format_key(spec), ReplacementOwners())
    self._address_parts = detect_address_parts(
        self._detector,
        dataframe,
        [spec for spec in self._plan.columns_to_replace if spec.entity_type is EntityType.STREET_ADDRESS],
    )
    free_text_detections = self._free_text.detect(dataframe, free_text_specs)
    ordered_specs = self._plan.replacement_order()
    for spec in ordered_specs:
        if spec.entity_type is not EntityType.FREE_TEXT:
            self._execute_target(working, spec, original_group_identities)
    free_text_result = self._free_text.execute(
        working,
        free_text_specs,
        original_group_identities,
        free_text_detections,
    )
    _verify_result(dataframe, working, self._plan)
    # Free-text fallbacks are warned about with the structured ones, per column.
    for column_name, free_text_statistics in free_text_result.column_statistics.items():
        if free_text_statistics.kept_name_part_count:
            self._kept_name_parts_by_target[column_name] = free_text_statistics.kept_name_part_count

    for column_name, repeated in sorted(self._repeated_by_target.items()):
        logger.user.warning(
            "Some replacements repeat in a column whose original values are all distinct, because no unused "
            "value was found",
            extra={"column": column_name, "repeated_count": repeated},
        )
    for column_name, kept in sorted(self._kept_name_parts_by_target.items()):
        logger.user.warning(
            "Some full-name replacements keep the original's first or last name because no other name was found",
            extra={"column": column_name, "kept_name_part_count": kept},
        )
    drifts = self._dependency_drift_results()
    for drift in drifts:
        logger.user.warning(
            "Some rows reused their group's first replacement although a column it depends on had a different "
            "value in that row",
            extra=drift.as_log_extra(),
        )
    return ReplacementExecutionResult(
        dataframe=working,
        column_statistics=_column_statistics(
            dataframe,
            working,
            self._plan,
            self._generator.backend.value,
            free_text_result.column_statistics,
            drift_by_target=_drift_counts_by_target(drifts),
            repeated_by_target=self._repeated_by_target,
            kept_name_parts_by_target=self._kept_name_parts_by_target,
        ),
        generation_statistics=ReplacementGenerationStatistics(
            generated_replacement_count=(self._generated_count + free_text_result.generated_replacement_count),
            generator_time_seconds=self._generation_elapsed + free_text_result.generator_time_seconds,
            dependency_drift_count=sum(drift.conflict_count for drift in drifts),
        ),
        dependency_drifts=drifts,
        replacement_map=(
            ReplacementMap(
                structured=tuple(self._replacement_records),
                free_text=free_text_result.replacement_records,
            )
            if self._capture_replacement_map
            else None
        ),
    )

detect(dataframe)

Return the column statistics execute reports, without generating or writing any replacement.

Structured targets report their non-missing original values. Free-text targets report the spans execute would replace: detector spans and, with find_and_replace, matches of known values. Each structured value is registered for that matching as its own replacement, so no generator is called. The matches are those of execute, except that a full name always contributes its name words here, while execute skips them when the replacement has a different number of words.

Example

With PII replacement off, the pipeline calls this on the training split so PII replay can still count which original values reappear in the synthetic data.

Source code in src/nemo_safe_synthesizer/pii_replacer/replacement/executor.py
def detect(self, dataframe: pd.DataFrame) -> dict[str, ColumnStatistics]:
    """Return the column statistics ``execute`` reports, without generating or writing any replacement.

    Structured targets report their non-missing original values. Free-text targets report the spans
    ``execute`` would replace: detector spans and, with ``find_and_replace``, matches of known values.
    Each structured value is registered for that matching as its own replacement, so no generator is
    called. The matches are those of ``execute``, except that a full name always contributes its name
    words here, while ``execute`` skips them when the replacement has a different number of words.

    Example:
        With PII replacement off, the pipeline calls this on the training split so PII replay can still
        count which original values reappear in the synthetic data.
    """
    self._reset_execution_state()
    free_text_specs = [spec for spec in self._plan.columns_to_replace if spec.entity_type is EntityType.FREE_TEXT]
    group_identities = self._snapshot_group_identities(dataframe)
    if free_text_specs:
        self._address_parts = detect_address_parts(
            self._detector,
            dataframe,
            [spec for spec in self._plan.columns_to_replace if spec.entity_type is EntityType.STREET_ADDRESS],
        )
        for spec in self._plan.replacement_order():
            if spec.entity_type is not EntityType.FREE_TEXT:
                self._register_originals(dataframe, spec, group_identities)
    free_text_detections = self._free_text.detect(dataframe, free_text_specs)
    free_text_statistics = self._free_text.detect_statistics(
        dataframe, free_text_specs, group_identities, free_text_detections
    )
    return _column_statistics(
        dataframe,
        dataframe,
        self._plan,
        self._generator.backend.value,
        free_text_statistics,
        drift_by_target={},
        repeated_by_target={},
        kept_name_parts_by_target={},
    )