prompt_tokens

prompt_tokens

Shared prompt and sequence token-boundary assembly.

Classes:

Name	Description
EncodeOnlyTokenizer	Tokenizer interface required for prompt construction.

Functions:

Name	Description
encode_prompt_token_ids	Encode a prompt with its configured BOS and EOS boundaries.
wrap_sequence_token_ids	Return sequence IDs with a leading BOS and an optional trailing EOS.

EncodeOnlyTokenizer

Bases: Protocol

Tokenizer interface required for prompt construction.

Methods:

Name	Description
encode	Encode text while explicitly controlling tokenizer special tokens.

encode(text, *, add_special_tokens)

Encode text while explicitly controlling tokenizer special tokens.

Source code in src/nemo_safe_synthesizer/data_processing/prompt_tokens.py

def encode(self, text: str, *, add_special_tokens: bool) -> list[int]:
    """Encode text while explicitly controlling tokenizer special tokens."""

encode_prompt_token_ids(prompt, *, tokenizer, prompt_config)

Encode a prompt with its configured BOS and EOS boundaries.

Source code in src/nemo_safe_synthesizer/data_processing/prompt_tokens.py

def encode_prompt_token_ids(
    prompt: str,
    *,
    tokenizer: EncodeOnlyTokenizer,
    prompt_config: LLMPromptConfig,
) -> list[int]:
    """Encode a prompt with its configured BOS and EOS boundaries."""
    prompt_ids = list(tokenizer.encode(prompt, add_special_tokens=False))
    if prompt_config.add_bos_token_to_prompt:
        prompt_ids.insert(0, prompt_config.bos_token_id)
    if prompt_config.add_eos_token_to_prompt:
        prompt_ids.append(prompt_config.eos_token_id)
    return prompt_ids

wrap_sequence_token_ids(token_ids, *, prompt_config, include_eos)

Return sequence IDs with a leading BOS and an optional trailing EOS.

Source code in src/nemo_safe_synthesizer/data_processing/prompt_tokens.py

def wrap_sequence_token_ids(
    token_ids: Sequence[int],
    *,
    prompt_config: LLMPromptConfig,
    include_eos: bool,
) -> list[int]:
    """Return sequence IDs with a leading BOS and an optional trailing EOS."""
    sequence_ids = [prompt_config.bos_token_id, *token_ids]
    if include_eos:
        sequence_ids.append(prompt_config.eos_token_id)
    return sequence_ids