src/nvidia_pipecat/frames/action.py

# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD 2-Clause License

"""ACE Action Frames.

The Action Frames implement the UMIM standard for frame processors.
See here to learn more about the UMIM standard:
https://docs.nvidia.com/ace/umim/latest/index.html

Note that at the moment the support for action frames is limited to the animation graph service.
More services and support for additional action frames will be added in the future.
"""

from dataclasses import dataclass, field
from datetime import UTC, datetime
from uuid import uuid4

from pipecat.frames.frames import ControlFrame, SystemFrame


def now_timestamp() -> datetime:
    """Helper to generate current timestamp."""
    return datetime.now(UTC)


def get_source_id() -> str:
    """Helper to generate a default source_id."""
    return "pipecat"


def new_uid() -> str:
    """Helper to create a new UID."""
    return str(uuid4())


@dataclass
class ActionFrame:
    """Frame that belongs to an action with a defined start and end.

    Args:
        action_id (str): A unique id for the action.
        parent_action_ids (list[str]): The ids of parent actions (causing this action)

    """

    action_id: str = field(kw_only=True, default_factory=new_uid)
    parent_action_ids: list[str] = field(kw_only=True, default_factory=list)


@dataclass
class BotActionFrame(ActionFrame):
    """Frame related to a bot action.

    Args:
        bot_id (Optional[str]):  An ID identifying the bot performing the action. This field is required if you
            support multi-bot interactions.
    """

    bot_id: str | None = field(kw_only=True, default=None)


@dataclass
class UserActionFrame(ActionFrame):
    """Frame related to a user action.

    Args:
        user_id (Optional[str]):  An ID identifying the user performing the action. This field is required if you
            support multi-user interactions.
    """

    user_id: str | None = field(kw_only=True, default=None)


@dataclass
class StartActionFrame(ControlFrame, ActionFrame):
    """Event to start an action.

    All other actions that can be started inherit from this base spec.
    The action_id is used to differentiate between multiple runs of the same action.
    """


@dataclass
class StartedActionFrame(ControlFrame, ActionFrame):
    """The execution of an action has started.

    Args:
        action_started_at (datetime): The timestamp of when the action has started.

    """

    action_id: str = field(kw_only=True)
    action_started_at: datetime = field(kw_only=True, default_factory=now_timestamp)


@dataclass
class StopActionFrame(ControlFrame, ActionFrame):
    """An action needs to be stopped.

    This should be used to proactively stop an action that can take a
    longer period of time, e.g., a gesture.
    """

    action_id: str = field(kw_only=True)


@dataclass
class ChangeActionFrame(ControlFrame, ActionFrame):
    """The parameters of a running action needs to be changed.

    Updating running actions is useful for longer running
    actions (e.g. an avatar animation) which can adapt their behavior dynamically. For example, a nodding animation
    can change its speed depending on the voice activity level.
    """

    action_id: str = field(kw_only=True)


@dataclass
class UpdatedActionFrame(ControlFrame, ActionFrame):
    """A running action provides a (partial) result.

    Ongoing actions can provide partial updates on the current status
    of the action. An ActionUpdated should always update the payload of the action object and provide
    the type of update.

    Args:
        action_updated_at (datetime): The timestamp of when the action was updated. The timestamp should represent
            the system time the action actually changed, not the timestamp of when the `Updated` event was created
            (for this, there is the `event_created_at` field).

    """

    action_updated_at: datetime = field(kw_only=True, default_factory=now_timestamp)


@dataclass
class FinishedActionFrame(ControlFrame, ActionFrame):
    """An action has finished its execution.

    An action can finish either because the action has completed or
    failed (natural completion) or it can finish because it was stopped by the IM. The success (or failure) of the
    execution is marked using the status_code attribute.

    Args:
        action_finished_at (datetime): The timestamp of when the action has finished.
        is_success (bool): Did the action finish successfully
        was_stopped (Optional[bool]): Was the action stopped by a Stop event
        failure_reason (Optional[str]): Reason for action failure in case the action did not execute successfully
    """

    action_id: str = field(kw_only=True)
    action_finished_at: datetime = field(kw_only=True, default_factory=now_timestamp)
    is_success: bool = field(kw_only=True, default=True)
    was_stopped: bool | None = field(kw_only=True, default=None)
    failure_reason: str | None = field(kw_only=True, default=None)


# Facial Gesture Bot Action


@dataclass
class StartFacialGestureBotActionFrame(StartActionFrame, BotActionFrame):
    """The bot should start making a facial gesture.

    Args:
        facial_gesture: Natural language description (NLD) of the facial gesture
            or expression. Availability of facial gestures depends on the
            interactive system.

            Minimal NLD set:
            The following gestures should be supported by every interactive system
            implementing this action:
            - smile
            - laugh
            - frown
            - wink
    """

    facial_gesture: str


@dataclass
class StartedFacialGestureBotActionFrame(StartedActionFrame, BotActionFrame):
    """The bot has started to perform the facial gesture."""


@dataclass
class StopFacialGestureBotActionFrame(StopActionFrame, BotActionFrame):
    """Stop the facial gesture or expression.

    All gestures have a limited lifetime and finish on "their own"
    (e.g., in an interactive avatar system a "smile" gesture could be implemented by a 1 second animation clip where
    some facial bones are animated). The IM can use this action to stop an expression
    before it would be naturally done.
    """


@dataclass
class FinishedFacialGestureBotActionFrame(FinishedActionFrame, BotActionFrame):
    """The facial gesture was performed."""


# Gesture Bot Action


@dataclass
class StartGestureBotActionFrame(StartActionFrame, BotActionFrame):
    """The bot should start making a specific gesture.

    Args:
        gesture: Natural language description (NLD) of the gesture. Availability of gestures depends on the
            interaction system. If a system supports this action, the following base gestures need to be supported:
            `affirm`, `negate`, `attract`
    """

    gesture: str


@dataclass
class StartedGestureBotActionFrame(StartedActionFrame, BotActionFrame):
    """The bot has started to perform the gesture."""


@dataclass
class StopGestureBotActionFrame(StopActionFrame, BotActionFrame):
    """Stop the gesture.

    All gestures have a limited lifetime and finish on 'their own'. Gesture are meant to accentuate
    a certain situation or statement. For example, in an interactive avatar system a `affirm` gesture could be
    implemented by a 1 second animation clip where the avatar nods twice.
    The IM can use this action to stop a gesture before it would be naturally done.
    """


@dataclass
class FinishedGestureBotActionFrame(FinishedActionFrame, BotActionFrame):
    """The gesture was performed."""


# Posture Bot Action


@dataclass
class StartPostureBotActionFrame(StartActionFrame, BotActionFrame):
    """The bot should start adopting the specified posture.

    Args:
        posture (str) : Natural language description (NLD) of the posture. The availability of postures depends on the
            interaction system. Postures should be expressed hierarchically such that interactive systems that provide
            less nuanced postures can fall back onto higher level postures.
            The following base postures need to be supported
            by all interactive systems supporting this action.: "idle", "attentive"
    """

    posture: str


@dataclass
class StartedPostureBotActionFrame(StartedActionFrame, BotActionFrame):
    """The bot has attained the posture."""


@dataclass
class StopPostureBotActionFrame(StopActionFrame, BotActionFrame):
    """Stop the posture.

    Postures have no lifetime, so unless the IM calls the Stop action the bot will
    keep the posture indefinitely.
    """


@dataclass
class FinishedPostureBotActionFrame(FinishedActionFrame, BotActionFrame):
    """The posture was stopped."""


# Position Bot Action


@dataclass
class StartPositionBotActionFrame(StartActionFrame, BotActionFrame):
    """The bot needs to hold a new position.

    Args:
        position (str) : Specify the position the bot needs to move to and maintain.
            Availability of positions depends on the interactive system. Positions are typically
            structured hierarchically into base position and position modifiers ("off center").

            Minimal NLD set:
            The following base positions are supported by all interactive systems (that support this action):
            center : Default position of the bot
            left : Bot should be positioned to the left (from the point of view of the bot)
            right: Bot should be positioned to the right (from the point of view of the bot)
    """

    position: str


@dataclass
class StartedPositionBotActionFrame(StartedActionFrame, BotActionFrame):
    """The bot has started to transition to the new position."""


@dataclass
class UpdatedPositionBotActionFrame(UpdatedActionFrame, BotActionFrame):
    """The bot has arrived at the position and is maintaining that position for the entire action duration.

    Args:
        position_reached (str) : The position the bot has reached.
    """

    position_reached: str


@dataclass
class StopPositionBotActionFrame(StopActionFrame, BotActionFrame):
    """Stop holding the position.

    The bot will return to the position it had before the call.
    Position holding actions have an infinite lifetime, so unless the IM calls the Stop action the bot maintains
    the position indefinitely.  Alternatively PositionBotAction actions can be overwritten, since the modality policy
    is Override.
    """


@dataclass
class FinishedPositionBotActionFrame(FinishedActionFrame, BotActionFrame):
    """The bot shifted back to the original position before this action.

    This might be a neutral position or the position of any PositionBotAction overwritten
    by this action that now gains the "focus".
    """


# Presence User Action
@dataclass
class StartedPresenceUserActionFrame(StartedActionFrame, UserActionFrame, SystemFrame):
    """The interactive system detects the presence of a user in the system.

    TODO: We inherit from SystemFrame to circumvent the frame deletion issue with StartInterruptionFrame.
    This is a temporary fix only and needs to be reconsidered once the action concept is properly
    introduced.
    """


@dataclass
class FinishedPresenceUserActionFrame(FinishedActionFrame, UserActionFrame, SystemFrame):
    """The interactive system detects the user's absence.

    TODO: We inherit from SystemFrame to circumvent the frame deletion issue with StartInterruptionFrame.
    This is a temporary fix only and needs to be reconsidered once the action concept is properly
    introduced.
    """


# Attention User Action
@dataclass
class StartedAttentionUserActionFrame(StartedActionFrame, UserActionFrame):
    """The interactive system detects some level of engagement of the user.

    Args:
        attention_level (str): Attention level. Minimal supported values are "engaged" and "disengaged".
            Many systems support more granular levels, such as "engaged, partially" "disengaged, looking at phone
    """

    attention_level: str


@dataclass
class UpdatedAttentionUserActionFrame(UpdatedActionFrame, UserActionFrame):
    """The interactive system provides an update to the engagement level.

    Args:
        attention_level (str): Attention level. Minimal supported values are "engaged" and "disengaged".
            Many systems support more granular levels, such as "engaged, partially" "disengaged, looking at phone
    """

    attention_level: str


@dataclass
class FinishedAttentionUserActionFrame(FinishedActionFrame, UserActionFrame):
    """The system detects the user to be disengaged with the interactive system."""


# Motion Effect Camera Action
@dataclass
class StartMotionEffectCameraActionFrame(StartActionFrame, BotActionFrame):
    """Perform the described camera motion effect.

    Args:
        effect (str): Natural language description (NLD) of the effect. Availability of effects depends
            on the interactive system. Minimal NLD set:
            The following camera effects should be supported by every interactive system implementing this action:
            shake, jump cut in, jump cut out
    """

    effect: str


@dataclass
class StartedMotionEffectCameraActionFrame(StartedActionFrame, BotActionFrame):
    """Camera effect started."""


@dataclass
class StopMotionEffectCameraActionFrame(StopActionFrame, BotActionFrame):
    """Stop the camera effect.

    All effects have a limited lifetime and finish "on their own"
    (e.g., in an interactive avatar system a "shake" effect could be implemented by a 1 second camera motion).
    The IM can use this action to stop a camera effect before it would be naturally done.
    """


@dataclass
class FinishedMotionEffectCameraActionFrame(FinishedActionFrame, BotActionFrame):
    """Camera effect finished."""


# Shot Camera Action
@dataclass
class StartShotCameraActionFrame(StartActionFrame, BotActionFrame):
    """Start a new shot.

    Args:
        shot (str): Natural language description (NLD) of the shot. Availability of shots depends on
            the interactive system. Minimal NLD set:
            The following shots should be supported by every interactive system implementing this action:
                full, medium, close-up
        start_transition (str): NLD of the transition to the new shot. This should describe the movement.
            Minimal NLD set:
            The following shots should be supported by every interactive system implementing this action: cut, dolly
    """

    shot: str
    start_transition: str


@dataclass
class StartedShotCameraActionFrame(StartedActionFrame, BotActionFrame):
    """The camera shot started."""


@dataclass
class StopShotCameraActionFrame(StopActionFrame, BotActionFrame):
    """Stop the camera shot.

    The camera will return to the shot it had before this action was started.
    ShotCameraAction actions have an infinite lifetime, so unless the IM calls the Stop action the
    camera maintains the shot indefinitely.

    Args:
        stop_transition (str): NLD of the transition back to the previous shot (override modality).
            This should describe the movement.
            Minimal NLD set:
            The following shots should be supported by every interactive system implementing this action: cut, dolly
    """

    stop_transition: str


@dataclass
class FinishedShotCameraActionFrame(FinishedActionFrame, BotActionFrame):
    """The camera shot was stopped.

    The camera has returned to the shot it had before (either a neutral shot) or the shot specified by any
    overwritten ShotCameraAction actions.
    """