dify/api/services/workflow_runtime/workflow_generator.py

from __future__ import annotations

import contextvars
import logging
import threading
import uuid
from collections.abc import Generator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Literal, Protocol, overload, runtime_checkable

from flask import Flask, current_app
from pydantic import ValidationError
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker

import contexts
from configs import dify_config
from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
from core.app.apps.base_app_generator import BaseAppGenerator
from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
from core.app.apps.draft_variable_saver import DraftVariableSaver, DraftVariableSaverFactory, NoopDraftVariableSaver
from core.app.apps.exc import GenerateTaskStoppedError
from core.app.apps.workflow.active_workflow_tasks import active_workflow_task
from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager
from core.app.apps.workflow.generate_response_converter import WorkflowAppGenerateResponseConverter
from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, WorkflowAppGenerateEntity
from core.app.entities.task_entities import (
    WorkflowAppBlockingResponse,
    WorkflowAppPausedBlockingResponse,
    WorkflowAppStreamResponse,
)
from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig
from core.db.session_factory import session_factory
from core.helper.trace_id_helper import (
    extract_external_trace_id_from_args,
    extract_parent_trace_context_from_args,
    extract_trace_session_id_from_args,
)
from core.repositories import DifyCoreRepositoryFactory
from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepository
from core.trigger.constants import is_trigger_node_type
from extensions.ext_database import db
from extensions.tracing import create_trace_scope
from factories import file_factory
from graphon.engine.filter import ResponseStreamFilter
from graphon.engine.layer import Layer
from graphon.enums import NodeType
from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
from graphon.runtime import RuntimeState
from graphon.variable_loader import DUMMY_VARIABLE_LOADER, VariableLoader
from libs.flask_utils import preserve_flask_contexts
from models.account import Account
from models.enums import WorkflowRunTriggeredFrom
from models.model import App, AppMode, EndUser
from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom
from repositories.entities.workflow_execution import WorkflowExecutionIdentity
from runtime_contracts.builder.execution import (
    BuilderExecutionContext,
    BuilderExecutionPolicyError,
    BuilderExecutionRecorder,
    ConfirmedToolExecutionContext,
    JsonToolExecutionContext,
    scalar_inputs_digest,
)
from runtime_contracts.builder.native.agent_context import BuilderAgentExecutionContext
from runtime_contracts.builder.native.contracts import NativeTestContextRef
from services.app.file_input_defaults import grant_file_default_access
from services.dify_builder.native_testing.context_binding import NativeContextUnavailable
from services.dify_builder.native_testing.observation_adapters import NativeObservationError, NativeObservationGuard
from services.dify_builder.native_testing.runtime_bridge import NativeRuntimeBridge
from services.workflow_draft_variable_service import DraftVarLoader, WorkflowDraftVariableService
from services.workflow_runtime.node_factory import get_default_root_node_id
from services.workflow_runtime.workflow_pipeline import WorkflowAppGenerateTaskPipeline
from services.workflow_runtime.workflow_queue import WorkflowAppQueueManager
from services.workflow_runtime.workflow_runner import WorkflowAppRunner

if TYPE_CHECKING:
    from fields.workflow_run_fields import LoopNodeRunPayload
    from services.dify_builder.native_testing.host_execution import NativeHostLaunch
    from services.workflow_runtime.base_runner import WorkflowRunDriver

logger = logging.getLogger(__name__)


@runtime_checkable
class _DebugTraceArgs(Protocol):
    @property
    def trace_session_id(self) -> object: ...


def _extract_trace_session_id_from_debug_args(args: object) -> dict[str, str]:
    if isinstance(args, Mapping):
        return extract_trace_session_id_from_args(args)
    if isinstance(args, _DebugTraceArgs):
        try:
            trace_session_id = args.trace_session_id
        except AttributeError:
            return {}
        return extract_trace_session_id_from_args({"trace_session_id": trace_session_id})
    return {}


def _no_draft_variable_saver(
    app_id: str,
    node_id: str,
    node_type: NodeType,
    node_execution_id: str,
    enclosing_node_id: str | None = None,
) -> DraftVariableSaver:
    """``DraftVariableSaverFactory`` that never persists draft variables (existing no-op saver)."""
    _ = app_id, node_id, node_type, node_execution_id, enclosing_node_id
    return NoopDraftVariableSaver()


class BuilderExecutionAdmission(Protocol):
    """Transient service-owned admission, explicitly injected into native execution.

    Owns both Builder policy variants: ``admit_raw`` refuses raw metadata for
    the context it was built with, and ``__call__`` revalidates the reserved
    sidecar and claims the launch identity once before any native node exists.
    """

    def __call__(
        self,
        workflow: Workflow,
        app: App,
        entity: WorkflowAppGenerateEntity,
        root_node_id: str,
        resume: bool,
    ) -> BuilderExecutionRecorder: ...

    def admit_raw(self, workflow: Workflow, app: App) -> None: ...

    def on_refusal(self, error: BuilderExecutionPolicyError, *, preworker: bool = False) -> None: ...

    def on_worker_finished(
        self,
        *,
        context: BuilderExecutionContext | ConfirmedToolExecutionContext | JsonToolExecutionContext,
        native_run_id: str,
        task_id: str,
        recorder: BuilderExecutionRecorder | None,
        worker_thread: threading.Thread,
        exit_kind: Literal["returned", "policy_refused", "stopped", "failed"],
    ) -> None: ...


class WorkflowAppGenerator(BaseAppGenerator):
    def __init__(self, *, execution_driver: WorkflowRunDriver) -> None:
        self._execution_driver = execution_driver

    @staticmethod
    def _ensure_snippet_start_node_in_worker(*, session: Session, workflow: Workflow) -> Workflow:
        """Re-apply snippet virtual Start injection after worker reloads workflow from DB."""
        if workflow.kind_or_standard != "snippet":
            return workflow

        from models.snippet import CustomizedSnippet
        from services.snippet_generate_service import SnippetGenerateService

        snippet = session.scalar(
            select(CustomizedSnippet).where(
                CustomizedSnippet.id == workflow.app_id,
                CustomizedSnippet.tenant_id == workflow.tenant_id,
            )
        )
        if snippet is None:
            return workflow
        return SnippetGenerateService.ensure_start_node_for_worker(workflow, snippet)

    @overload
    def generate(
        self,
        *,
        execution_identity: WorkflowExecutionIdentity,
        app_model: App,
        workflow: Workflow,
        user: Account | EndUser,
        args: Mapping[str, Any],
        invoke_from: InvokeFrom,
        streaming: Literal[True],
        call_depth: int,
        workflow_run_id: str | uuid.UUID | None = None,
        triggered_from: WorkflowRunTriggeredFrom | None = None,
        root_node_id: str | None = None,
        graph_engine_layers: Sequence[Layer] = (),
        allow_human_input: bool = True,
        pause_state_config: PauseStateLayerConfig | None = None,
        builder_execution: BuilderExecutionContext
        | ConfirmedToolExecutionContext
        | JsonToolExecutionContext
        | None = None,
        builder_execution_admit: BuilderExecutionAdmission | None = None,
        builder_agent_execution: BuilderAgentExecutionContext | None = None,
        native_test_required: bool = False,
        native_test_ref: NativeTestContextRef | None = None,
        native_runtime: NativeRuntimeBridge | None = None,
        native_host: NativeHostLaunch | None = None,
        builder_mocked_node_ids: frozenset[str] = frozenset(),
        builder_expected_revision: str | None = None,
        builder_tool_credential_bindings: Mapping[str, str] | None = None,
    ) -> Generator[Mapping[str, Any] | str]: ...

    @overload
    def generate(
        self,
        *,
        execution_identity: WorkflowExecutionIdentity,
        app_model: App,
        workflow: Workflow,
        user: Account | EndUser,
        args: Mapping[str, Any],
        invoke_from: InvokeFrom,
        streaming: Literal[False],
        call_depth: int,
        workflow_run_id: str | uuid.UUID | None = None,
        triggered_from: WorkflowRunTriggeredFrom | None = None,
        root_node_id: str | None = None,
        graph_engine_layers: Sequence[Layer] = (),
        allow_human_input: bool = True,
        pause_state_config: PauseStateLayerConfig | None = None,
        builder_execution: BuilderExecutionContext
        | ConfirmedToolExecutionContext
        | JsonToolExecutionContext
        | None = None,
        builder_execution_admit: BuilderExecutionAdmission | None = None,
        builder_agent_execution: BuilderAgentExecutionContext | None = None,
        native_test_required: bool = False,
        native_test_ref: NativeTestContextRef | None = None,
        native_runtime: NativeRuntimeBridge | None = None,
        native_host: NativeHostLaunch | None = None,
        builder_mocked_node_ids: frozenset[str] = frozenset(),
        builder_expected_revision: str | None = None,
        builder_tool_credential_bindings: Mapping[str, str] | None = None,
    ) -> Mapping[str, Any]: ...

    @overload
    def generate(
        self,
        *,
        execution_identity: WorkflowExecutionIdentity,
        app_model: App,
        workflow: Workflow,
        user: Account | EndUser,
        args: Mapping[str, Any],
        invoke_from: InvokeFrom,
        streaming: bool,
        call_depth: int,
        workflow_run_id: str | uuid.UUID | None = None,
        triggered_from: WorkflowRunTriggeredFrom | None = None,
        root_node_id: str | None = None,
        graph_engine_layers: Sequence[Layer] = (),
        allow_human_input: bool = True,
        pause_state_config: PauseStateLayerConfig | None = None,
        builder_execution: BuilderExecutionContext
        | ConfirmedToolExecutionContext
        | JsonToolExecutionContext
        | None = None,
        builder_execution_admit: BuilderExecutionAdmission | None = None,
        builder_agent_execution: BuilderAgentExecutionContext | None = None,
        native_test_required: bool = False,
        native_test_ref: NativeTestContextRef | None = None,
        native_runtime: NativeRuntimeBridge | None = None,
        native_host: NativeHostLaunch | None = None,
        builder_mocked_node_ids: frozenset[str] = frozenset(),
        builder_expected_revision: str | None = None,
        builder_tool_credential_bindings: Mapping[str, str] | None = None,
    ) -> Mapping[str, Any] | Generator[Mapping[str, Any] | str]: ...

    def generate(
        self,
        *,
        execution_identity: WorkflowExecutionIdentity,
        app_model: App,
        workflow: Workflow,
        user: Account | EndUser,
        args: Mapping[str, Any],
        invoke_from: InvokeFrom,
        streaming: bool = True,
        call_depth: int = 0,
        workflow_run_id: str | uuid.UUID | None = None,
        triggered_from: WorkflowRunTriggeredFrom | None = None,
        root_node_id: str | None = None,
        graph_engine_layers: Sequence[Layer] = (),
        allow_human_input: bool = True,
        pause_state_config: PauseStateLayerConfig | None = None,
        builder_execution: BuilderExecutionContext
        | ConfirmedToolExecutionContext
        | JsonToolExecutionContext
        | None = None,
        builder_execution_admit: BuilderExecutionAdmission | None = None,
        builder_agent_execution: BuilderAgentExecutionContext | None = None,
        native_test_required: bool = False,
        native_test_ref: NativeTestContextRef | None = None,
        native_runtime: NativeRuntimeBridge | None = None,
        native_host: NativeHostLaunch | None = None,
        builder_mocked_node_ids: frozenset[str] = frozenset(),
        builder_expected_revision: str | None = None,
        builder_tool_credential_bindings: Mapping[str, str] | None = None,
    ) -> Mapping[str, Any] | Generator[Mapping[str, Any] | str]:
        from services.dify_builder.native_launch_observation import begin_preparation
        from services.dify_builder.native_testing.generation_wiring import validate_native_host

        begin_preparation()

        if builder_agent_execution is not None:
            if (
                builder_execution is not None
                or native_test_required
                or native_test_ref is not None
                or native_runtime is not None
                or native_host is not None
                or invoke_from != InvokeFrom.DEBUGGER
                or not isinstance(user, Account)
                or app_model.mode != AppMode.WORKFLOW
                or call_depth != 0
                or (workflow_run_id is not None and str(workflow_run_id) != builder_agent_execution.native_run_id)
            ):
                raise BuilderExecutionPolicyError("unsupported_execution_transport")
            workflow_run_id = builder_agent_execution.native_run_id
        validate_native_host(
            required=native_test_required or native_test_ref is not None,
            ref=native_test_ref,
            runtime=native_runtime,
            host=native_host,
        )
        native_context = None
        native_inputs = None
        if native_test_required or native_test_ref is not None:
            if native_test_ref is None:
                raise NativeContextUnavailable("native_context_missing")
            if native_runtime is None:
                raise NativeContextUnavailable("native_runtime_unavailable")
            native_context = native_runtime.resolve(native_test_ref)
            if (
                builder_execution is not None
                or (app_model.tenant_id, app_model.id, workflow.id, user.id, str(workflow_run_id), call_depth)
                != (
                    native_context.tenant_id,
                    native_context.app_id,
                    native_context.workflow_id,
                    native_context.actor_id,
                    native_context.native_run_id,
                    native_context.call_depth,
                )
                or workflow.graph_dict != native_context.graph
            ):
                raise NativeContextUnavailable("native_context_mismatch")
            assert native_host is not None
            guard = native_host.bind_observation()
            with native_host.observation_scope():
                if (
                    args.get("files")
                    or pause_state_config is not None
                    or graph_engine_layers
                    or not streaming
                    or workflow.features_dict.get("text_to_speech", {}).get("enabled")
                    or any(
                        args.get(key) is not None
                        for key in ("trace_session_id", "external_trace_id", "parent_trace_context")
                    )
                ):
                    guard.fail()
                native_inputs = native_runtime.prepare_inputs(native_context)
        if (
            builder_mocked_node_ids
            or builder_expected_revision is not None
            or builder_tool_credential_bindings is not None
        ) and (builder_execution is not None or native_context is not None or invoke_from != InvokeFrom.DEBUGGER):
            raise BuilderExecutionPolicyError("unsupported_execution_policy")
        try:
            if builder_execution is not None and (
                builder_execution_admit is None
                or args.get("files")
                or call_depth != 0
                or not isinstance(user, Account)
                or invoke_from != InvokeFrom.DEBUGGER
                or pause_state_config is not None
                or graph_engine_layers
            ):
                raise BuilderExecutionPolicyError("unsupported_execution_transport")
            if builder_execution is not None:
                assert builder_execution_admit is not None
                builder_execution_admit.admit_raw(workflow, app_model)
                scalar_inputs_digest(args.get("inputs", {}))
        except BuilderExecutionPolicyError as error:
            if builder_execution_admit is not None:
                builder_execution_admit.on_refusal(error, preworker=True)
            raise
        with self._bind_file_access_scope(tenant_id=app_model.tenant_id, user=user, invoke_from=invoke_from):
            files: Sequence[Mapping[str, Any]] = args.get("files") or []

            # parse files
            # TODO(QuantumGhost): Move file parsing logic to the API controller layer
            # for better separation of concerns.
            #
            # For implementation reference, see the `_parse_file` function and
            # `DraftWorkflowNodeRunApi` class which handle this properly.
            file_extra_config = FileUploadConfigManager.convert(workflow.features_dict, is_vision=False)
            system_files = (
                []
                if builder_execution is not None or native_context is not None
                else file_factory.build_from_mappings(
                    mappings=files,
                    tenant_id=app_model.tenant_id,
                    config=file_extra_config,
                    strict_type_validation=invoke_from == InvokeFrom.SERVICE_API,
                    access_controller=self._file_access_controller,
                )
            )

            # convert to app config
            app_config = WorkflowAppConfigManager.get_app_config(
                app_model=app_model,
                workflow=workflow,
            )

            inputs: Mapping[str, Any] = native_inputs.inputs if native_inputs is not None else args["inputs"]

            extras = {
                **extract_external_trace_id_from_args(args),
                **extract_parent_trace_context_from_args(args),
                **extract_trace_session_id_from_args(args),
            }
            trace_scope = None
            if native_context is None:
                trace_scope = create_trace_scope(
                    tenant_id=app_model.tenant_id,
                    app_id=app_model.id,
                    app_tracing=app_model.tracing,
                    user_id=user.id if isinstance(user, Account) else user.session_id,
                    extras=extras,
                    external_tracing_disabled=builder_execution is not None,
                )
            workflow_run_id = str(workflow_run_id or uuid.uuid4())
            root_node_id = root_node_id or get_default_root_node_id(workflow.graph_dict)
            root_node_config = workflow.get_node_config_by_id(root_node_id)
            root_node_type = workflow.get_node_type_from_node_config(root_node_config)
            if builder_execution is not None and root_node_type != "start":
                error = BuilderExecutionPolicyError("unsupported_execution_root")
                if builder_execution_admit is not None:
                    builder_execution_admit.on_refusal(error, preworker=True)
                raise error
            # Trigger inputs are already adapted event data, not Start form fields.
            if native_context is None and not is_trigger_node_type(root_node_type):
                inputs = self._prepare_user_inputs(
                    user_inputs=inputs,
                    variables=app_config.variables,
                    tenant_id=app_model.tenant_id,
                    strict_type_validation=invoke_from == InvokeFrom.SERVICE_API,
                )
            # init application generate entity
            application_generate_entity = WorkflowAppGenerateEntity(
                task_id=str(uuid.uuid4()),
                app_config=app_config,
                file_upload_config=file_extra_config,
                inputs=inputs,
                files=list(native_inputs.files if native_inputs is not None else system_files),
                user_id=execution_identity.user_id,
                user_from=execution_identity.user_from,
                stream=streaming,
                invoke_from=invoke_from,
                allow_human_input=allow_human_input,
                call_depth=call_depth,
                builder_execution=builder_execution,
                builder_agent_execution=builder_agent_execution,
                native_test_required=native_test_required,
                native_test_ref=native_test_ref,
                builder_mocked_node_ids=builder_mocked_node_ids,
                builder_expected_revision=builder_expected_revision,
                builder_tool_credential_bindings=builder_tool_credential_bindings,
                trace_scope=trace_scope,
                workflow_execution_id=workflow_run_id,
                extras=extras,
            )

            if builder_execution is not None:
                if scalar_inputs_digest(dict(inputs)) != builder_execution.effective_inputs_digest:
                    error = BuilderExecutionPolicyError("effective_input_binding_mismatch")
                    if builder_execution_admit is not None:
                        builder_execution_admit.on_refusal(error, preworker=True)
                    raise error

            contexts.plugin_tool_providers.set({})
            contexts.plugin_tool_providers_lock.set(threading.Lock())

            # Create repositories
            #
            # Create session factory
            session_factory = sessionmaker(bind=db.engine, expire_on_commit=False)
            # Create workflow execution(aka workflow run) repository
            if triggered_from is not None:
                # Use explicitly provided triggered_from (for async triggers)
                workflow_triggered_from = triggered_from
            elif invoke_from == InvokeFrom.DEBUGGER:
                workflow_triggered_from = WorkflowRunTriggeredFrom.DEBUGGING
            else:
                workflow_triggered_from = WorkflowRunTriggeredFrom.APP_RUN
            workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
                session_factory=session_factory,
                tenant_id=app_model.tenant_id,
                user=user,
                app_id=application_generate_entity.app_config.app_id,
                triggered_from=workflow_triggered_from,
            )
            # Create workflow node execution repository
            workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
                session_factory=session_factory,
                tenant_id=app_model.tenant_id,
                user=user,
                app_id=application_generate_entity.app_config.app_id,
                triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
            )

            return self._generate(
                app_model=app_model,
                workflow=workflow,
                user=user,
                application_generate_entity=application_generate_entity,
                invoke_from=invoke_from,
                workflow_execution_repository=workflow_execution_repository,
                workflow_node_execution_repository=workflow_node_execution_repository,
                streaming=streaming,
                root_node_id=root_node_id,
                graph_engine_layers=graph_engine_layers,
                pause_state_config=pause_state_config,
                builder_execution_admit=builder_execution_admit,
                native_runtime=native_runtime,
                native_host=native_host,
            )

    def resume(
        self,
        *,
        app_model: App,
        workflow: Workflow,
        user: Account | EndUser,
        application_generate_entity: WorkflowAppGenerateEntity,
        graph_runtime_state: RuntimeState,
        workflow_execution_repository: WorkflowExecutionRepository,
        workflow_node_execution_repository: WorkflowNodeExecutionRepository,
        graph_engine_layers: Sequence[Layer] = (),
        pause_state_config: PauseStateLayerConfig | None = None,
        variable_loader: VariableLoader = DUMMY_VARIABLE_LOADER,
        response_stream_filter: ResponseStreamFilter | None = None,
        native_runtime: NativeRuntimeBridge | None = None,
        native_host: NativeHostLaunch | None = None,
    ) -> Mapping[str, Any] | Generator[str | Mapping[str, Any]]:
        """
        Resume a paused workflow execution using the persisted runtime state.

        ``trace_scope`` is transient and excluded from generate-entity serialization,
        so resumed executions create it again before persistence layers receive the entity.
        """
        if application_generate_entity.builder_agent_execution is not None and (
            application_generate_entity.builder_execution is not None
            or application_generate_entity.native_test_required
            or application_generate_entity.native_test_ref is not None
            or native_runtime is not None
            or native_host is not None
            or application_generate_entity.invoke_from != InvokeFrom.DEBUGGER
            or not isinstance(user, Account)
            or app_model.mode != AppMode.WORKFLOW
            or application_generate_entity.call_depth != 0
            or application_generate_entity.workflow_execution_id
            != application_generate_entity.builder_agent_execution.native_run_id
        ):
            raise BuilderExecutionPolicyError("unsupported_execution_transport")
        if application_generate_entity.native_test_required or application_generate_entity.native_test_ref is not None:
            if application_generate_entity.native_test_ref is None:
                raise NativeContextUnavailable("native_context_missing")
            from services.dify_builder.native_testing.generation_wiring import validate_native_host

            validate_native_host(
                required=True, ref=application_generate_entity.native_test_ref, runtime=native_runtime, host=native_host
            )
            assert native_host is not None
            native_host.require_resume_bindings(
                entity=application_generate_entity,
                runtime_state=graph_runtime_state,
                response_filter=response_stream_filter,
            )
        if application_generate_entity.builder_execution is not None:
            raise BuilderExecutionPolicyError("unsupported_execution_transport")
        if native_host is None and application_generate_entity.trace_scope is None:
            application_generate_entity = application_generate_entity.model_copy(
                update={
                    "trace_scope": create_trace_scope(
                        tenant_id=app_model.tenant_id,
                        app_id=app_model.id,
                        app_tracing=app_model.tracing,
                        user_id=user.id if isinstance(user, Account) else user.session_id,
                        extras=application_generate_entity.extras,
                    )
                }
            )

        return self._generate(
            app_model=app_model,
            workflow=workflow,
            user=user,
            application_generate_entity=application_generate_entity,
            invoke_from=application_generate_entity.invoke_from,
            workflow_execution_repository=workflow_execution_repository,
            workflow_node_execution_repository=workflow_node_execution_repository,
            streaming=application_generate_entity.stream,
            variable_loader=variable_loader,
            graph_engine_layers=graph_engine_layers,
            graph_runtime_state=graph_runtime_state,
            pause_state_config=pause_state_config,
            response_stream_filter=response_stream_filter,
            native_runtime=native_runtime,
            native_host=native_host,
        )

    def _generate(
        self,
        *,
        app_model: App,
        workflow: Workflow,
        user: Account | EndUser,
        application_generate_entity: WorkflowAppGenerateEntity,
        invoke_from: InvokeFrom,
        workflow_execution_repository: WorkflowExecutionRepository,
        workflow_node_execution_repository: WorkflowNodeExecutionRepository,
        streaming: bool = True,
        variable_loader: VariableLoader = DUMMY_VARIABLE_LOADER,
        root_node_id: str | None = None,
        graph_engine_layers: Sequence[Layer] = (),
        graph_runtime_state: RuntimeState | None = None,
        pause_state_config: PauseStateLayerConfig | None = None,
        response_stream_filter: ResponseStreamFilter | None = None,
        builder_execution_admit: BuilderExecutionAdmission | None = None,
        native_runtime: NativeRuntimeBridge | None = None,
        native_host: NativeHostLaunch | None = None,
    ) -> Mapping[str, Any] | Generator[str | Mapping[str, Any]]:
        """
        Generate App response.

        :param app_model: App
        :param workflow: Workflow
        :param user: account or end user
        :param application_generate_entity: application generate entity
        :param invoke_from: invoke from source
        :param workflow_execution_repository: repository for workflow execution
        :param workflow_node_execution_repository: repository for workflow node execution
        :param streaming: is stream
        """
        from services.dify_builder.native_testing.generation_wiring import bind_native_generation, validate_native_host

        validate_native_host(
            required=application_generate_entity.native_test_required
            or application_generate_entity.native_test_ref is not None,
            ref=application_generate_entity.native_test_ref,
            runtime=native_runtime,
            host=native_host,
        )
        if native_host is not None and (graph_engine_layers or application_generate_entity.trace_scope is not None):
            raise NativeContextUnavailable("native_observation_unqualified")
        if native_host is not None:
            if not streaming or pause_state_config is not None:
                raise NativeContextUnavailable("native_resume_unavailable")
            if graph_runtime_state is not None:
                native_host.require_resume_bindings(
                    entity=application_generate_entity,
                    runtime_state=graph_runtime_state,
                    response_filter=response_stream_filter,
                )
        native_observation = None
        if native_host is not None:
            from services.dify_builder.native_testing.generation_wiring import preflight_native_observation

            native_observation = native_host.bind_observation()
            preflight_native_observation(host=native_host, entity=application_generate_entity, workflow=workflow)
        with self._bind_file_access_scope(
            tenant_id=application_generate_entity.app_config.tenant_id,
            user=user,
            invoke_from=invoke_from,
        ):
            if native_host is None:
                grant_file_default_access(application_generate_entity.app_config.variables)
            # init queue manager
            queue_manager = WorkflowAppQueueManager(
                task_id=application_generate_entity.task_id,
                user_id=application_generate_entity.user_id,
                invoke_from=application_generate_entity.invoke_from,
                app_mode=app_model.mode,
                native_observation=native_observation,
                native_host=native_host,
            )

            resolved_response_stream_filter = response_stream_filter or ResponseStreamFilter()
            if native_host is not None:
                bind_native_generation(
                    host=native_host,
                    entity=application_generate_entity,
                    queue=queue_manager,
                    response_filter=resolved_response_stream_filter,
                    workflow_repository=workflow_execution_repository,
                    node_repository=workflow_node_execution_repository,
                )

            # new thread with request context and contextvars
            context = contextvars.copy_context()

            # release database connection, because the following new thread operations may take a long time
            db.session.close()

            worker_thread = threading.Thread(
                target=self._generate_worker,
                kwargs={
                    "flask_app": current_app._get_current_object(),  # type: ignore
                    "application_generate_entity": application_generate_entity,
                    "queue_manager": queue_manager,
                    "context": context,
                    "variable_loader": variable_loader,
                    "root_node_id": root_node_id,
                    "workflow_execution_repository": workflow_execution_repository,
                    "workflow_node_execution_repository": workflow_node_execution_repository,
                    "graph_engine_layers": graph_engine_layers,
                    "pause_state_config": pause_state_config,
                    "graph_runtime_state": graph_runtime_state,
                    "response_stream_filter": resolved_response_stream_filter,
                    "builder_execution_admit": builder_execution_admit,
                    "native_runtime": native_runtime,
                    "native_host": native_host,
                },
            )

            from services.dify_builder.native_launch_observation import mark_dispatched

            mark_dispatched()
            if native_host is None:
                worker_thread.start()
            else:
                native_host.start_worker(worker=worker_thread)

            draft_var_saver_factory: DraftVariableSaverFactory
            if native_host is not None or isinstance(
                application_generate_entity.builder_execution, ConfirmedToolExecutionContext
            ):
                # The authorized native Run/node rows are the only retained copy of a
                # confirmed Tool's content: no mutable workspace draft-variable copy.
                draft_var_saver_factory = _no_draft_variable_saver
            else:
                draft_var_saver_factory = self._get_draft_var_saver_factory(
                    invoke_from,
                    user,
                    tenant_id=app_model.tenant_id,
                )

            try:
                response = self._handle_response(
                    application_generate_entity=application_generate_entity,
                    workflow=workflow,
                    queue_manager=queue_manager,
                    user=user,
                    draft_var_saver_factory=draft_var_saver_factory,
                    stream=streaming,
                    native_observation=native_observation,
                    native_host=native_host,
                )
                converted_response = WorkflowAppGenerateResponseConverter.convert(
                    response=response,
                    invoke_from=invoke_from,
                )
            except BaseException:
                if native_host is None:
                    self._join_worker_thread(worker_thread)
                else:
                    native_host.request_close(reason="failed")
                raise

            if native_host is not None:
                if not isinstance(converted_response, Generator):
                    native_host.request_close(reason="failed")
                    raise NativeContextUnavailable("native_stream_unavailable")
                return converted_response
            if isinstance(converted_response, Generator):
                return self._wrap_stream_with_worker_thread_join(converted_response, worker_thread)

            self._join_worker_thread(worker_thread)
            return converted_response

    def single_iteration_generate(
        self,
        app_model: App,
        workflow: Workflow,
        node_id: str,
        user: Account | EndUser,
        args: Mapping[str, Any],
        streaming: bool = True,
        *,
        execution_identity: WorkflowExecutionIdentity,
        session: Session,
    ) -> Mapping[str, Any] | Generator[str | Mapping[str, Any]]:
        """
        Generate App response.

        :param app_model: App
        :param workflow: Workflow
        :param node_id: the node id
        :param user: account or end user
        :param args: request args
        :param streaming: is streamed
        :param session: database session supplied by the caller
        """
        if not node_id:
            raise ValueError("node_id is required")

        if args.get("inputs") is None:
            raise ValueError("inputs is required")

        # convert to app config
        app_config = WorkflowAppConfigManager.get_app_config(app_model=app_model, workflow=workflow)

        # init application generate entity
        application_generate_entity = WorkflowAppGenerateEntity(
            task_id=str(uuid.uuid4()),
            app_config=app_config,
            inputs={},
            files=[],
            user_id=execution_identity.user_id,
            user_from=execution_identity.user_from,
            stream=streaming,
            invoke_from=InvokeFrom.DEBUGGER,
            extras={
                "auto_generate_conversation_name": False,
                **_extract_trace_session_id_from_debug_args(args),
            },
            single_iteration_run=WorkflowAppGenerateEntity.SingleIterationRunEntity(
                node_id=node_id, inputs=args["inputs"]
            ),
            workflow_execution_id=str(uuid.uuid4()),
        )
        contexts.plugin_tool_providers.set({})
        contexts.plugin_tool_providers_lock.set(threading.Lock())

        # Create repositories
        #
        # Create session factory
        session_factory = sessionmaker(bind=db.engine, expire_on_commit=False)
        # Create workflow execution(aka workflow run) repository
        workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
            session_factory=session_factory,
            tenant_id=app_model.tenant_id,
            user=user,
            app_id=application_generate_entity.app_config.app_id,
            triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,
        )
        # Create workflow node execution repository
        workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
            session_factory=session_factory,
            tenant_id=app_model.tenant_id,
            user=user,
            app_id=application_generate_entity.app_config.app_id,
            triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,
        )
        draft_var_srv = WorkflowDraftVariableService(session)
        draft_var_srv.prefill_conversation_variable_default_values(workflow, user_id=user.id)
        var_loader = DraftVarLoader(
            engine=db.engine,
            app_id=application_generate_entity.app_config.app_id,
            tenant_id=application_generate_entity.app_config.tenant_id,
            user_id=user.id,
        )

        return self._generate(
            app_model=app_model,
            workflow=workflow,
            user=user,
            invoke_from=InvokeFrom.DEBUGGER,
            application_generate_entity=application_generate_entity,
            workflow_execution_repository=workflow_execution_repository,
            workflow_node_execution_repository=workflow_node_execution_repository,
            streaming=streaming,
            variable_loader=var_loader,
            pause_state_config=None,
        )

    def single_loop_generate(
        self,
        app_model: App,
        workflow: Workflow,
        node_id: str,
        user: Account | EndUser,
        args: LoopNodeRunPayload,
        streaming: bool = True,
        *,
        execution_identity: WorkflowExecutionIdentity,
        session: Session,
    ) -> Mapping[str, Any] | Generator[str | Mapping[str, Any]]:
        """
        Generate App response.

        :param app_model: App
        :param workflow: Workflow
        :param node_id: the node id
        :param user: account or end user
        :param args: request args
        :param streaming: is streamed
        :param session: database session supplied by the caller
        """
        if not node_id:
            raise ValueError("node_id is required")

        if args.inputs is None:
            raise ValueError("inputs is required")

        # convert to app config
        app_config = WorkflowAppConfigManager.get_app_config(app_model=app_model, workflow=workflow)

        # init application generate entity
        application_generate_entity = WorkflowAppGenerateEntity(
            task_id=str(uuid.uuid4()),
            app_config=app_config,
            inputs={},
            files=[],
            user_id=execution_identity.user_id,
            user_from=execution_identity.user_from,
            stream=streaming,
            invoke_from=InvokeFrom.DEBUGGER,
            extras={
                "auto_generate_conversation_name": False,
                **_extract_trace_session_id_from_debug_args(args),
            },
            single_loop_run=WorkflowAppGenerateEntity.SingleLoopRunEntity(node_id=node_id, inputs=args.inputs or {}),
            workflow_execution_id=str(uuid.uuid4()),
        )
        contexts.plugin_tool_providers.set({})
        contexts.plugin_tool_providers_lock.set(threading.Lock())

        # Create repositories
        #
        # Create session factory
        session_factory = sessionmaker(bind=db.engine, expire_on_commit=False)
        # Create workflow execution(aka workflow run) repository
        workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
            session_factory=session_factory,
            tenant_id=app_model.tenant_id,
            user=user,
            app_id=application_generate_entity.app_config.app_id,
            triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,
        )
        # Create workflow node execution repository
        workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
            session_factory=session_factory,
            tenant_id=app_model.tenant_id,
            user=user,
            app_id=application_generate_entity.app_config.app_id,
            triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,
        )
        draft_var_srv = WorkflowDraftVariableService(session)
        draft_var_srv.prefill_conversation_variable_default_values(workflow, user_id=user.id)
        var_loader = DraftVarLoader(
            engine=db.engine,
            app_id=application_generate_entity.app_config.app_id,
            tenant_id=application_generate_entity.app_config.tenant_id,
            user_id=user.id,
        )
        return self._generate(
            app_model=app_model,
            workflow=workflow,
            user=user,
            invoke_from=InvokeFrom.DEBUGGER,
            application_generate_entity=application_generate_entity,
            workflow_execution_repository=workflow_execution_repository,
            workflow_node_execution_repository=workflow_node_execution_repository,
            streaming=streaming,
            variable_loader=var_loader,
            pause_state_config=None,
        )

    def _generate_worker(
        self,
        flask_app: Flask,
        application_generate_entity: WorkflowAppGenerateEntity,
        queue_manager: AppQueueManager,
        context: contextvars.Context,
        variable_loader: VariableLoader,
        workflow_execution_repository: WorkflowExecutionRepository,
        workflow_node_execution_repository: WorkflowNodeExecutionRepository,
        root_node_id: str | None = None,
        graph_engine_layers: Sequence[Layer] = (),
        graph_runtime_state: RuntimeState | None = None,
        response_stream_filter: ResponseStreamFilter | None = None,
        builder_execution_admit: BuilderExecutionAdmission | None = None,
        native_runtime: NativeRuntimeBridge | None = None,
        native_host: NativeHostLaunch | None = None,
        pause_state_config: PauseStateLayerConfig | None = None,
    ) -> None:
        """
        Generate worker in a new thread.
        :param flask_app: Flask app
        :param application_generate_entity: application generate entity
        :param queue_manager: queue manager
        :param workflow_thread_pool_id: workflow thread pool id
        :return:
        """
        execution_recorder = None
        exit_kind: Literal["returned", "policy_refused", "stopped", "failed"] = "failed"
        try:
            from services.dify_builder.native_testing.context_runtime import native_flask_worker_scope

            worker_scope = (
                native_flask_worker_scope(host=native_host, flask_app=flask_app, context_vars=context)
                if native_host is not None
                else preserve_flask_contexts(flask_app, context_vars=context)
            )
            with worker_scope:
                try:
                    native_context = None
                    if (
                        application_generate_entity.native_test_required
                        or application_generate_entity.native_test_ref is not None
                    ):
                        if application_generate_entity.native_test_ref is None:
                            raise NativeContextUnavailable("native_context_missing")
                        if native_runtime is None:
                            raise NativeContextUnavailable("native_runtime_unavailable")
                        native_context = native_runtime.resolve(application_generate_entity.native_test_ref)
                        if (
                            application_generate_entity.app_config.tenant_id,
                            application_generate_entity.app_config.app_id,
                            application_generate_entity.app_config.workflow_id,
                            application_generate_entity.user_id,
                            application_generate_entity.workflow_execution_id,
                        ) != (
                            native_context.tenant_id,
                            native_context.app_id,
                            native_context.workflow_id,
                            native_context.actor_id,
                            native_context.native_run_id,
                        ):
                            raise NativeContextUnavailable("native_context_mismatch")
                        from services.dify_builder.native_testing.generation_wiring import validate_native_host

                        validate_native_host(
                            required=True,
                            ref=application_generate_entity.native_test_ref,
                            runtime=native_runtime,
                            host=native_host,
                        )
                        assert native_host is not None
                        if native_host.generation_bindings.entity is not application_generate_entity:
                            raise NativeContextUnavailable("native_host_identity_mismatch")
                        prepared = native_runtime.prepare_inputs(native_context)
                        if graph_runtime_state is None:
                            application_generate_entity.inputs = prepared.inputs
                            application_generate_entity.files = list(prepared.files)
                        else:
                            native_host.require_resume_bindings(
                                entity=application_generate_entity,
                                runtime_state=graph_runtime_state,
                                response_filter=response_stream_filter,
                            )
                        application_generate_entity.call_depth = native_context.call_depth

                    with session_factory.create_session() as session:
                        workflow = session.scalar(
                            select(Workflow).where(
                                Workflow.tenant_id == application_generate_entity.app_config.tenant_id,
                                Workflow.app_id == application_generate_entity.app_config.app_id,
                                Workflow.id == application_generate_entity.app_config.workflow_id,
                            )
                        )
                        if workflow is None:
                            if application_generate_entity.builder_execution is not None:
                                raise BuilderExecutionPolicyError("execution_owner_mismatch")
                            raise ValueError("Workflow not found")

                        if application_generate_entity.builder_execution is not None:
                            if builder_execution_admit is None:
                                raise BuilderExecutionPolicyError("missing_execution_admission")
                            app = session.scalar(
                                select(App).where(
                                    App.id == application_generate_entity.app_config.app_id,
                                    App.tenant_id == application_generate_entity.app_config.tenant_id,
                                )
                            )
                            if app is None:
                                raise BuilderExecutionPolicyError("execution_owner_mismatch")
                            if graph_engine_layers:
                                raise BuilderExecutionPolicyError("unsupported_execution_transport")
                            if isinstance(application_generate_entity.builder_execution, JsonToolExecutionContext):
                                # The owned worker read must release its SQL transaction
                                # before the issuer revalidates daemon metadata.
                                session.expunge_all()
                                session.close()
                            execution_recorder = builder_execution_admit(
                                workflow,
                                app,
                                application_generate_entity,
                                root_node_id or get_default_root_node_id(workflow.graph_dict),
                                graph_runtime_state is not None,
                            )
                            if execution_recorder is None or not execution_recorder.healthy:
                                raise BuilderExecutionPolicyError("missing_execution_recorder")

                        if native_context is not None:
                            if workflow.graph_dict != native_context.graph:
                                raise NativeContextUnavailable("native_context_mismatch")
                        else:
                            workflow = self._ensure_snippet_start_node_in_worker(session=session, workflow=workflow)

                        if graph_runtime_state is not None and native_context is None:
                            self._restore_workflow_run_graph(
                                session=session,
                                workflow=workflow,
                                workflow_run_id=application_generate_entity.workflow_execution_id,
                            )

                        user_from = application_generate_entity.user_from
                        if user_from == UserFrom.END_USER:
                            end_user = session.scalar(
                                select(EndUser).where(
                                    EndUser.tenant_id == application_generate_entity.app_config.tenant_id,
                                    EndUser.id == application_generate_entity.user_id,
                                )
                            )
                            system_user_id = end_user.session_id if end_user else ""
                        else:
                            system_user_id = application_generate_entity.user_id

                    if application_generate_entity.builder_expected_revision is not None:
                        from services.dify_builder.revision import execution_revision

                        if execution_revision(workflow) != application_generate_entity.builder_expected_revision:
                            raise BuilderExecutionPolicyError("stale_execution_revision")

                    runner = WorkflowAppRunner(
                        application_generate_entity=application_generate_entity,
                        execution_driver=self._execution_driver,
                        queue_manager=queue_manager,
                        variable_loader=variable_loader,
                        workflow=workflow,
                        system_user_id=system_user_id,
                        native_runtime=native_runtime,
                        native_host=native_host,
                        root_node_id=root_node_id,
                        graph_engine_layers=graph_engine_layers,
                        graph_runtime_state=graph_runtime_state,
                        response_stream_filter=response_stream_filter,
                        execution_recorder=execution_recorder,
                    )

                    with active_workflow_task(application_generate_entity.task_id):
                        self._execution_driver(
                            runner,
                            pause_state_config,
                            workflow_execution_repository,
                            workflow_node_execution_repository,
                        )
                    exit_kind = "returned"
                except GenerateTaskStoppedError:
                    exit_kind = "stopped"
                    logger.warning("Task stopped", exc_info=native_host is None)
                    pass
                except InvokeAuthorizationError:
                    queue_manager.publish_error(
                        InvokeAuthorizationError("Incorrect API key provided"), PublishFrom.APPLICATION_MANAGER
                    )
                except NativeContextUnavailable as e:
                    exit_kind = "policy_refused"
                    queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER)
                except ValidationError as e:
                    if native_host is not None:
                        queue_manager.publish_error(NativeObservationError(), PublishFrom.APPLICATION_MANAGER)
                    else:
                        logger.exception("Validation Error when generating")
                        queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER)
                except BuilderExecutionPolicyError as e:
                    exit_kind = "policy_refused"
                    if builder_execution_admit is not None:
                        builder_execution_admit.on_refusal(e)
                    queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER)
                except ValueError as e:
                    if native_host is not None:
                        queue_manager.publish_error(NativeObservationError(), PublishFrom.APPLICATION_MANAGER)
                    else:
                        if dify_config.DEBUG:
                            logger.exception("Error when generating")
                        queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER)
                except Exception as e:
                    if native_host is not None:
                        queue_manager.publish_error(NativeObservationError(), PublishFrom.APPLICATION_MANAGER)
                    else:
                        logger.exception("Unknown Error when generating")
                        queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER)
        except BaseException:
            exit_kind = "failed"
            if native_host is None:
                raise
            # Scope entry itself can fail before the inner publisher exists.
            # A failed guard must not project another payload. Close the listener
            # segment only; physical cleanup remains owned by the host/reader.
            from core.app.apps.execution_coordinator import AppExecutionState

            native_host.mark_unknown()
            try:
                from libs.sensitive_http_telemetry import sensitive_http_telemetry
                from services.dify_builder.native_testing.context_runtime import NATIVE_LOG_EMITTERS

                # Scope entry/exit already restored the worker ContextVars.
                # Listener cache cleanup can log swallowed Redis failures, so
                # protect it independently of the now-failed observation guard.
                with sensitive_http_telemetry(additional_emitters=NATIVE_LOG_EMITTERS):
                    queue_manager.stop_listen(execution_state=AppExecutionState.TERMINAL)
            except BaseException:
                raise NativeObservationError() from None
        finally:
            if native_host is not None:
                native_host.worker_finished(worker=threading.current_thread(), exit_kind=exit_kind)
            marker = application_generate_entity.builder_execution
            if marker is not None and builder_execution_admit is not None:
                try:
                    builder_execution_admit.on_worker_finished(
                        context=marker,
                        native_run_id=application_generate_entity.workflow_execution_id,
                        task_id=application_generate_entity.task_id,
                        recorder=execution_recorder,
                        worker_thread=threading.current_thread(),
                        exit_kind=exit_kind,
                    )
                except BaseException:
                    # A missing/invalid candidate stays unknown; preserve the primary worker error.
                    logger.warning("Builder worker completion acknowledgment failed")

    def _handle_response(
        self,
        application_generate_entity: WorkflowAppGenerateEntity,
        workflow: Workflow,
        queue_manager: AppQueueManager,
        user: Account | EndUser,
        draft_var_saver_factory: DraftVariableSaverFactory,
        stream: bool = False,
        *,
        native_observation: NativeObservationGuard | None = None,
        native_host: NativeHostLaunch | None = None,
    ) -> WorkflowAppBlockingResponse | WorkflowAppPausedBlockingResponse | Generator[WorkflowAppStreamResponse]:
        """
        Handle response.
        :param application_generate_entity: application generate entity
        :param workflow: workflow
        :param queue_manager: queue manager
        :param user: account or end user
        :param stream: is stream
        :param workflow_node_execution_repository: optional repository for workflow node execution
        :return:
        """
        # init generate task pipeline
        generate_task_pipeline = WorkflowAppGenerateTaskPipeline(
            application_generate_entity=application_generate_entity,
            workflow=workflow,
            queue_manager=queue_manager,
            user=user,
            draft_var_saver_factory=draft_var_saver_factory,
            stream=stream,
            native_observation=native_observation,
            native_host=native_host,
        )

        try:
            return generate_task_pipeline.process()
        except ValueError as e:
            if native_observation is not None:
                native_observation.fail()
            if len(e.args) > 0 and e.args[0] == "I/O operation on closed file.":  # ignore this error
                raise GenerateTaskStoppedError() from e
            else:
                logger.exception(
                    "Fails to process generate task pipeline, task_id: %s", application_generate_entity.task_id
                )
                raise e