from __future__ import annotations import contextvars import logging import threading import uuid from collections.abc import Generator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Literal, Protocol, overload, runtime_checkable from flask import Flask, current_app from pydantic import ValidationError from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker import contexts from configs import dify_config from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.app.apps.base_app_generator import BaseAppGenerator from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom from core.app.apps.draft_variable_saver import DraftVariableSaver, DraftVariableSaverFactory, NoopDraftVariableSaver from core.app.apps.exc import GenerateTaskStoppedError from core.app.apps.workflow.active_workflow_tasks import active_workflow_task from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager from core.app.apps.workflow.generate_response_converter import WorkflowAppGenerateResponseConverter from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity from core.app.entities.task_entities import ( WorkflowAppBlockingResponse, WorkflowAppPausedBlockingResponse, WorkflowAppStreamResponse, ) from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig from core.db.session_factory import session_factory from core.helper.trace_id_helper import ( extract_external_trace_id_from_args, extract_parent_trace_context_from_args, extract_trace_session_id_from_args, ) from core.ops.ops_trace_manager import TraceQueueManager from core.repositories import DifyCoreRepositoryFactory from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepository from core.trigger.constants import is_trigger_node_type from extensions.ext_database import db from factories import file_factory from graphon.engine.filter import ResponseStreamFilter from graphon.engine.layer import Layer from graphon.enums import NodeType from graphon.model_runtime.errors.invoke import InvokeAuthorizationError from graphon.runtime import RuntimeState from graphon.variable_loader import DUMMY_VARIABLE_LOADER, VariableLoader from libs.flask_utils import preserve_flask_contexts from models.account import Account from models.enums import WorkflowRunTriggeredFrom from models.model import App, AppMode, EndUser from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom from runtime_contracts.builder.execution import ( BuilderExecutionContext, BuilderExecutionPolicyError, BuilderExecutionRecorder, ConfirmedToolExecutionContext, JsonToolExecutionContext, scalar_inputs_digest, ) from runtime_contracts.builder.native.agent_context import BuilderAgentExecutionContext from runtime_contracts.builder.native.contracts import NativeTestContextRef from services.dify_builder.native_testing.context_binding import NativeContextUnavailable from services.dify_builder.native_testing.observation_adapters import NativeObservationError, NativeObservationGuard from services.dify_builder.native_testing.runtime_bridge import NativeRuntimeBridge from services.workflow_draft_variable_service import DraftVarLoader, WorkflowDraftVariableService from services.workflow_runtime.node_factory import get_default_root_node_id from services.workflow_runtime.workflow_pipeline import WorkflowAppGenerateTaskPipeline from services.workflow_runtime.workflow_queue import WorkflowAppQueueManager from services.workflow_runtime.workflow_runner import WorkflowAppRunner if TYPE_CHECKING: from fields.workflow_run_fields import LoopNodeRunPayload from services.dify_builder.native_testing.host_execution import NativeHostLaunch from services.workflow_runtime.base_runner import WorkflowRunDriver logger = logging.getLogger(__name__) @runtime_checkable class _DebugTraceArgs(Protocol): @property def trace_session_id(self) -> object: ... def _extract_trace_session_id_from_debug_args(args: object) -> dict[str, str]: if isinstance(args, Mapping): return extract_trace_session_id_from_args(args) if isinstance(args, _DebugTraceArgs): try: trace_session_id = args.trace_session_id except AttributeError: return {} return extract_trace_session_id_from_args({"trace_session_id": trace_session_id}) return {} def _no_draft_variable_saver( app_id: str, node_id: str, node_type: NodeType, node_execution_id: str, enclosing_node_id: str | None = None, ) -> DraftVariableSaver: """``DraftVariableSaverFactory`` that never persists draft variables (existing no-op saver).""" _ = app_id, node_id, node_type, node_execution_id, enclosing_node_id return NoopDraftVariableSaver() class BuilderExecutionAdmission(Protocol): """Transient service-owned admission, explicitly injected into native execution. Owns both Builder policy variants: ``admit_raw`` refuses raw metadata for the context it was built with, and ``__call__`` revalidates the reserved sidecar and claims the launch identity once before any native node exists. """ def __call__( self, workflow: Workflow, app: App, entity: WorkflowAppGenerateEntity, root_node_id: str, resume: bool, ) -> BuilderExecutionRecorder: ... def admit_raw(self, workflow: Workflow, app: App) -> None: ... def on_refusal(self, error: BuilderExecutionPolicyError, *, preworker: bool = False) -> None: ... def on_worker_finished( self, *, context: BuilderExecutionContext | ConfirmedToolExecutionContext | JsonToolExecutionContext, native_run_id: str, task_id: str, recorder: BuilderExecutionRecorder | None, worker_thread: threading.Thread, exit_kind: Literal["returned", "policy_refused", "stopped", "failed"], ) -> None: ... class WorkflowAppGenerator(BaseAppGenerator): def __init__(self, *, execution_driver: WorkflowRunDriver) -> None: self._execution_driver = execution_driver @staticmethod def _ensure_snippet_start_node_in_worker(*, session: Session, workflow: Workflow) -> Workflow: """Re-apply snippet virtual Start injection after worker reloads workflow from DB.""" if workflow.kind_or_standard != "snippet": return workflow from models.snippet import CustomizedSnippet from services.snippet_generate_service import SnippetGenerateService snippet = session.scalar( select(CustomizedSnippet).where( CustomizedSnippet.id == workflow.app_id, CustomizedSnippet.tenant_id == workflow.tenant_id, ) ) if snippet is None: return workflow return SnippetGenerateService.ensure_start_node_for_worker(workflow, snippet) @overload def generate( self, *, app_model: App, workflow: Workflow, user: Account | EndUser, args: Mapping[str, Any], invoke_from: InvokeFrom, streaming: Literal[True], call_depth: int, workflow_run_id: str | uuid.UUID | None = None, triggered_from: WorkflowRunTriggeredFrom | None = None, root_node_id: str | None = None, graph_engine_layers: Sequence[Layer] = (), allow_human_input: bool = True, pause_state_config: PauseStateLayerConfig | None = None, builder_execution: BuilderExecutionContext | ConfirmedToolExecutionContext | JsonToolExecutionContext | None = None, builder_execution_admit: BuilderExecutionAdmission | None = None, builder_agent_execution: BuilderAgentExecutionContext | None = None, native_test_required: bool = False, native_test_ref: NativeTestContextRef | None = None, native_runtime: NativeRuntimeBridge | None = None, native_host: NativeHostLaunch | None = None, builder_mocked_node_ids: frozenset[str] = frozenset(), builder_expected_revision: str | None = None, builder_tool_credential_bindings: Mapping[str, str] | None = None, ) -> Generator[Mapping[str, Any] | str]: ... @overload def generate( self, *, app_model: App, workflow: Workflow, user: Account | EndUser, args: Mapping[str, Any], invoke_from: InvokeFrom, streaming: Literal[False], call_depth: int, workflow_run_id: str | uuid.UUID | None = None, triggered_from: WorkflowRunTriggeredFrom | None = None, root_node_id: str | None = None, graph_engine_layers: Sequence[Layer] = (), allow_human_input: bool = True, pause_state_config: PauseStateLayerConfig | None = None, builder_execution: BuilderExecutionContext | ConfirmedToolExecutionContext | JsonToolExecutionContext | None = None, builder_execution_admit: BuilderExecutionAdmission | None = None, builder_agent_execution: BuilderAgentExecutionContext | None = None, native_test_required: bool = False, native_test_ref: NativeTestContextRef | None = None, native_runtime: NativeRuntimeBridge | None = None, native_host: NativeHostLaunch | None = None, builder_mocked_node_ids: frozenset[str] = frozenset(), builder_expected_revision: str | None = None, builder_tool_credential_bindings: Mapping[str, str] | None = None, ) -> Mapping[str, Any]: ... @overload def generate( self, *, app_model: App, workflow: Workflow, user: Account | EndUser, args: Mapping[str, Any], invoke_from: InvokeFrom, streaming: bool, call_depth: int, workflow_run_id: str | uuid.UUID | None = None, triggered_from: WorkflowRunTriggeredFrom | None = None, root_node_id: str | None = None, graph_engine_layers: Sequence[Layer] = (), allow_human_input: bool = True, pause_state_config: PauseStateLayerConfig | None = None, builder_execution: BuilderExecutionContext | ConfirmedToolExecutionContext | JsonToolExecutionContext | None = None, builder_execution_admit: BuilderExecutionAdmission | None = None, builder_agent_execution: BuilderAgentExecutionContext | None = None, native_test_required: bool = False, native_test_ref: NativeTestContextRef | None = None, native_runtime: NativeRuntimeBridge | None = None, native_host: NativeHostLaunch | None = None, builder_mocked_node_ids: frozenset[str] = frozenset(), builder_expected_revision: str | None = None, builder_tool_credential_bindings: Mapping[str, str] | None = None, ) -> Mapping[str, Any] | Generator[Mapping[str, Any] | str]: ... def generate( self, *, app_model: App, workflow: Workflow, user: Account | EndUser, args: Mapping[str, Any], invoke_from: InvokeFrom, streaming: bool = True, call_depth: int = 0, workflow_run_id: str | uuid.UUID | None = None, triggered_from: WorkflowRunTriggeredFrom | None = None, root_node_id: str | None = None, graph_engine_layers: Sequence[Layer] = (), allow_human_input: bool = True, pause_state_config: PauseStateLayerConfig | None = None, builder_execution: BuilderExecutionContext | ConfirmedToolExecutionContext | JsonToolExecutionContext | None = None, builder_execution_admit: BuilderExecutionAdmission | None = None, builder_agent_execution: BuilderAgentExecutionContext | None = None, native_test_required: bool = False, native_test_ref: NativeTestContextRef | None = None, native_runtime: NativeRuntimeBridge | None = None, native_host: NativeHostLaunch | None = None, builder_mocked_node_ids: frozenset[str] = frozenset(), builder_expected_revision: str | None = None, builder_tool_credential_bindings: Mapping[str, str] | None = None, ) -> Mapping[str, Any] | Generator[Mapping[str, Any] | str]: from services.dify_builder.native_launch_observation import begin_preparation from services.dify_builder.native_testing.generation_wiring import validate_native_host begin_preparation() if builder_agent_execution is not None: if ( builder_execution is not None or native_test_required or native_test_ref is not None or native_runtime is not None or native_host is not None or invoke_from != InvokeFrom.DEBUGGER or not isinstance(user, Account) or app_model.mode != AppMode.WORKFLOW or call_depth != 0 or (workflow_run_id is not None and str(workflow_run_id) != builder_agent_execution.native_run_id) ): raise BuilderExecutionPolicyError("unsupported_execution_transport") workflow_run_id = builder_agent_execution.native_run_id validate_native_host( required=native_test_required or native_test_ref is not None, ref=native_test_ref, runtime=native_runtime, host=native_host, ) native_context = None native_inputs = None if native_test_required or native_test_ref is not None: if native_test_ref is None: raise NativeContextUnavailable("native_context_missing") if native_runtime is None: raise NativeContextUnavailable("native_runtime_unavailable") native_context = native_runtime.resolve(native_test_ref) if ( builder_execution is not None or (app_model.tenant_id, app_model.id, workflow.id, user.id, str(workflow_run_id), call_depth) != ( native_context.tenant_id, native_context.app_id, native_context.workflow_id, native_context.actor_id, native_context.native_run_id, native_context.call_depth, ) or workflow.graph_dict != native_context.graph ): raise NativeContextUnavailable("native_context_mismatch") assert native_host is not None guard = native_host.bind_observation() with native_host.observation_scope(): if ( args.get("files") or pause_state_config is not None or graph_engine_layers or not streaming or workflow.features_dict.get("text_to_speech", {}).get("enabled") or any( args.get(key) is not None for key in ("trace_session_id", "external_trace_id", "parent_trace_context") ) ): guard.fail() native_inputs = native_runtime.prepare_inputs(native_context) if ( builder_mocked_node_ids or builder_expected_revision is not None or builder_tool_credential_bindings is not None ) and (builder_execution is not None or native_context is not None or invoke_from != InvokeFrom.DEBUGGER): raise BuilderExecutionPolicyError("unsupported_execution_policy") try: if builder_execution is not None and ( builder_execution_admit is None or args.get("files") or call_depth != 0 or not isinstance(user, Account) or invoke_from != InvokeFrom.DEBUGGER or pause_state_config is not None or graph_engine_layers ): raise BuilderExecutionPolicyError("unsupported_execution_transport") if builder_execution is not None: assert builder_execution_admit is not None builder_execution_admit.admit_raw(workflow, app_model) scalar_inputs_digest(args.get("inputs", {})) except BuilderExecutionPolicyError as error: if builder_execution_admit is not None: builder_execution_admit.on_refusal(error, preworker=True) raise with self._bind_file_access_scope(tenant_id=app_model.tenant_id, user=user, invoke_from=invoke_from): files: Sequence[Mapping[str, Any]] = args.get("files") or [] # parse files # TODO(QuantumGhost): Move file parsing logic to the API controller layer # for better separation of concerns. # # For implementation reference, see the `_parse_file` function and # `DraftWorkflowNodeRunApi` class which handle this properly. file_extra_config = FileUploadConfigManager.convert(workflow.features_dict, is_vision=False) system_files = ( [] if builder_execution is not None or native_context is not None else file_factory.build_from_mappings( mappings=files, tenant_id=app_model.tenant_id, config=file_extra_config, strict_type_validation=invoke_from == InvokeFrom.SERVICE_API, access_controller=self._file_access_controller, ) ) # convert to app config app_config = WorkflowAppConfigManager.get_app_config( app_model=app_model, workflow=workflow, ) trace_manager = None if builder_execution is None and native_context is None: trace_manager = TraceQueueManager( app_id=app_model.id, user_id=user.id if isinstance(user, Account) else user.session_id, ) inputs: Mapping[str, Any] = native_inputs.inputs if native_inputs is not None else args["inputs"] extras = { **extract_external_trace_id_from_args(args), **extract_parent_trace_context_from_args(args), **extract_trace_session_id_from_args(args), } workflow_run_id = str(workflow_run_id or uuid.uuid4()) root_node_id = root_node_id or get_default_root_node_id(workflow.graph_dict) root_node_config = workflow.get_node_config_by_id(root_node_id) root_node_type = workflow.get_node_type_from_node_config(root_node_config) if builder_execution is not None and root_node_type != "start": error = BuilderExecutionPolicyError("unsupported_execution_root") if builder_execution_admit is not None: builder_execution_admit.on_refusal(error, preworker=True) raise error # Trigger inputs are already adapted event data, not Start form fields. if native_context is None and not is_trigger_node_type(root_node_type): inputs = self._prepare_user_inputs( user_inputs=inputs, variables=app_config.variables, tenant_id=app_model.tenant_id, strict_type_validation=invoke_from == InvokeFrom.SERVICE_API, ) # init application generate entity application_generate_entity = WorkflowAppGenerateEntity( task_id=str(uuid.uuid4()), app_config=app_config, file_upload_config=file_extra_config, inputs=inputs, files=list(native_inputs.files if native_inputs is not None else system_files), user_id=user.id, stream=streaming, invoke_from=invoke_from, allow_human_input=allow_human_input, call_depth=call_depth, builder_execution=builder_execution, builder_agent_execution=builder_agent_execution, native_test_required=native_test_required, native_test_ref=native_test_ref, builder_mocked_node_ids=builder_mocked_node_ids, builder_expected_revision=builder_expected_revision, builder_tool_credential_bindings=builder_tool_credential_bindings, trace_manager=trace_manager, workflow_execution_id=workflow_run_id, extras=extras, ) if builder_execution is not None: if scalar_inputs_digest(dict(inputs)) != builder_execution.effective_inputs_digest: error = BuilderExecutionPolicyError("effective_input_binding_mismatch") if builder_execution_admit is not None: builder_execution_admit.on_refusal(error, preworker=True) raise error application_generate_entity.trace_manager = TraceQueueManager( app_id=app_model.id, user_id=user.id, external_tracing_disabled=True, ) contexts.plugin_tool_providers.set({}) contexts.plugin_tool_providers_lock.set(threading.Lock()) # Create repositories # # Create session factory session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) # Create workflow execution(aka workflow run) repository if triggered_from is not None: # Use explicitly provided triggered_from (for async triggers) workflow_triggered_from = triggered_from elif invoke_from == InvokeFrom.DEBUGGER: workflow_triggered_from = WorkflowRunTriggeredFrom.DEBUGGING else: workflow_triggered_from = WorkflowRunTriggeredFrom.APP_RUN workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository( session_factory=session_factory, tenant_id=app_model.tenant_id, user=user, app_id=application_generate_entity.app_config.app_id, triggered_from=workflow_triggered_from, ) # Create workflow node execution repository workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=session_factory, tenant_id=app_model.tenant_id, user=user, app_id=application_generate_entity.app_config.app_id, triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, ) return self._generate( app_model=app_model, workflow=workflow, user=user, application_generate_entity=application_generate_entity, invoke_from=invoke_from, workflow_execution_repository=workflow_execution_repository, workflow_node_execution_repository=workflow_node_execution_repository, streaming=streaming, root_node_id=root_node_id, graph_engine_layers=graph_engine_layers, pause_state_config=pause_state_config, builder_execution_admit=builder_execution_admit, native_runtime=native_runtime, native_host=native_host, ) def resume( self, *, app_model: App, workflow: Workflow, user: Account | EndUser, application_generate_entity: WorkflowAppGenerateEntity, graph_runtime_state: RuntimeState, workflow_execution_repository: WorkflowExecutionRepository, workflow_node_execution_repository: WorkflowNodeExecutionRepository, graph_engine_layers: Sequence[Layer] = (), pause_state_config: PauseStateLayerConfig | None = None, variable_loader: VariableLoader = DUMMY_VARIABLE_LOADER, response_stream_filter: ResponseStreamFilter | None = None, native_runtime: NativeRuntimeBridge | None = None, native_host: NativeHostLaunch | None = None, ) -> Mapping[str, Any] | Generator[str | Mapping[str, Any]]: """ Resume a paused workflow execution using the persisted runtime state. ``trace_manager`` is transient and excluded from generate-entity serialization, so resumed executions rebuild it here before persistence layers receive the entity. """ if application_generate_entity.builder_agent_execution is not None and ( application_generate_entity.builder_execution is not None or application_generate_entity.native_test_required or application_generate_entity.native_test_ref is not None or native_runtime is not None or native_host is not None or application_generate_entity.invoke_from != InvokeFrom.DEBUGGER or not isinstance(user, Account) or app_model.mode != AppMode.WORKFLOW or application_generate_entity.call_depth != 0 or application_generate_entity.workflow_execution_id != application_generate_entity.builder_agent_execution.native_run_id ): raise BuilderExecutionPolicyError("unsupported_execution_transport") if application_generate_entity.native_test_required or application_generate_entity.native_test_ref is not None: if application_generate_entity.native_test_ref is None: raise NativeContextUnavailable("native_context_missing") from services.dify_builder.native_testing.generation_wiring import validate_native_host validate_native_host( required=True, ref=application_generate_entity.native_test_ref, runtime=native_runtime, host=native_host ) assert native_host is not None native_host.require_resume_bindings( entity=application_generate_entity, runtime_state=graph_runtime_state, response_filter=response_stream_filter, ) if application_generate_entity.builder_execution is not None: raise BuilderExecutionPolicyError("unsupported_execution_transport") if native_host is None and application_generate_entity.trace_manager is None: application_generate_entity = application_generate_entity.model_copy( update={ "trace_manager": TraceQueueManager( app_id=app_model.id, user_id=user.id if isinstance(user, Account) else user.session_id, ) } ) return self._generate( app_model=app_model, workflow=workflow, user=user, application_generate_entity=application_generate_entity, invoke_from=application_generate_entity.invoke_from, workflow_execution_repository=workflow_execution_repository, workflow_node_execution_repository=workflow_node_execution_repository, streaming=application_generate_entity.stream, variable_loader=variable_loader, graph_engine_layers=graph_engine_layers, graph_runtime_state=graph_runtime_state, pause_state_config=pause_state_config, response_stream_filter=response_stream_filter, native_runtime=native_runtime, native_host=native_host, ) def _generate( self, *, app_model: App, workflow: Workflow, user: Account | EndUser, application_generate_entity: WorkflowAppGenerateEntity, invoke_from: InvokeFrom, workflow_execution_repository: WorkflowExecutionRepository, workflow_node_execution_repository: WorkflowNodeExecutionRepository, streaming: bool = True, variable_loader: VariableLoader = DUMMY_VARIABLE_LOADER, root_node_id: str | None = None, graph_engine_layers: Sequence[Layer] = (), graph_runtime_state: RuntimeState | None = None, pause_state_config: PauseStateLayerConfig | None = None, response_stream_filter: ResponseStreamFilter | None = None, builder_execution_admit: BuilderExecutionAdmission | None = None, native_runtime: NativeRuntimeBridge | None = None, native_host: NativeHostLaunch | None = None, ) -> Mapping[str, Any] | Generator[str | Mapping[str, Any]]: """ Generate App response. :param app_model: App :param workflow: Workflow :param user: account or end user :param application_generate_entity: application generate entity :param invoke_from: invoke from source :param workflow_execution_repository: repository for workflow execution :param workflow_node_execution_repository: repository for workflow node execution :param streaming: is stream """ from services.dify_builder.native_testing.generation_wiring import bind_native_generation, validate_native_host validate_native_host( required=application_generate_entity.native_test_required or application_generate_entity.native_test_ref is not None, ref=application_generate_entity.native_test_ref, runtime=native_runtime, host=native_host, ) if native_host is not None and (graph_engine_layers or application_generate_entity.trace_manager is not None): raise NativeContextUnavailable("native_observation_unqualified") if native_host is not None: if not streaming or pause_state_config is not None: raise NativeContextUnavailable("native_resume_unavailable") if graph_runtime_state is not None: native_host.require_resume_bindings( entity=application_generate_entity, runtime_state=graph_runtime_state, response_filter=response_stream_filter, ) native_observation = None if native_host is not None: from services.dify_builder.native_testing.generation_wiring import preflight_native_observation native_observation = native_host.bind_observation() preflight_native_observation(host=native_host, entity=application_generate_entity, workflow=workflow) with self._bind_file_access_scope( tenant_id=application_generate_entity.app_config.tenant_id, user=user, invoke_from=invoke_from, ): # init queue manager queue_manager = WorkflowAppQueueManager( task_id=application_generate_entity.task_id, user_id=application_generate_entity.user_id, invoke_from=application_generate_entity.invoke_from, app_mode=app_model.mode, native_observation=native_observation, native_host=native_host, ) resolved_response_stream_filter = response_stream_filter or ResponseStreamFilter() if native_host is not None: bind_native_generation( host=native_host, entity=application_generate_entity, queue=queue_manager, response_filter=resolved_response_stream_filter, workflow_repository=workflow_execution_repository, node_repository=workflow_node_execution_repository, ) # new thread with request context and contextvars context = contextvars.copy_context() # release database connection, because the following new thread operations may take a long time db.session.close() worker_thread = threading.Thread( target=self._generate_worker, kwargs={ "flask_app": current_app._get_current_object(), # type: ignore "application_generate_entity": application_generate_entity, "queue_manager": queue_manager, "context": context, "variable_loader": variable_loader, "root_node_id": root_node_id, "workflow_execution_repository": workflow_execution_repository, "workflow_node_execution_repository": workflow_node_execution_repository, "graph_engine_layers": graph_engine_layers, "pause_state_config": pause_state_config, "graph_runtime_state": graph_runtime_state, "response_stream_filter": resolved_response_stream_filter, "builder_execution_admit": builder_execution_admit, "native_runtime": native_runtime, "native_host": native_host, }, ) from services.dify_builder.native_launch_observation import mark_dispatched mark_dispatched() if native_host is None: worker_thread.start() else: native_host.start_worker(worker=worker_thread) draft_var_saver_factory: DraftVariableSaverFactory if native_host is not None or isinstance( application_generate_entity.builder_execution, ConfirmedToolExecutionContext ): # The authorized native Run/node rows are the only retained copy of a # confirmed Tool's content: no mutable workspace draft-variable copy. draft_var_saver_factory = _no_draft_variable_saver else: draft_var_saver_factory = self._get_draft_var_saver_factory( invoke_from, user, tenant_id=app_model.tenant_id, ) try: response = self._handle_response( application_generate_entity=application_generate_entity, workflow=workflow, queue_manager=queue_manager, user=user, draft_var_saver_factory=draft_var_saver_factory, stream=streaming, native_observation=native_observation, native_host=native_host, ) converted_response = WorkflowAppGenerateResponseConverter.convert( response=response, invoke_from=invoke_from, ) except BaseException: if native_host is None: self._join_worker_thread(worker_thread) else: native_host.request_close(reason="failed") raise if native_host is not None: if not isinstance(converted_response, Generator): native_host.request_close(reason="failed") raise NativeContextUnavailable("native_stream_unavailable") return converted_response if isinstance(converted_response, Generator): return self._wrap_stream_with_worker_thread_join(converted_response, worker_thread) self._join_worker_thread(worker_thread) return converted_response def single_iteration_generate( self, app_model: App, workflow: Workflow, node_id: str, user: Account | EndUser, args: Mapping[str, Any], streaming: bool = True, *, session: Session, ) -> Mapping[str, Any] | Generator[str | Mapping[str, Any]]: """ Generate App response. :param app_model: App :param workflow: Workflow :param node_id: the node id :param user: account or end user :param args: request args :param streaming: is streamed :param session: database session supplied by the caller """ if not node_id: raise ValueError("node_id is required") if args.get("inputs") is None: raise ValueError("inputs is required") # convert to app config app_config = WorkflowAppConfigManager.get_app_config(app_model=app_model, workflow=workflow) # init application generate entity application_generate_entity = WorkflowAppGenerateEntity( task_id=str(uuid.uuid4()), app_config=app_config, inputs={}, files=[], user_id=user.id, stream=streaming, invoke_from=InvokeFrom.DEBUGGER, extras={ "auto_generate_conversation_name": False, **_extract_trace_session_id_from_debug_args(args), }, single_iteration_run=WorkflowAppGenerateEntity.SingleIterationRunEntity( node_id=node_id, inputs=args["inputs"] ), workflow_execution_id=str(uuid.uuid4()), ) contexts.plugin_tool_providers.set({}) contexts.plugin_tool_providers_lock.set(threading.Lock()) # Create repositories # # Create session factory session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) # Create workflow execution(aka workflow run) repository workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository( session_factory=session_factory, tenant_id=app_model.tenant_id, user=user, app_id=application_generate_entity.app_config.app_id, triggered_from=WorkflowRunTriggeredFrom.DEBUGGING, ) # Create workflow node execution repository workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=session_factory, tenant_id=app_model.tenant_id, user=user, app_id=application_generate_entity.app_config.app_id, triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP, ) draft_var_srv = WorkflowDraftVariableService(session) draft_var_srv.prefill_conversation_variable_default_values(workflow, user_id=user.id) var_loader = DraftVarLoader( engine=db.engine, app_id=application_generate_entity.app_config.app_id, tenant_id=application_generate_entity.app_config.tenant_id, user_id=user.id, ) return self._generate( app_model=app_model, workflow=workflow, user=user, invoke_from=InvokeFrom.DEBUGGER, application_generate_entity=application_generate_entity, workflow_execution_repository=workflow_execution_repository, workflow_node_execution_repository=workflow_node_execution_repository, streaming=streaming, variable_loader=var_loader, pause_state_config=None, ) def single_loop_generate( self, app_model: App, workflow: Workflow, node_id: str, user: Account | EndUser, args: LoopNodeRunPayload, streaming: bool = True, *, session: Session, ) -> Mapping[str, Any] | Generator[str | Mapping[str, Any]]: """ Generate App response. :param app_model: App :param workflow: Workflow :param node_id: the node id :param user: account or end user :param args: request args :param streaming: is streamed :param session: database session supplied by the caller """ if not node_id: raise ValueError("node_id is required") if args.inputs is None: raise ValueError("inputs is required") # convert to app config app_config = WorkflowAppConfigManager.get_app_config(app_model=app_model, workflow=workflow) # init application generate entity application_generate_entity = WorkflowAppGenerateEntity( task_id=str(uuid.uuid4()), app_config=app_config, inputs={}, files=[], user_id=user.id, stream=streaming, invoke_from=InvokeFrom.DEBUGGER, extras={ "auto_generate_conversation_name": False, **_extract_trace_session_id_from_debug_args(args), }, single_loop_run=WorkflowAppGenerateEntity.SingleLoopRunEntity(node_id=node_id, inputs=args.inputs or {}), workflow_execution_id=str(uuid.uuid4()), ) contexts.plugin_tool_providers.set({}) contexts.plugin_tool_providers_lock.set(threading.Lock()) # Create repositories # # Create session factory session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) # Create workflow execution(aka workflow run) repository workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository( session_factory=session_factory, tenant_id=app_model.tenant_id, user=user, app_id=application_generate_entity.app_config.app_id, triggered_from=WorkflowRunTriggeredFrom.DEBUGGING, ) # Create workflow node execution repository workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=session_factory, tenant_id=app_model.tenant_id, user=user, app_id=application_generate_entity.app_config.app_id, triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP, ) draft_var_srv = WorkflowDraftVariableService(session) draft_var_srv.prefill_conversation_variable_default_values(workflow, user_id=user.id) var_loader = DraftVarLoader( engine=db.engine, app_id=application_generate_entity.app_config.app_id, tenant_id=application_generate_entity.app_config.tenant_id, user_id=user.id, ) return self._generate( app_model=app_model, workflow=workflow, user=user, invoke_from=InvokeFrom.DEBUGGER, application_generate_entity=application_generate_entity, workflow_execution_repository=workflow_execution_repository, workflow_node_execution_repository=workflow_node_execution_repository, streaming=streaming, variable_loader=var_loader, pause_state_config=None, ) def _generate_worker( self, flask_app: Flask, application_generate_entity: WorkflowAppGenerateEntity, queue_manager: AppQueueManager, context: contextvars.Context, variable_loader: VariableLoader, workflow_execution_repository: WorkflowExecutionRepository, workflow_node_execution_repository: WorkflowNodeExecutionRepository, root_node_id: str | None = None, graph_engine_layers: Sequence[Layer] = (), graph_runtime_state: RuntimeState | None = None, response_stream_filter: ResponseStreamFilter | None = None, builder_execution_admit: BuilderExecutionAdmission | None = None, native_runtime: NativeRuntimeBridge | None = None, native_host: NativeHostLaunch | None = None, pause_state_config: PauseStateLayerConfig | None = None, ) -> None: """ Generate worker in a new thread. :param flask_app: Flask app :param application_generate_entity: application generate entity :param queue_manager: queue manager :param workflow_thread_pool_id: workflow thread pool id :return: """ execution_recorder = None exit_kind: Literal["returned", "policy_refused", "stopped", "failed"] = "failed" try: from services.dify_builder.native_testing.context_runtime import native_flask_worker_scope worker_scope = ( native_flask_worker_scope(host=native_host, flask_app=flask_app, context_vars=context) if native_host is not None else preserve_flask_contexts(flask_app, context_vars=context) ) with worker_scope: try: native_context = None if ( application_generate_entity.native_test_required or application_generate_entity.native_test_ref is not None ): if application_generate_entity.native_test_ref is None: raise NativeContextUnavailable("native_context_missing") if native_runtime is None: raise NativeContextUnavailable("native_runtime_unavailable") native_context = native_runtime.resolve(application_generate_entity.native_test_ref) if ( application_generate_entity.app_config.tenant_id, application_generate_entity.app_config.app_id, application_generate_entity.app_config.workflow_id, application_generate_entity.user_id, application_generate_entity.workflow_execution_id, ) != ( native_context.tenant_id, native_context.app_id, native_context.workflow_id, native_context.actor_id, native_context.native_run_id, ): raise NativeContextUnavailable("native_context_mismatch") from services.dify_builder.native_testing.generation_wiring import validate_native_host validate_native_host( required=True, ref=application_generate_entity.native_test_ref, runtime=native_runtime, host=native_host, ) assert native_host is not None if native_host.generation_bindings.entity is not application_generate_entity: raise NativeContextUnavailable("native_host_identity_mismatch") prepared = native_runtime.prepare_inputs(native_context) if graph_runtime_state is None: application_generate_entity.inputs = prepared.inputs application_generate_entity.files = list(prepared.files) else: native_host.require_resume_bindings( entity=application_generate_entity, runtime_state=graph_runtime_state, response_filter=response_stream_filter, ) application_generate_entity.call_depth = native_context.call_depth with session_factory.create_session() as session: workflow = session.scalar( select(Workflow).where( Workflow.tenant_id == application_generate_entity.app_config.tenant_id, Workflow.app_id == application_generate_entity.app_config.app_id, Workflow.id == application_generate_entity.app_config.workflow_id, ) ) if workflow is None: if application_generate_entity.builder_execution is not None: raise BuilderExecutionPolicyError("execution_owner_mismatch") raise ValueError("Workflow not found") if application_generate_entity.builder_execution is not None: if builder_execution_admit is None: raise BuilderExecutionPolicyError("missing_execution_admission") app = session.scalar( select(App).where( App.id == application_generate_entity.app_config.app_id, App.tenant_id == application_generate_entity.app_config.tenant_id, ) ) if app is None: raise BuilderExecutionPolicyError("execution_owner_mismatch") if graph_engine_layers: raise BuilderExecutionPolicyError("unsupported_execution_transport") if isinstance(application_generate_entity.builder_execution, JsonToolExecutionContext): # The owned worker read must release its SQL transaction # before the issuer revalidates daemon metadata. session.expunge_all() session.close() execution_recorder = builder_execution_admit( workflow, app, application_generate_entity, root_node_id or get_default_root_node_id(workflow.graph_dict), graph_runtime_state is not None, ) if execution_recorder is None or not execution_recorder.healthy: raise BuilderExecutionPolicyError("missing_execution_recorder") if native_context is not None: if workflow.graph_dict != native_context.graph: raise NativeContextUnavailable("native_context_mismatch") else: workflow = self._ensure_snippet_start_node_in_worker(session=session, workflow=workflow) if graph_runtime_state is not None and native_context is None: self._restore_workflow_run_graph( session=session, workflow=workflow, workflow_run_id=application_generate_entity.workflow_execution_id, ) # Determine system_user_id based on invocation source is_external_api_call = application_generate_entity.invoke_from in { InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API, } if is_external_api_call: # For external API calls, use end user's session ID end_user = session.scalar( select(EndUser).where(EndUser.id == application_generate_entity.user_id) ) system_user_id = end_user.session_id if end_user else "" else: # For internal calls, use the original user ID system_user_id = application_generate_entity.user_id if application_generate_entity.builder_expected_revision is not None: from services.dify_builder.revision import execution_revision if execution_revision(workflow) != application_generate_entity.builder_expected_revision: raise BuilderExecutionPolicyError("stale_execution_revision") runner = WorkflowAppRunner( application_generate_entity=application_generate_entity, execution_driver=self._execution_driver, queue_manager=queue_manager, variable_loader=variable_loader, workflow=workflow, system_user_id=system_user_id, native_runtime=native_runtime, native_host=native_host, root_node_id=root_node_id, graph_engine_layers=graph_engine_layers, graph_runtime_state=graph_runtime_state, response_stream_filter=response_stream_filter, execution_recorder=execution_recorder, ) with active_workflow_task(application_generate_entity.task_id): self._execution_driver( runner, pause_state_config, workflow_execution_repository, workflow_node_execution_repository, ) exit_kind = "returned" except GenerateTaskStoppedError: exit_kind = "stopped" logger.warning("Task stopped", exc_info=native_host is None) pass except InvokeAuthorizationError: queue_manager.publish_error( InvokeAuthorizationError("Incorrect API key provided"), PublishFrom.APPLICATION_MANAGER ) except NativeContextUnavailable as e: exit_kind = "policy_refused" queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER) except ValidationError as e: if native_host is not None: queue_manager.publish_error(NativeObservationError(), PublishFrom.APPLICATION_MANAGER) else: logger.exception("Validation Error when generating") queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER) except BuilderExecutionPolicyError as e: exit_kind = "policy_refused" if builder_execution_admit is not None: builder_execution_admit.on_refusal(e) queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER) except ValueError as e: if native_host is not None: queue_manager.publish_error(NativeObservationError(), PublishFrom.APPLICATION_MANAGER) else: if dify_config.DEBUG: logger.exception("Error when generating") queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER) except Exception as e: if native_host is not None: queue_manager.publish_error(NativeObservationError(), PublishFrom.APPLICATION_MANAGER) else: logger.exception("Unknown Error when generating") queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER) except BaseException: exit_kind = "failed" if native_host is None: raise # Scope entry itself can fail before the inner publisher exists. # A failed guard must not project another payload. Close the listener # segment only; physical cleanup remains owned by the host/reader. from core.app.apps.execution_coordinator import AppExecutionState native_host.mark_unknown() try: from libs.sensitive_http_telemetry import sensitive_http_telemetry from services.dify_builder.native_testing.context_runtime import NATIVE_LOG_EMITTERS # Scope entry/exit already restored the worker ContextVars. # Listener cache cleanup can log swallowed Redis failures, so # protect it independently of the now-failed observation guard. with sensitive_http_telemetry(additional_emitters=NATIVE_LOG_EMITTERS): queue_manager.stop_listen(execution_state=AppExecutionState.TERMINAL) except BaseException: raise NativeObservationError() from None finally: if native_host is not None: native_host.worker_finished(worker=threading.current_thread(), exit_kind=exit_kind) marker = application_generate_entity.builder_execution if marker is not None and builder_execution_admit is not None: try: builder_execution_admit.on_worker_finished( context=marker, native_run_id=application_generate_entity.workflow_execution_id, task_id=application_generate_entity.task_id, recorder=execution_recorder, worker_thread=threading.current_thread(), exit_kind=exit_kind, ) except BaseException: # A missing/invalid candidate stays unknown; preserve the primary worker error. logger.warning("Builder worker completion acknowledgment failed") def _handle_response( self, application_generate_entity: WorkflowAppGenerateEntity, workflow: Workflow, queue_manager: AppQueueManager, user: Account | EndUser, draft_var_saver_factory: DraftVariableSaverFactory, stream: bool = False, *, native_observation: NativeObservationGuard | None = None, native_host: NativeHostLaunch | None = None, ) -> WorkflowAppBlockingResponse | WorkflowAppPausedBlockingResponse | Generator[WorkflowAppStreamResponse]: """ Handle response. :param application_generate_entity: application generate entity :param workflow: workflow :param queue_manager: queue manager :param user: account or end user :param stream: is stream :param workflow_node_execution_repository: optional repository for workflow node execution :return: """ # init generate task pipeline generate_task_pipeline = WorkflowAppGenerateTaskPipeline( application_generate_entity=application_generate_entity, workflow=workflow, queue_manager=queue_manager, user=user, draft_var_saver_factory=draft_var_saver_factory, stream=stream, native_observation=native_observation, native_host=native_host, ) try: return generate_task_pipeline.process() except ValueError as e: if native_observation is not None: native_observation.fail() if len(e.args) > 0 and e.args[0] == "I/O operation on closed file.": # ignore this error raise GenerateTaskStoppedError() from e else: logger.exception( "Fails to process generate task pipeline, task_id: %s", application_generate_entity.task_id ) raise e