Feat/support multimodal embedding (#29115)

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

api/controllers/console/datasets/datasets.py

@@ -151,6 +151,7 @@ class DatasetUpdatePayload(BaseModel):
     external_knowledge_id: str | None = None
     external_knowledge_api_id: str | None = None
     icon_info: dict[str, Any] | None = None
+    is_multimodal: bool | None = False
 
     @field_validator("indexing_technique")
     @classmethod
@@ -423,17 +424,16 @@ class DatasetApi(Resource):
         payload = DatasetUpdatePayload.model_validate(console_ns.payload or {})
         payload_data = payload.model_dump(exclude_unset=True)
         current_user, current_tenant_id = current_account_with_tenant()
-
         # check embedding model setting
         if (
             payload.indexing_technique == "high_quality"
             and payload.embedding_model_provider is not None
             and payload.embedding_model is not None
         ):
-            DatasetService.check_embedding_model_setting(
+            is_multimodal = DatasetService.check_is_multimodal_model(
                 dataset.tenant_id, payload.embedding_model_provider, payload.embedding_model
             )
-
+            payload.is_multimodal = is_multimodal
         # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
         DatasetPermissionService.check_permission(
             current_user, dataset, payload.permission, payload.partial_member_list

api/controllers/console/datasets/datasets_document.py

@@ -424,6 +424,10 @@ class DatasetInitApi(Resource):
                     model_type=ModelType.TEXT_EMBEDDING,
                     model=knowledge_config.embedding_model,
                 )
+                is_multimodal = DatasetService.check_is_multimodal_model(
+                    current_tenant_id, knowledge_config.embedding_model_provider, knowledge_config.embedding_model
+                )
+                knowledge_config.is_multimodal = is_multimodal
             except InvokeAuthorizationError:
                 raise ProviderNotInitializeError(
                     "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."

api/controllers/console/datasets/datasets_segments.py

@@ -51,6 +51,7 @@ class SegmentCreatePayload(BaseModel):
     content: str
     answer: str | None = None
     keywords: list[str] | None = None
+    attachment_ids: list[str] | None = None
 
 
 class SegmentUpdatePayload(BaseModel):
@@ -58,6 +59,7 @@ class SegmentUpdatePayload(BaseModel):
     answer: str | None = None
     keywords: list[str] | None = None
     regenerate_child_chunks: bool = False
+    attachment_ids: list[str] | None = None
 
 
 class BatchImportPayload(BaseModel):

api/controllers/console/datasets/hit_testing_base.py

@@ -1,7 +1,7 @@
 import logging
 from typing import Any
 
-from flask_restx import marshal
+from flask_restx import marshal, reqparse
 from pydantic import BaseModel, Field
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
@@ -33,6 +33,7 @@ class HitTestingPayload(BaseModel):
     query: str = Field(max_length=250)
     retrieval_model: dict[str, Any] | None = None
     external_retrieval_model: dict[str, Any] | None = None
+    attachment_ids: list[str] | None = None
 
 
 class DatasetsHitTestingBase:
@@ -54,16 +55,28 @@ class DatasetsHitTestingBase:
     def hit_testing_args_check(args: dict[str, Any]):
         HitTestingService.hit_testing_args_check(args)
 
+    @staticmethod
+    def parse_args():
+        parser = (
+            reqparse.RequestParser()
+            .add_argument("query", type=str, required=False, location="json")
+            .add_argument("attachment_ids", type=list, required=False, location="json")
+            .add_argument("retrieval_model", type=dict, required=False, location="json")
+            .add_argument("external_retrieval_model", type=dict, required=False, location="json")
+        )
+        return parser.parse_args()
+
     @staticmethod
     def perform_hit_testing(dataset, args):
         assert isinstance(current_user, Account)
         try:
             response = HitTestingService.retrieve(
                 dataset=dataset,
-                query=args["query"],
+                query=args.get("query"),
                 account=current_user,
-                retrieval_model=args["retrieval_model"],
-                external_retrieval_model=args["external_retrieval_model"],
+                retrieval_model=args.get("retrieval_model"),
+                external_retrieval_model=args.get("external_retrieval_model"),
+                attachment_ids=args.get("attachment_ids"),
                 limit=10,
             )
             return {"query": response["query"], "records": marshal(response["records"], hit_testing_record_fields)}

api/controllers/console/files.py

@@ -45,6 +45,9 @@ class FileApi(Resource):
             "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
             "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
             "workflow_file_upload_limit": dify_config.WORKFLOW_FILE_UPLOAD_LIMIT,
+            "image_file_batch_limit": dify_config.IMAGE_FILE_BATCH_LIMIT,
+            "single_chunk_attachment_limit": dify_config.SINGLE_CHUNK_ATTACHMENT_LIMIT,
+            "attachment_image_file_size_limit": dify_config.ATTACHMENT_IMAGE_FILE_SIZE_LIMIT,
         }, 200
 
     @setup_required