Fix: add describe_image_with_prompt for ZHIPU AI (#11317)

### What problem does this PR solve?

Fix: add describe_image_with_prompt for ZHIPU AI  #11289 

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

admin/client/admin_client.py

@@ -378,7 +378,7 @@ class AdminCLI(Cmd):
                         self.session.headers.update({
                             'Content-Type': 'application/json',
                             'Authorization': response.headers['Authorization'],
-                            'User-Agent': 'RAGFlow-CLI/0.22.0'
+                            'User-Agent': 'RAGFlow-CLI/0.22.1'
                         })
                         print("Authentication successful.")
                         return True

admin/client/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ragflow-cli"
-version = "0.22.0"
+version = "0.22.1"
 description = "Admin Service's client of [RAGFlow](https://github.com/infiniflow/ragflow). The Admin Service provides user management and system monitoring. "
 authors = [{ name = "Lynn", email = "lynn_inf@hotmail.com" }]
 license = { text = "Apache License, Version 2.0" }

api/apps/kb_app.py

@@ -918,6 +918,6 @@ def check_embedding():
     }
     if summary["avg_cos_sim"] > 0.9:
         return get_json_result(data={"summary": summary, "results": results})
-    return get_json_result(code=RetCode.NOT_EFFECTIVE, message="failed", data={"summary": summary, "results": results})
+    return get_json_result(code=RetCode.NOT_EFFECTIVE, message="Embedding model switch failed: the average similarity between old and new vectors is below 0.9, indicating incompatible vector spaces.", data={"summary": summary, "results": results})
 
 

common/data_source/blob_connector.py

@@ -87,6 +87,13 @@ class BlobStorageConnector(LoadConnector, PollConnector):
             ):
                 raise ConnectorMissingCredentialError("Oracle Cloud Infrastructure")
 
+        elif self.bucket_type == BlobType.S3_COMPATIBLE:
+            if not all(
+                credentials.get(key)
+                for key in ["endpoint_url", "aws_access_key_id", "aws_secret_access_key"]
+            ):
+                raise ConnectorMissingCredentialError("S3 Compatible Storage")
+
         else:
             raise ValueError(f"Unsupported bucket type: {self.bucket_type}")
 

common/data_source/config.py

@@ -32,6 +32,7 @@ class BlobType(str, Enum):
     R2 = "r2"
     GOOGLE_CLOUD_STORAGE = "google_cloud_storage"
     OCI_STORAGE = "oci_storage"
+    S3_COMPATIBLE = "s3_compatible"
 
 
 class DocumentSource(str, Enum):
@@ -47,6 +48,7 @@ class DocumentSource(str, Enum):
     GOOGLE_DRIVE = "google_drive"
     GMAIL = "gmail"
     DISCORD = "discord"
+    S3_COMPATIBLE = "s3_compatible"
 
 
 class FileOrigin(str, Enum):

common/data_source/utils.py

@@ -311,6 +311,13 @@ def create_s3_client(bucket_type: BlobType, credentials: dict[str, Any], europea
             aws_secret_access_key=credentials["secret_access_key"],
             region_name=credentials["region"],
         )
+    elif bucket_type == BlobType.S3_COMPATIBLE:
+        return boto3.client(
+            "s3",
+            endpoint_url=credentials["endpoint_url"],
+            aws_access_key_id=credentials["aws_access_key_id"],
+            aws_secret_access_key=credentials["aws_secret_access_key"],
+        )    
 
     else:
         raise ValueError(f"Unsupported bucket type: {bucket_type}")

docker/entrypoint.sh

@@ -71,7 +71,7 @@ for arg in "$@"; do
       ENABLE_TASKEXECUTOR=0
       shift
       ;;
-    --disable-datasyn)
+    --disable-datasync)
       ENABLE_DATASYNC=0
       shift
       ;;

rag/llm/cv_model.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 #
 
+import re
 import base64
 import json
 import os
@@ -32,7 +33,6 @@ from rag.nlp import is_english
 from rag.prompts.generator import vision_llm_describe_prompt
 from common.token_utils import num_tokens_from_string, total_token_count_from_response
 
-
 class Base(ABC):
     def __init__(self, **kwargs):
         # Configure retry parameters
@@ -208,6 +208,7 @@ class GptV4(Base):
             model=self.model_name,
             messages=self.prompt(b64),
             extra_body=self.extra_body,
+            unused = None,
         )
         return res.choices[0].message.content.strip(), total_token_count_from_response(res)
 
@@ -324,6 +325,122 @@ class Zhipu4V(GptV4):
         Base.__init__(self, **kwargs)
 
 
+    def _clean_conf(self, gen_conf):
+        if "max_tokens" in gen_conf:
+            del gen_conf["max_tokens"]
+        gen_conf = self._clean_conf_plealty(gen_conf)
+        return gen_conf
+
+
+    def _clean_conf_plealty(self, gen_conf):
+        if "presence_penalty" in gen_conf:
+            del gen_conf["presence_penalty"]
+        if "frequency_penalty" in gen_conf:
+            del gen_conf["frequency_penalty"]
+        return gen_conf
+
+
+    def _request(self, msg, stream, gen_conf={}):
+        response = requests.post(
+            self.base_url,
+            json={
+                "model": self.model_name,
+                "messages": msg,
+                "stream": stream,
+                **gen_conf
+            },
+            headers= {
+            "Authorization": f"Bearer {self.api_key}",  
+            "Content-Type": "application/json",
+            }
+        )
+        return response.json()
+
+
+    def chat(self, system, history, gen_conf, images=None, stream=False, **kwargs):
+        if system and history and history[0].get("role") != "system":
+            history.insert(0, {"role": "system", "content": system})
+
+        gen_conf = self._clean_conf(gen_conf)
+
+        logging.info(json.dumps(history, ensure_ascii=False, indent=2))
+        response = self.client.chat.completions.create(model=self.model_name, messages=self._form_history(system, history, images), stream=False, **gen_conf)
+        content = response.choices[0].message.content.strip()
+
+        cleaned = re.sub(r"<\|(begin_of_box|end_of_box)\|>", "", content).strip()
+        return cleaned, total_token_count_from_response(response)
+    
+
+    def chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
+        from rag.llm.chat_model import LENGTH_NOTIFICATION_CN, LENGTH_NOTIFICATION_EN 
+        from rag.nlp import is_chinese
+
+        if system and history and history[0].get("role") != "system":
+            history.insert(0, {"role": "system", "content": system})
+        gen_conf = self._clean_conf(gen_conf)
+        ans = ""
+        tk_count = 0
+        try:
+            logging.info(json.dumps(history, ensure_ascii=False, indent=2))
+            response = self.client.chat.completions.create(model=self.model_name, messages=self._form_history(system, history, images), stream=True, **gen_conf)
+            for resp in response:
+                if not resp.choices[0].delta.content:
+                    continue
+                delta = resp.choices[0].delta.content
+                ans = delta
+                if resp.choices[0].finish_reason == "length":
+                    if is_chinese(ans):
+                        ans += LENGTH_NOTIFICATION_CN
+                    else:
+                        ans += LENGTH_NOTIFICATION_EN
+                    tk_count = total_token_count_from_response(resp)
+                if resp.choices[0].finish_reason == "stop":
+                    tk_count = total_token_count_from_response(resp)
+                yield ans
+        except Exception as e:
+            yield ans + "\n**ERROR**: " + str(e)
+
+        yield tk_count
+
+
+    def describe(self, image):
+        return self.describe_with_prompt(image)
+
+
+    def describe_with_prompt(self, image, prompt=None):
+        b64 = self.image2base64(image)
+        if prompt is None:
+            prompt = "Describe this image."
+
+        # Chat messages
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image_url",
+                        "image_url": { "url": b64 }
+                    },
+                    {
+                        "type": "text",
+                        "text": prompt
+                    }
+                ]
+            }
+        ]
+
+        resp = self.client.chat.completions.create(
+            model=self.model_name,
+            messages=messages,
+            stream=False
+        )
+
+        content = resp.choices[0].message.content.strip()
+        cleaned = re.sub(r"<\|(begin_of_box|end_of_box)\|>", "", content).strip()
+
+        return cleaned, num_tokens_from_string(cleaned)
+    
+
 class StepFunCV(GptV4):
     _FACTORY_NAME = "StepFun"
 

web/src/components/dynamic-form.tsx

@@ -67,6 +67,7 @@ export interface FormFieldConfig {
   ) => string | boolean | Promise<string | boolean>;
   dependencies?: string[];
   schema?: ZodSchema;
+  shouldRender?: (formValues: any) => boolean;
 }
 
 // Component props interface
@@ -654,6 +655,9 @@ const DynamicForm = {
         }
       };
 
+      // Watch all form values to re-render when they change (for shouldRender checks)
+      const formValues = form.watch();
+
       return (
         <Form {...form}>
           <form
@@ -664,11 +668,19 @@ const DynamicForm = {
             }}
           >
             <>
-              {fields.map((field) => (
-                <div key={field.name} className={cn({ hidden: field.hidden })}>
-                  {renderField(field)}
-                </div>
-              ))}
+              {fields.map((field) => {
+                const shouldShow = field.shouldRender
+                  ? field.shouldRender(formValues)
+                  : true;
+                return (
+                  <div
+                    key={field.name}
+                    className={cn({ hidden: field.hidden || !shouldShow })}
+                  >
+                    {renderField(field)}
+                  </div>
+                );
+              })}
               {children}
             </>
           </form>

web/src/constants/agent.tsx

@@ -169,3 +169,13 @@ export const SwitchOperatorOptions = [
     icon: <CircleSlash2 className="size-4" />,
   },
 ];
+
+export const AgentStructuredOutputField = 'structured';
+
+export enum JsonSchemaDataType {
+  String = 'string',
+  Number = 'number',
+  Boolean = 'boolean',
+  Array = 'array',
+  Object = 'object',
+}

web/src/locales/en.ts

@@ -705,6 +705,8 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-s
         'The base URL of your Confluence instance (e.g., https://your-domain.atlassian.net/wiki)',
       s3PrefixTip: `Specify the folder path within your S3 bucket to fetch files from. 
 Example: general/v2/`,
+      S3CompatibleEndpointUrlTip: `Required for S3 compatible Storage Box. Specify the S3-compatible endpoint URL.
+Example: https://fsn1.your-objectstorage.com`,
       addDataSourceModalTital: 'Create your {{name}} connector',
       deleteSourceModalTitle: 'Delete data source',
       deleteSourceModalContent: `

web/src/pages/agent/constant/index.tsx

@@ -5,6 +5,7 @@ import {
 import {
   AgentGlobals,
   AgentGlobalsSysQueryWithBrace,
+  AgentStructuredOutputField,
   CodeTemplateStrMap,
   ComparisonOperator,
   Operator,
@@ -12,7 +13,11 @@ import {
   SwitchOperatorOptions,
   initialLlmBaseValues,
 } from '@/constants/agent';
-export { Operator } from '@/constants/agent';
+export {
+  AgentStructuredOutputField,
+  JsonSchemaDataType,
+  Operator,
+} from '@/constants/agent';
 
 export * from './pipeline';
 
@@ -441,8 +446,6 @@ export const initialCodeValues = {
 
 export const initialWaitingDialogueValues = {};
 
-export const AgentStructuredOutputField = 'structured';
-
 export const initialAgentValues = {
   ...initialLlmBaseValues,
   description: '',
@@ -839,14 +842,6 @@ export const DROPDOWN_HORIZONTAL_OFFSET = 28;
 export const DROPDOWN_VERTICAL_OFFSET = 74;
 export const PREVENT_CLOSE_DELAY = 300;
 
-export enum JsonSchemaDataType {
-  String = 'string',
-  Number = 'number',
-  Boolean = 'boolean',
-  Array = 'array',
-  Object = 'object',
-}
-
 export enum VariableAssignerLogicalOperator {
   Overwrite = 'overwrite',
   Clear = 'clear',

web/src/pages/agent/form/variable-assigner-form/dynamic-variables.tsx

@@ -7,7 +7,7 @@ import { Label } from '@/components/ui/label';
 import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group';
 import { Separator } from '@/components/ui/separator';
 import { Textarea } from '@/components/ui/textarea';
-import { Editor } from '@monaco-editor/react';
+import Editor, { loader } from '@monaco-editor/react';
 import * as RadioGroupPrimitive from '@radix-ui/react-radio-group';
 import { X } from 'lucide-react';
 import { ReactNode, useCallback } from 'react';
@@ -23,6 +23,8 @@ import { DynamicFormHeader } from '../components/dynamic-fom-header';
 import { QueryVariable } from '../components/query-variable';
 import { useBuildLogicalOptions } from './use-build-logical-options';
 
+loader.config({ paths: { vs: '/vs' } });
+
 type SelectKeysProps = {
   name: string;
   label: ReactNode;
@@ -70,7 +72,7 @@ const EmptyValueMap = {
   [JsonSchemaDataType.String]: '',
   [JsonSchemaDataType.Number]: 0,
   [JsonSchemaDataType.Boolean]: 'yes',
-  [JsonSchemaDataType.Object]: {},
+  [JsonSchemaDataType.Object]: '{}',
   [JsonSchemaDataType.Array]: [],
 };
 
@@ -86,7 +88,7 @@ export function DynamicVariables({
   const { getType } = useGetVariableLabelOrTypeByValue();
   const isDarkTheme = useIsDarkTheme();
 
-  const { fields, remove, append, update } = useFieldArray({
+  const { fields, remove, append } = useFieldArray({
     name: name,
     control: form.control,
   });
@@ -102,15 +104,7 @@ export function DynamicVariables({
   );
 
   const renderParameter = useCallback(
-    (
-      keyFieldName: string,
-      operatorFieldName: string,
-      valueFieldAlias: string,
-    ) => {
-      console.log(
-        '🚀 ~ DynamicVariables ~ valueFieldAlias:',
-        form.getValues(valueFieldAlias),
-      );
+    (keyFieldName: string, operatorFieldName: string) => {
       const logicalOperator = form.getValues(operatorFieldName);
       const type = getVariableType(keyFieldName);
 
@@ -169,10 +163,6 @@ export function DynamicVariables({
 
   const handleVariableChange = useCallback(
     (operatorFieldAlias: string, valueFieldAlias: string) => {
-      console.log(
-        '🚀 ~ DynamicVariables ~ operatorFieldAlias:',
-        operatorFieldAlias,
-      );
       return () => {
         form.setValue(
           operatorFieldAlias,
@@ -190,14 +180,8 @@ export function DynamicVariables({
   );
 
   const handleOperatorChange = useCallback(
-    (
-      valueFieldAlias: string,
-      keyFieldAlias: string,
-      value: string,
-      index: number,
-    ) => {
+    (valueFieldAlias: string, keyFieldAlias: string, value: string) => {
       const type = getVariableType(keyFieldAlias);
-      console.log('🚀 ~ DynamicVariables ~ type:', type);
 
       let parameter = EmptyValueMap[type as keyof typeof EmptyValueMap];
 
@@ -210,10 +194,6 @@ export function DynamicVariables({
           shouldDirty: true,
           shouldValidate: true,
         });
-
-        // form.trigger(valueFieldAlias);
-
-        // update(index, { [valueField]: parameter });
       }
     },
     [form, getVariableType],
@@ -258,7 +238,6 @@ export function DynamicVariables({
                             valueFieldAlias,
                             keyFieldAlias,
                             val,
-                            index,
                           );
                           onChange(val);
                         }}
@@ -270,11 +249,7 @@ export function DynamicVariables({
                   </RAGFlowFormItem>
                 </div>
                 <RAGFlowFormItem name={valueFieldAlias} className="w-full">
-                  {renderParameter(
-                    keyFieldAlias,
-                    operatorFieldAlias,
-                    valueFieldAlias,
-                  )}
+                  {renderParameter(keyFieldAlias, operatorFieldAlias)}
                 </RAGFlowFormItem>
               </div>
 

web/src/pages/agent/form/variable-assigner-form/index.tsx

@@ -33,7 +33,6 @@ function VariableAssignerForm({ node }: INextOperatorForm) {
     defaultValues: defaultValues,
     mode: 'onChange',
     resolver: zodResolver(FormSchema),
-    shouldUnregister: true,
   });
 
   useWatchFormChange(node?.id, form, true);

web/src/pages/agent/gobal-variable-sheet/hooks/use-form.tsx

@@ -15,27 +15,30 @@ export const useHandleForm = () => {
       return value;
     }
   };
-  const handleSubmit = useCallback(async (fieldValue: FieldValues) => {
-    const param = {
-      ...(data.dsl?.variables || {}),
-      [fieldValue.name]: {
-        ...fieldValue,
-        value:
-          fieldValue.type === TypesWithArray.Object ||
-          fieldValue.type === TypesWithArray.ArrayObject
-            ? handleObjectData(fieldValue.value)
-            : fieldValue.value,
-      },
-    } as Record<string, GlobalVariableType>;
+  const handleSubmit = useCallback(
+    async (fieldValue: FieldValues) => {
+      const param = {
+        ...(data.dsl?.variables || {}),
+        [fieldValue.name]: {
+          ...fieldValue,
+          value:
+            fieldValue.type === TypesWithArray.Object ||
+            fieldValue.type === TypesWithArray.ArrayObject
+              ? handleObjectData(fieldValue.value)
+              : fieldValue.value,
+        },
+      } as Record<string, GlobalVariableType>;
 
-    const res = await saveGraph(undefined, {
-      globalVariables: param,
-    });
+      const res = await saveGraph(undefined, {
+        globalVariables: param,
+      });
 
-    if (res.code === 0) {
-      refetch();
-    }
-  }, []);
+      if (res.code === 0) {
+        refetch();
+      }
+    },
+    [data.dsl?.variables, refetch, saveGraph],
+  );
 
   return { handleSubmit, loading };
 };

web/src/pages/agent/hooks/use-get-begin-query.tsx

@@ -1,7 +1,7 @@
-import { AgentGlobals } from '@/constants/agent';
+import { AgentGlobals, AgentStructuredOutputField } from '@/constants/agent';
 import { useFetchAgent } from '@/hooks/use-agent-request';
 import { RAGFlowNodeType } from '@/interfaces/database/flow';
-import { buildNodeOutputOptions } from '@/utils/canvas-util';
+import { buildNodeOutputOptions, isAgentStructured } from '@/utils/canvas-util';
 import { DefaultOptionType } from 'antd/es/select';
 import { t } from 'i18next';
 import { isEmpty, toLower } from 'lodash';
@@ -236,7 +236,11 @@ export function useFilterQueryVariableOptionsByTypes(
                   toLower(x).startsWith('array')
                     ? toLower(y.type).includes(toLower(x))
                     : toLower(y.type) === toLower(x),
-                ) || y.type === undefined, // agent structured output
+                ) ||
+                isAgentStructured(
+                  y.value,
+                  y.value.slice(-AgentStructuredOutputField.length),
+                ), // agent structured output
             ),
           };
         })

web/src/pages/user-setting/data-source/contant.tsx

@@ -105,9 +105,21 @@ export const DataSourceFormFields = {
         { label: 'R2', value: 'r2' },
         { label: 'Google Cloud Storage', value: 'google_cloud_storage' },
         { label: 'OCI Storage', value: 'oci_storage' },
+        { label: 'S3 Compatible', value: 's3_compatible' },
       ],
       required: true,
     },
+    {
+      label: 'Endpoint URL',
+      name: 'config.credentials.endpoint_url',
+      type: FormFieldType.Text,
+      required: false,
+      placeholder: 'https://fsn1.your-objectstorage.com',
+      tooltip: t('setting.S3CompatibleEndpointUrlTip'),
+      shouldRender: (formValues) => {
+        return formValues?.config?.bucket_type === 's3_compatible';
+      },
+    },
     {
       label: 'Prefix',
       name: 'config.prefix',
@@ -483,6 +495,7 @@ export const DataSourceFormDefaultValues = {
       credentials: {
         aws_access_key_id: '',
         aws_secret_access_key: '',
+        endpoint_url: '',
       },
     },
   },

web/src/utils/canvas-util.tsx

@@ -1,4 +1,10 @@
+import {
+  AgentStructuredOutputField,
+  JsonSchemaDataType,
+  Operator,
+} from '@/constants/agent';
 import { BaseNode } from '@/interfaces/database/agent';
+
 import { Edge } from '@xyflow/react';
 import { isEmpty } from 'lodash';
 import { ComponentType, ReactNode } from 'react';
@@ -23,6 +29,12 @@ export function filterAllUpstreamNodeIds(edges: Edge[], nodeIds: string[]) {
   }, []);
 }
 
+export function isAgentStructured(id?: string, label?: string) {
+  return (
+    label === AgentStructuredOutputField && id?.startsWith(`${Operator.Agent}:`)
+  );
+}
+
 export function buildOutputOptions(
   outputs: Record<string, any> = {},
   nodeId?: string,
@@ -34,7 +46,9 @@ export function buildOutputOptions(
     value: `${nodeId}@${x}`,
     parentLabel,
     icon,
-    type: outputs[x]?.type,
+    type: isAgentStructured(nodeId, x)
+      ? JsonSchemaDataType.Object
+      : outputs[x]?.type,
   }));
 }