Feat: add s3-compatible storage boxes (#11313)

### What problem does this PR solve?

PR for implementing s3 compatible storage units #11240 

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

common/data_source/blob_connector.py

@@ -87,6 +87,13 @@ class BlobStorageConnector(LoadConnector, PollConnector):
             ):
                 raise ConnectorMissingCredentialError("Oracle Cloud Infrastructure")
 
+        elif self.bucket_type == BlobType.S3_COMPATIBLE:
+            if not all(
+                credentials.get(key)
+                for key in ["endpoint_url", "aws_access_key_id", "aws_secret_access_key"]
+            ):
+                raise ConnectorMissingCredentialError("S3 Compatible Storage")
+
         else:
             raise ValueError(f"Unsupported bucket type: {self.bucket_type}")
 

common/data_source/config.py

@@ -32,6 +32,7 @@ class BlobType(str, Enum):
     R2 = "r2"
     GOOGLE_CLOUD_STORAGE = "google_cloud_storage"
     OCI_STORAGE = "oci_storage"
+    S3_COMPATIBLE = "s3_compatible"
 
 
 class DocumentSource(str, Enum):
@@ -47,6 +48,7 @@ class DocumentSource(str, Enum):
     GOOGLE_DRIVE = "google_drive"
     GMAIL = "gmail"
     DISCORD = "discord"
+    S3_COMPATIBLE = "s3_compatible"
 
 
 class FileOrigin(str, Enum):

common/data_source/utils.py

@@ -311,6 +311,13 @@ def create_s3_client(bucket_type: BlobType, credentials: dict[str, Any], europea
             aws_secret_access_key=credentials["secret_access_key"],
             region_name=credentials["region"],
         )
+    elif bucket_type == BlobType.S3_COMPATIBLE:
+        return boto3.client(
+            "s3",
+            endpoint_url=credentials["endpoint_url"],
+            aws_access_key_id=credentials["aws_access_key_id"],
+            aws_secret_access_key=credentials["aws_secret_access_key"],
+        )    
 
     else:
         raise ValueError(f"Unsupported bucket type: {bucket_type}")

web/src/components/dynamic-form.tsx

@@ -67,6 +67,7 @@ export interface FormFieldConfig {
   ) => string | boolean | Promise<string | boolean>;
   dependencies?: string[];
   schema?: ZodSchema;
+  shouldRender?: (formValues: any) => boolean;
 }
 
 // Component props interface
@@ -654,6 +655,9 @@ const DynamicForm = {
         }
       };
 
+      // Watch all form values to re-render when they change (for shouldRender checks)
+      const formValues = form.watch();
+
       return (
         <Form {...form}>
           <form
@@ -664,11 +668,19 @@ const DynamicForm = {
             }}
           >
             <>
-              {fields.map((field) => (
-                <div key={field.name} className={cn({ hidden: field.hidden })}>
-                  {renderField(field)}
-                </div>
-              ))}
+              {fields.map((field) => {
+                const shouldShow = field.shouldRender
+                  ? field.shouldRender(formValues)
+                  : true;
+                return (
+                  <div
+                    key={field.name}
+                    className={cn({ hidden: field.hidden || !shouldShow })}
+                  >
+                    {renderField(field)}
+                  </div>
+                );
+              })}
               {children}
             </>
           </form>

web/src/locales/en.ts

@@ -705,6 +705,8 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-s
         'The base URL of your Confluence instance (e.g., https://your-domain.atlassian.net/wiki)',
       s3PrefixTip: `Specify the folder path within your S3 bucket to fetch files from. 
 Example: general/v2/`,
+      S3CompatibleEndpointUrlTip: `Required for S3 compatible Storage Box. Specify the S3-compatible endpoint URL.
+Example: https://fsn1.your-objectstorage.com`,
       addDataSourceModalTital: 'Create your {{name}} connector',
       deleteSourceModalTitle: 'Delete data source',
       deleteSourceModalContent: `

web/src/pages/user-setting/data-source/contant.tsx

@@ -105,9 +105,21 @@ export const DataSourceFormFields = {
         { label: 'R2', value: 'r2' },
         { label: 'Google Cloud Storage', value: 'google_cloud_storage' },
         { label: 'OCI Storage', value: 'oci_storage' },
+        { label: 'S3 Compatible', value: 's3_compatible' },
       ],
       required: true,
     },
+    {
+      label: 'Endpoint URL',
+      name: 'config.credentials.endpoint_url',
+      type: FormFieldType.Text,
+      required: false,
+      placeholder: 'https://fsn1.your-objectstorage.com',
+      tooltip: t('setting.S3CompatibleEndpointUrlTip'),
+      shouldRender: (formValues) => {
+        return formValues?.config?.bucket_type === 's3_compatible';
+      },
+    },
     {
       label: 'Prefix',
       name: 'config.prefix',
@@ -483,6 +495,7 @@ export const DataSourceFormDefaultValues = {
       credentials: {
         aws_access_key_id: '',
         aws_secret_access_key: '',
+        endpoint_url: '',
       },
     },
   },