Feat: An image carousel is displayed at the bottom of the agent's chat messages. #12076 (#12215)

### What problem does this PR solve?

Feat: An image carousel is displayed at the bottom of the agent's chat
messages. #12076

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/package.json

@@ -133,6 +133,7 @@
     "@storybook/addon-styling-webpack": "^2.0.0",
     "@storybook/addon-webpack5-compiler-swc": "^4.0.1",
     "@storybook/react-webpack5": "^9.1.4",
+    "@tailwindcss/container-queries": "^0.1.1",
     "@testing-library/jest-dom": "^6.4.5",
     "@testing-library/react": "^15.0.7",
     "@types/dompurify": "^3.0.5",

web/src/components/next-markdown-content/index.tsx

@@ -17,10 +17,10 @@ import { useTranslation } from 'react-i18next';
 import 'katex/dist/katex.min.css'; // `rehype-katex` does not import the CSS for you
 
 import {
+  currentReg,
   preprocessLaTeX,
   replaceTextByOldReg,
   replaceThinkToSection,
-  showImage,
 } from '@/utils/chat';
 
 import { useFetchDocumentThumbnailsByIds } from '@/hooks/use-document-request';
@@ -28,12 +28,7 @@ import { cn } from '@/lib/utils';
 import classNames from 'classnames';
 import { omit } from 'lodash';
 import { pipe } from 'lodash/fp';
-import { CircleAlert } from 'lucide-react';
+import reactStringReplace from 'react-string-replace';
-import { ImageCarousel } from '../markdown-content/image-carousel';
-import {
-  groupConsecutiveReferences,
-  shouldShowCarousel,
-} from '../markdown-content/reference-utils';
 import { Button } from '../ui/button';
 import {
   HoverCard,
@@ -42,19 +37,6 @@ import {
 } from '../ui/hover-card';
 import styles from './index.less';
 
-// Helper function to convert IReferenceObject to IReference
-const convertReferenceObjectToReference = (
-  referenceObject: IReferenceObject,
-) => {
-  const chunks = Object.values(referenceObject.chunks);
-  const docAggs = Object.values(referenceObject.doc_aggs);
-  return {
-    chunks,
-    doc_aggs: docAggs,
-    total: chunks.length,
-  };
-};
-
 const getChunkIndex = (match: string) => Number(match);
 // TODO: The display of the table is inconsistent with the display previously placed in the MessageItem.
 function MarkdownContent({
@@ -227,95 +209,26 @@ function MarkdownContent({
 
   const renderReference = useCallback(
     (text: string) => {
-      const groups = groupConsecutiveReferences(text);
+      let replacedText = reactStringReplace(text, currentReg, (match, i) => {
-      const elements = [];
+        const chunkIndex = getChunkIndex(match);
-      let lastIndex = 0;
 
-      const convertedReference = reference
+        return (
-        ? convertReferenceObjectToReference(reference)
+          <HoverCard key={i}>
-        : null;
+            <HoverCardTrigger>
-
+              <span className="text-text-secondary bg-bg-card rounded-2xl px-1 mx-1 text-nowrap">
-      groups.forEach((group, groupIndex) => {
+                Fig. {chunkIndex + 1}
-        if (group[0].start > lastIndex) {
+              </span>
-          elements.push(text.substring(lastIndex, group[0].start));
+            </HoverCardTrigger>
-        }
+            <HoverCardContent className="max-w-3xl">
-
+              {renderPopoverContent(chunkIndex)}
-        if (
+            </HoverCardContent>
-          convertedReference &&
+          </HoverCard>
-          shouldShowCarousel(group, convertedReference)
+        );
-        ) {
-          elements.push(
-            <ImageCarousel
-              key={`carousel-${groupIndex}`}
-              group={group}
-              reference={convertedReference}
-              fileThumbnails={fileThumbnails}
-              onImageClick={handleDocumentButtonClick}
-            />,
-          );
-        } else {
-          group.forEach((ref) => {
-            const chunkIndex = getChunkIndex(ref.id);
-            const {
-              documentUrl,
-              fileExtension,
-              imageId,
-              chunkItem,
-              documentId,
-            } = getReferenceInfo(chunkIndex);
-            const docType = chunkItem?.doc_type;
-
-            if (showImage(docType)) {
-              elements.push(
-                <section key={ref.id}>
-                  <Image
-                    id={imageId}
-                    className={styles.referenceInnerChunkImage}
-                    onClick={
-                      documentId
-                        ? handleDocumentButtonClick(
-                            documentId,
-                            chunkItem,
-                            fileExtension === 'pdf',
-                            documentUrl,
-                          )
-                        : () => {}
-                    }
-                  />
-                  <span className="text-accent-primary"> {imageId}</span>
-                </section>,
-              );
-            } else {
-              elements.push(
-                <HoverCard key={ref.id}>
-                  <HoverCardTrigger>
-                    <CircleAlert className="size-4 inline-block" />
-                  </HoverCardTrigger>
-                  <HoverCardContent className="max-w-3xl">
-                    {renderPopoverContent(chunkIndex)}
-                  </HoverCardContent>
-                </HoverCard>,
-              );
-            }
-          });
-        }
-
-        lastIndex = group[group.length - 1].end;
       });
 
-      if (lastIndex < text.length) {
+      return replacedText;
-        elements.push(text.substring(lastIndex));
-      }
-
-      return elements;
     },
-    [
+    [renderPopoverContent],
-      renderPopoverContent,
-      getReferenceInfo,
-      handleDocumentButtonClick,
-      reference,
-      fileThumbnails,
-    ],
   );
 
   return (

web/src/components/next-message-item/index.tsx

@@ -36,6 +36,7 @@ import { Button } from '../ui/button';
 import { AssistantGroupButton, UserGroupButton } from './group-button';
 import styles from './index.less';
 import { ReferenceDocumentList } from './reference-document-list';
+import { ReferenceImageList } from './reference-image-list';
 import { UploadedMessageFiles } from './uploaded-message-files';
 
 interface IProps
@@ -295,6 +296,13 @@ function MessageItem({
 
             {renderContent()}
 
+            {isAssistant && (
+              <ReferenceImageList
+                referenceChunks={reference?.chunks}
+                messageContent={messageContent}
+              ></ReferenceImageList>
+            )}
+
             {isAssistant && referenceDocuments.length > 0 && (
               <ReferenceDocumentList
                 list={referenceDocuments}

web/src/components/next-message-item/reference-image-list.tsx

@@ -7,22 +7,34 @@ import {
   CarouselPrevious,
 } from '@/components/ui/carousel';
 import { IReferenceChunk } from '@/interfaces/database/chat';
-import { useResponsive } from 'ahooks';
+import { isPlainObject } from 'lodash';
 import { useMemo } from 'react';
 import { extractNumbersFromMessageContent } from './utils';
 
 type IProps = {
-  referenceChunks: IReferenceChunk[];
+  referenceChunks?: IReferenceChunk[] | Record<string, IReferenceChunk>;
   messageContent: string;
 };
 
-function ImageCarousel({
+type ImageItem = {
-  imageIds,
+  id: string;
-  hideButtons,
+  index: number;
-}: {
+};
-  hideButtons?: boolean;
+
-  imageIds: string[];
+const getButtonVisibilityClass = (imageCount: number) => {
-}) {
+  const map: Record<number, string> = {
+    1: 'hidden',
+    2: '@sm:hidden',
+    3: '@md:hidden',
+    4: '@lg:hidden',
+    5: '@lg:hidden',
+  };
+  return map[imageCount] || (imageCount >= 6 ? '@2xl:hidden' : '');
+};
+
+function ImageCarousel({ images }: { images: ImageItem[] }) {
+  const buttonVisibilityClass = getButtonVisibilityClass(images.length);
+
   return (
     <Carousel
       className="w-full"
@@ -31,22 +43,27 @@ function ImageCarousel({
       }}
     >
       <CarouselContent>
-        {imageIds.map((imageId, index) => (
+        {images.map(({ id, index }) => (
-          <CarouselItem key={index} className="md:basis-1/2 2xl:basis-1/6">
+          <CarouselItem
+            key={index}
+            className="
+            basis-full
+            @sm:basis-1/2
+            @md:basis-1/3
+            @lg:basis-1/4
+            @2xl:basis-1/6
+            "
+          >
             <Image
-              id={imageId}
+              id={id}
               className="h-40 w-full"
               label={`Fig. ${(index + 1).toString()}`}
             />
           </CarouselItem>
         ))}
       </CarouselContent>
-      {!hideButtons && (
+      <CarouselPrevious className={buttonVisibilityClass} />
-        <>
+      <CarouselNext className={buttonVisibilityClass} />
-          <CarouselPrevious />
-          <CarouselNext />
-        </>
-      )}
     </Carousel>
   );
 }
@@ -55,30 +72,38 @@ export function ReferenceImageList({
   referenceChunks,
   messageContent,
 }: IProps) {
-  const imageIds = useMemo(() => {
+  const allChunkIndexes = extractNumbersFromMessageContent(messageContent);
-    return referenceChunks
+  const images = useMemo(() => {
-      .filter((_, idx) =>
+    if (Array.isArray(referenceChunks)) {
-        extractNumbersFromMessageContent(messageContent).includes(idx),
+      return referenceChunks
-      )
+        .map((chunk, idx) => ({ id: chunk.image_id, index: idx }))
-      .map((chunk) => chunk.image_id);
+        .filter((item, idx) => allChunkIndexes.includes(idx) && item.id);
-  }, [messageContent, referenceChunks]);
+    }
-  const imageCount = imageIds.length;
 
-  const responsive = useResponsive();
+    if (isPlainObject(referenceChunks)) {
+      return Object.entries(referenceChunks || {}).reduce<ImageItem[]>(
+        (pre, [idx, chunk]) => {
+          if (allChunkIndexes.includes(Number(idx)) && chunk.image_id) {
+            return pre.concat({ id: chunk.image_id, index: Number(idx) });
+          }
+          return pre;
+        },
+        [],
+      );
+    }
 
-  const { isMd, is2xl } = useMemo(() => {
+    return [];
-    return {
+  }, [allChunkIndexes, referenceChunks]);
-      isMd: responsive.md,
-      is2xl: responsive['2xl'],
-    };
-  }, [responsive]);
 
-  // If there are few images, hide the previous/next buttons.
+  const imageCount = images?.length || 0;
-  const hideButtons = is2xl ? imageCount <= 6 : isMd ? imageCount <= 2 : false;
 
   if (imageCount === 0) {
     return <></>;
   }
 
-  return <ImageCarousel imageIds={imageIds} hideButtons={hideButtons} />;
+  return (
+    <section className="@container w-full">
+      <ImageCarousel images={images} />
+    </section>
+  );
 }

web/src/pages/next-search/markdown-content/index.tsx

@@ -2,9 +2,8 @@ import Image from '@/components/image';
 import SvgIcon from '@/components/svg-icon';
 import { IReference, IReferenceChunk } from '@/interfaces/database/chat';
 import { getExtension } from '@/utils/document-util';
-import { InfoCircleOutlined } from '@ant-design/icons';
 import DOMPurify from 'dompurify';
-import React, { memo, useCallback, useEffect, useMemo } from 'react';
+import { memo, useCallback, useEffect, useMemo } from 'react';
 import Markdown from 'react-markdown';
 import SyntaxHighlighter from 'react-syntax-highlighter';
 import rehypeKatex from 'rehype-katex';
@@ -17,17 +16,11 @@ import { useTranslation } from 'react-i18next';
 
 import 'katex/dist/katex.min.css'; // `rehype-katex` does not import the CSS for you
 
-import ImageCarousel from '@/components/markdown-content/image-carousel';
-import {
-  groupConsecutiveReferences,
-  shouldShowCarousel,
-  type ReferenceGroup,
-} from '@/components/markdown-content/reference-utils';
 import {
+  currentReg,
   preprocessLaTeX,
   replaceTextByOldReg,
   replaceThinkToSection,
-  showImage,
 } from '@/utils/chat';
 
 import { Button } from '@/components/ui/button';
@@ -40,6 +33,7 @@ import { useFetchDocumentThumbnailsByIds } from '@/hooks/use-document-request';
 import classNames from 'classnames';
 import { omit } from 'lodash';
 import { pipe } from 'lodash/fp';
+import reactStringReplace from 'react-string-replace';
 
 // Defining Tailwind CSS class name constants
 const styles = {
@@ -52,6 +46,8 @@ const styles = {
   fileThumbnail: 'inline-block max-w-[40px]',
 };
 
+const getChunkIndex = (match: string) => Number(match);
+
 // TODO: The display of the table is inconsistent with the display previously placed in the MessageItem.
 const MarkdownContent = ({
   reference,
@@ -192,7 +188,10 @@ const MarkdownContent = ({
                 )}
                 <Button
                   variant="link"
-                  className={classNames(styles.documentLink, 'text-wrap')}
+                  className={classNames(
+                    styles.documentLink,
+                    'text-wrap flex-1 h-auto',
+                  )}
                   onClick={handleDocumentButtonClick(
                     documentId,
                     chunkItem,
@@ -213,83 +212,26 @@ const MarkdownContent = ({
 
   const renderReference = useCallback(
     (text: string) => {
-      const groups = groupConsecutiveReferences(text);
+      let replacedText = reactStringReplace(text, currentReg, (match) => {
-      const elements: React.ReactNode[] = [];
+        const chunkIndex = getChunkIndex(match);
-      let lastIndex = 0;
 
-      groups.forEach((group: ReferenceGroup) => {
+        return (
-        // Add text before the group
+          <Popover>
-        if (group[0].start > lastIndex) {
+            <PopoverTrigger>
-          elements.push(text.slice(lastIndex, group[0].start));
+              <span className="text-text-secondary bg-bg-card rounded-2xl px-1 mx-1 text-nowrap">
-        }
+                Fig. {chunkIndex + 1}
-
+              </span>
-        // Determine if this group should be a carousel
+            </PopoverTrigger>
-        if (shouldShowCarousel(group, reference)) {
+            <PopoverContent className="!w-fit">
-          // Render carousel for consecutive image group
+              {getPopoverContent(chunkIndex)}
-          elements.push(
+            </PopoverContent>
-            <ImageCarousel
+          </Popover>
-              key={`carousel-${group[0].id}`}
+        );
-              group={group}
-              reference={reference}
-              fileThumbnails={fileThumbnails}
-              onImageClick={handleDocumentButtonClick}
-            />,
-          );
-        } else {
-          // Render individual references in the group
-          group.forEach((ref) => {
-            const chunkIndex = Number(ref.id);
-            const { imageId, chunkItem, documentId } =
-              getReferenceInfo(chunkIndex);
-            const docType = chunkItem?.doc_type;
-
-            if (showImage(docType)) {
-              elements.push(
-                <Image
-                  key={ref.id}
-                  id={imageId}
-                  className={styles.referenceInnerChunkImage}
-                  onClick={
-                    documentId
-                      ? handleDocumentButtonClick(documentId, chunkItem)
-                      : () => {}
-                  }
-                />,
-              );
-            } else {
-              elements.push(
-                <Popover key={ref.id}>
-                  <PopoverTrigger>
-                    <InfoCircleOutlined className={styles.referenceIcon} />
-                  </PopoverTrigger>
-                  <PopoverContent className="!w-fit">
-                    {getPopoverContent(chunkIndex)}
-                  </PopoverContent>
-                </Popover>,
-              );
-            }
-            // Add the original reference text
-            elements.push(ref.fullMatch);
-          });
-        }
-
-        lastIndex = group[group.length - 1].end;
       });
 
-      // Add any remaining text after the last group
+      return replacedText;
-      if (lastIndex < text.length) {
-        elements.push(text.slice(lastIndex));
-      }
-
-      return elements;
     },
-    [
+    [getPopoverContent],
-      reference,
-      fileThumbnails,
-      handleDocumentButtonClick,
-      getReferenceInfo,
-      getPopoverContent,
-    ],
   );
 
   return (
@@ -300,11 +242,7 @@ const MarkdownContent = ({
       components={
         {
           'custom-typography': ({ children }: { children: string }) =>
-            Array.isArray(renderReference(children))
+            renderReference(children),
-              ? renderReference(children).map((element, index) => (
-                  <React.Fragment key={index}>{element}</React.Fragment>
-                ))
-              : renderReference(children),
           code(props: any) {
             const { children, className, ...rest } = props;
             const restProps = omit(rest, 'node');

web/tailwind.config.js

@@ -223,5 +223,6 @@ module.exports = {
     require('tailwindcss-animate'),
     require('@tailwindcss/line-clamp'),
     require('tailwind-scrollbar'),
+    require('@tailwindcss/container-queries'),
   ],
 };