Feat: Images referenced in chat messages are displayed as a carousel at the bottom of the message. #12076 (#12207)

### What problem does this PR solve?
Feat: Images referenced in chat messages are displayed as a carousel at
the bottom of the message. #12076

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/src/app.tsx

@@ -24,6 +24,18 @@ import { TooltipProvider } from './components/ui/tooltip';
 import { ThemeEnum } from './constants/common';
 import storage from './utils/authorization-util';
 
+import { configResponsive } from 'ahooks';
+
+configResponsive({
+  sm: 640,
+  md: 768,
+  lg: 1024,
+  xl: 1280,
+  '2xl': 1536,
+  '3xl': 1780,
+  '4xl': 1980,
+});
+
 dayjs.extend(customParseFormat);
 dayjs.extend(advancedFormat);
 dayjs.extend(weekday);

web/src/components/image/index.tsx

@@ -6,17 +6,30 @@ import { Popover, PopoverContent, PopoverTrigger } from '../ui/popover';
 interface IImage extends React.ImgHTMLAttributes<HTMLImageElement> {
   id: string;
   t?: string | number;
+  label?: string;
 }
 
-const Image = ({ id, t, className, ...props }: IImage) => {
+const Image = ({ id, t, label, className, ...props }: IImage) => {
-  return (
+  const imageElement = (
     <img
       {...props}
       src={`${api_host}/document/image/${id}${t ? `?_t=${t}` : ''}`}
-      alt=""
       className={classNames('max-w-[45vw] max-h-[40wh] block', className)}
     />
   );
+
+  if (!label) {
+    return imageElement;
+  }
+
+  return (
+    <div className="relative inline-block w-full">
+      {imageElement}
+      <div className="absolute bottom-2 right-2 bg-accent-primary text-white px-2 py-0.5 rounded-xl text-xs font-normal backdrop-blur-sm">
+        {label}
+      </div>
+    </div>
+  );
 };
 
 export default Image;

web/src/components/markdown-content/index.tsx

@@ -18,27 +18,22 @@ import 'katex/dist/katex.min.css'; // `rehype-katex` does not import the CSS for
 
 import { useFetchDocumentThumbnailsByIds } from '@/hooks/use-document-request';
 import {
+  currentReg,
   preprocessLaTeX,
   replaceTextByOldReg,
   replaceThinkToSection,
-  showImage,
 } from '@/utils/chat';
 import classNames from 'classnames';
 import { omit } from 'lodash';
 import { pipe } from 'lodash/fp';
-import { CircleAlert } from 'lucide-react';
+import reactStringReplace from 'react-string-replace';
 import { Button } from '../ui/button';
 import {
   HoverCard,
   HoverCardContent,
   HoverCardTrigger,
 } from '../ui/hover-card';
-import { ImageCarousel } from './image-carousel';
 import styles from './index.less';
-import {
-  groupConsecutiveReferences,
-  shouldShowCarousel,
-} from './reference-utils';
 
 const getChunkIndex = (match: string) => Number(match);
 
@@ -191,7 +186,7 @@ const MarkdownContent = ({
                 )}
                 <Button
                   variant="link"
-                  className={'text-wrap p-0'}
+                  className={'text-wrap p-0 flex-1 h-auto'}
                   onClick={handleDocumentButtonClick(
                     documentId,
                     chunkItem,
@@ -212,88 +207,26 @@ const MarkdownContent = ({
 
   const renderReference = useCallback(
     (text: string) => {
-      const groups = groupConsecutiveReferences(text);
+      let replacedText = reactStringReplace(text, currentReg, (match, i) => {
-      const elements = [];
+        const chunkIndex = getChunkIndex(match);
-      let lastIndex = 0;
 
-      groups.forEach((group, groupIndex) => {
+        return (
-        if (group[0].start > lastIndex) {
+          <HoverCard key={i}>
-          elements.push(text.substring(lastIndex, group[0].start));
+            <HoverCardTrigger>
-        }
+              <span className="text-text-secondary bg-bg-card rounded-2xl px-1 mx-1">
-
+                Fig. {chunkIndex + 1}
-        if (shouldShowCarousel(group, reference)) {
+              </span>
-          elements.push(
+            </HoverCardTrigger>
-            <ImageCarousel
+            <HoverCardContent className="max-w-3xl">
-              key={`carousel-${groupIndex}`}
+              {getPopoverContent(chunkIndex)}
-              group={group}
+            </HoverCardContent>
-              reference={reference}
+          </HoverCard>
-              fileThumbnails={fileThumbnails}
+        );
-              onImageClick={handleDocumentButtonClick}
-            />,
-          );
-        } else {
-          group.forEach((ref) => {
-            const chunkIndex = getChunkIndex(ref.id);
-            const {
-              documentUrl,
-              fileExtension,
-              imageId,
-              chunkItem,
-              documentId,
-            } = getReferenceInfo(chunkIndex);
-            const docType = chunkItem?.doc_type;
-
-            if (showImage(docType)) {
-              elements.push(
-                <section key={ref.id}>
-                  <Image
-                    id={imageId}
-                    className={styles.referenceInnerChunkImage}
-                    onClick={
-                      documentId
-                        ? handleDocumentButtonClick(
-                            documentId,
-                            chunkItem,
-                            fileExtension === 'pdf',
-                            documentUrl,
-                          )
-                        : () => {}
-                    }
-                  />
-                  <span className="text-accent-primary"> {imageId}</span>
-                </section>,
-              );
-            } else {
-              elements.push(
-                <HoverCard key={ref.id}>
-                  <HoverCardTrigger>
-                    <CircleAlert className="size-4 inline-block" />
-                  </HoverCardTrigger>
-                  <HoverCardContent className="max-w-3xl">
-                    {getPopoverContent(chunkIndex)}
-                  </HoverCardContent>
-                </HoverCard>,
-              );
-            }
-          });
-        }
-
-        lastIndex = group[group.length - 1].end;
       });
 
-      if (lastIndex < text.length) {
+      return replacedText;
-        elements.push(text.substring(lastIndex));
-      }
-
-      return elements;
     },
-    [
+    [getPopoverContent],
-      getPopoverContent,
-      getReferenceInfo,
-      handleDocumentButtonClick,
-      reference,
-      fileThumbnails,
-    ],
   );
 
   return (

web/src/components/message-item/index.tsx

@@ -13,6 +13,7 @@ import { IRegenerateMessage, IRemoveMessageById } from '@/hooks/logic-hooks';
 import { cn } from '@/lib/utils';
 import MarkdownContent from '../markdown-content';
 import { ReferenceDocumentList } from '../next-message-item/reference-document-list';
+import { ReferenceImageList } from '../next-message-item/reference-image-list';
 import { UploadedMessageFiles } from '../next-message-item/uploaded-message-files';
 import {
   PDFDownloadButton,
@@ -140,7 +141,6 @@ const MessageItem = ({
                 sendLoading={sendLoading}
               ></UserGroupButton>
             )}
-
             {/* Show PDF download button if download info is present */}
             {pdfDownloadInfo && (
               <PDFDownloadButton
@@ -148,7 +148,6 @@ const MessageItem = ({
                 className="mb-2"
               />
             )}
-
             {/* Show message content if there's any text besides the download */}
             {messageContent && (
               <div
@@ -169,6 +168,12 @@ const MessageItem = ({
                 ></MarkdownContent>
               </div>
             )}
+            {isAssistant && (
+              <ReferenceImageList
+                referenceChunks={reference.chunks}
+                messageContent={messageContent}
+              ></ReferenceImageList>
+            )}
             {isAssistant && referenceDocumentList.length > 0 && (
               <ReferenceDocumentList
                 list={referenceDocumentList}

web/src/components/next-message-item/reference-document-list.tsx

@@ -1,5 +1,6 @@
 import { Card, CardContent } from '@/components/ui/card';
 import { Docagg } from '@/interfaces/database/chat';
+import { middleEllipsis } from '@/utils/common-util';
 import FileIcon from '../file-icon';
 import NewDocumentLink from '../new-document-link';
 
@@ -17,7 +18,7 @@ export function ReferenceDocumentList({ list }: { list: Docagg[] }) {
               link={item.url}
               className="text-text-sub-title-invert"
             >
-              {item.doc_name}
+              {middleEllipsis(item.doc_name)}
             </NewDocumentLink>
           </CardContent>
         </Card>

web/src/components/next-message-item/reference-image-list.tsx

@@ -0,0 +1,84 @@
+import Image from '@/components/image';
+import {
+  Carousel,
+  CarouselContent,
+  CarouselItem,
+  CarouselNext,
+  CarouselPrevious,
+} from '@/components/ui/carousel';
+import { IReferenceChunk } from '@/interfaces/database/chat';
+import { useResponsive } from 'ahooks';
+import { useMemo } from 'react';
+import { extractNumbersFromMessageContent } from './utils';
+
+type IProps = {
+  referenceChunks: IReferenceChunk[];
+  messageContent: string;
+};
+
+function ImageCarousel({
+  imageIds,
+  hideButtons,
+}: {
+  hideButtons?: boolean;
+  imageIds: string[];
+}) {
+  return (
+    <Carousel
+      className="w-full"
+      opts={{
+        align: 'start',
+      }}
+    >
+      <CarouselContent>
+        {imageIds.map((imageId, index) => (
+          <CarouselItem key={index} className="md:basis-1/2 2xl:basis-1/6">
+            <Image
+              id={imageId}
+              className="h-40 w-full"
+              label={`Fig. ${(index + 1).toString()}`}
+            />
+          </CarouselItem>
+        ))}
+      </CarouselContent>
+      {!hideButtons && (
+        <>
+          <CarouselPrevious />
+          <CarouselNext />
+        </>
+      )}
+    </Carousel>
+  );
+}
+
+export function ReferenceImageList({
+  referenceChunks,
+  messageContent,
+}: IProps) {
+  const imageIds = useMemo(() => {
+    return referenceChunks
+      .filter((_, idx) =>
+        extractNumbersFromMessageContent(messageContent).includes(idx),
+      )
+      .map((chunk) => chunk.image_id);
+  }, [messageContent, referenceChunks]);
+  const imageCount = imageIds.length;
+
+  const responsive = useResponsive();
+
+  const { isMd, is2xl } = useMemo(() => {
+    return {
+      isMd: responsive.md,
+      is2xl: responsive['2xl'],
+    };
+  }, [responsive]);
+
+  // If there are few images, hide the previous/next buttons.
+  const hideButtons = is2xl ? imageCount <= 6 : isMd ? imageCount <= 2 : false;
+
+  if (imageCount === 0) {
+    return <></>;
+  }
+
+  return <ImageCarousel imageIds={imageIds} hideButtons={hideButtons} />;
+}

web/src/components/next-message-item/utils.ts

@@ -0,0 +1,16 @@
+import { currentReg } from '@/utils/chat';
+
+export const extractNumbersFromMessageContent = (content: string) => {
+  const matches = content.match(currentReg);
+  if (matches) {
+    const list = matches
+      .map((match) => {
+        const numMatch = match.match(/\[ID:(\d+)\]/);
+        return numMatch ? parseInt(numMatch[1], 10) : null;
+      })
+      .filter((num) => num !== null) as number[];
+
+    return list;
+  }
+  return [];
+};

web/src/utils/common-util.ts

@@ -252,3 +252,8 @@ export function parseColorToRGBA(color: string, opcity = 1): string {
   const [r, g, b] = parseColorToRGB(color);
   return `rgba(${r},${g},${b},${opcity})`;
 }
+
+export function middleEllipsis(str: string, front = 12, back = 8) {
+  if (str.length <= front + back) return str;
+  return `${str.slice(0, front)}…${str.slice(-back)}`;
+}