Feat: Images appearing consecutively in the dialogue are merged and displayed in a carousel. #10427 (#12051)

### What problem does this PR solve?

Feat: Images appearing consecutively in the dialogue are merged and
displayed in a carousel. #10427
### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/package-lock.json

@@ -59,6 +59,7 @@
         "cmdk": "^1.0.4",
         "dayjs": "^1.11.10",
         "dompurify": "^3.1.6",
+        "embla-carousel-react": "^8.6.0",
         "eventsource-parser": "^1.1.2",
         "human-id": "^4.1.1",
         "i18next": "^23.7.16",
@@ -17517,6 +17518,34 @@
       "resolved": "https://registry.npmmirror.com/bn.js/-/bn.js-4.12.0.tgz",
       "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA=="
     },
+    "node_modules/embla-carousel": {
+      "version": "8.6.0",
+      "resolved": "https://registry.npmmirror.com/embla-carousel/-/embla-carousel-8.6.0.tgz",
+      "integrity": "sha512-SjWyZBHJPbqxHOzckOfo8lHisEaJWmwd23XppYFYVh10bU66/Pn5tkVkbkCMZVdbUE5eTCI2nD8OyIP4Z+uwkA==",
+      "license": "MIT"
+    },
+    "node_modules/embla-carousel-react": {
+      "version": "8.6.0",
+      "resolved": "https://registry.npmmirror.com/embla-carousel-react/-/embla-carousel-react-8.6.0.tgz",
+      "integrity": "sha512-0/PjqU7geVmo6F734pmPqpyHqiM99olvyecY7zdweCw+6tKEXnrE90pBiBbMMU8s5tICemzpQ3hi5EpxzGW+JA==",
+      "license": "MIT",
+      "dependencies": {
+        "embla-carousel": "8.6.0",
+        "embla-carousel-reactive-utils": "8.6.0"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.1 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc"
+      }
+    },
+    "node_modules/embla-carousel-reactive-utils": {
+      "version": "8.6.0",
+      "resolved": "https://registry.npmmirror.com/embla-carousel-reactive-utils/-/embla-carousel-reactive-utils-8.6.0.tgz",
+      "integrity": "sha512-fMVUDUEx0/uIEDM0Mz3dHznDhfX+znCCDCeIophYb1QGVM7YThSWX+wz11zlYwWFOr74b4QLGg0hrGPJeG2s4A==",
+      "license": "MIT",
+      "peerDependencies": {
+        "embla-carousel": "8.6.0"
+      }
+    },
     "node_modules/emittery": {
       "version": "0.13.1",
       "resolved": "https://registry.npmmirror.com/emittery/-/emittery-0.13.1.tgz",

web/package.json

@@ -72,6 +72,7 @@
     "cmdk": "^1.0.4",
     "dayjs": "^1.11.10",
     "dompurify": "^3.1.6",
+    "embla-carousel-react": "^8.6.0",
     "eventsource-parser": "^1.1.2",
     "human-id": "^4.1.1",
     "i18next": "^23.7.16",

web/src/components/markdown-content/image-carousel.tsx

@@ -0,0 +1,139 @@
+import Image from '@/components/image';
+import {
+  Carousel,
+  CarouselContent,
+  CarouselItem,
+  CarouselNext,
+  CarouselPrevious,
+} from '@/components/ui/carousel';
+import { IReference, IReferenceChunk } from '@/interfaces/database/chat';
+import { getExtension } from '@/utils/document-util';
+import { useCallback } from 'react';
+
+interface ImageCarouselProps {
+  group: Array<{
+    id: string;
+    fullMatch: string;
+    start: number;
+  }>;
+  reference: IReference;
+  fileThumbnails: Record<string, string>;
+  onImageClick: (
+    documentId: string,
+    chunk: IReferenceChunk,
+    isPdf: boolean,
+    documentUrl?: string,
+  ) => void;
+}
+
+interface ReferenceInfo {
+  documentUrl?: string;
+  fileThumbnail?: string;
+  fileExtension?: string;
+  imageId?: string;
+  chunkItem?: IReferenceChunk;
+  documentId?: string;
+  document?: any;
+}
+
+const getReferenceInfo = (
+  chunkIndex: number,
+  reference: IReference,
+  fileThumbnails: Record<string, string>,
+): ReferenceInfo => {
+  const chunks = reference?.chunks ?? [];
+  const chunkItem = chunks[chunkIndex];
+  const document = reference?.doc_aggs?.find(
+    (x) => x?.doc_id === chunkItem?.document_id,
+  );
+  const documentId = document?.doc_id;
+  const documentUrl = document?.url;
+  const fileThumbnail = documentId ? fileThumbnails[documentId] : '';
+  const fileExtension = documentId ? getExtension(document?.doc_name) : '';
+  const imageId = chunkItem?.image_id;
+
+  return {
+    documentUrl,
+    fileThumbnail,
+    fileExtension,
+    imageId,
+    chunkItem,
+    documentId,
+    document,
+  };
+};
+
+/**
+ * Component to render image carousel for a group of consecutive image references
+ */
+export const ImageCarousel = ({
+  group,
+  reference,
+  fileThumbnails,
+  onImageClick,
+}: ImageCarouselProps) => {
+  const getChunkIndex = (match: string) => Number(match);
+
+  const handleImageClick = useCallback(
+    (
+      imageId: string,
+      chunkItem: IReferenceChunk,
+      documentId: string,
+      fileExtension: string,
+      documentUrl?: string,
+    ) =>
+      () =>
+        onImageClick(
+          documentId,
+          chunkItem,
+          fileExtension === 'pdf',
+          documentUrl,
+        ),
+    [onImageClick],
+  );
+
+  return (
+    <Carousel
+      className="w-44"
+      opts={{
+        align: 'start',
+        skipSnaps: false,
+      }}
+    >
+      <CarouselContent>
+        {group.map((ref) => {
+          const chunkIndex = getChunkIndex(ref.id);
+          const { documentUrl, fileExtension, imageId, chunkItem, documentId } =
+            getReferenceInfo(chunkIndex, reference, fileThumbnails);
+
+          return (
+            <CarouselItem key={ref.id}>
+              <section>
+                <Image
+                  id={imageId!}
+                  className="object-contain max-h-36"
+                  onClick={
+                    documentId && chunkItem
+                      ? handleImageClick(
+                          imageId!,
+                          chunkItem,
+                          documentId,
+                          fileExtension!,
+                          documentUrl,
+                        )
+                      : () => {}
+                  }
+                />
+                <span className="text-accent-primary">{imageId}</span>
+              </section>
+            </CarouselItem>
+          );
+        })}
+      </CarouselContent>
+      <CarouselPrevious className="h-8 w-8" />
+      <CarouselNext className="h-8 w-8" />
+    </Carousel>
+  );
+};
+
+export default ImageCarousel;

web/src/components/markdown-content/index.less

@@ -25,7 +25,7 @@
   display: block;
   object-fit: contain;
   max-width: 100%;
-  max-height: 6vh;
+  max-height: 10vh;
 }
 
 .referenceImagePreview {

web/src/components/markdown-content/index.tsx

@@ -5,7 +5,6 @@ import { getExtension } from '@/utils/document-util';
 import DOMPurify from 'dompurify';
 import { useCallback, useEffect, useMemo } from 'react';
 import Markdown from 'react-markdown';
-import reactStringReplace from 'react-string-replace';
 import SyntaxHighlighter from 'react-syntax-highlighter';
 import rehypeKatex from 'rehype-katex';
 import rehypeRaw from 'rehype-raw';
@@ -19,7 +18,6 @@ import 'katex/dist/katex.min.css'; // `rehype-katex` does not import the CSS for
 
 import { useFetchDocumentThumbnailsByIds } from '@/hooks/use-document-request';
 import {
-  currentReg,
   preprocessLaTeX,
   replaceTextByOldReg,
   replaceThinkToSection,
@@ -35,9 +33,15 @@ import {
   HoverCardContent,
   HoverCardTrigger,
 } from '../ui/hover-card';
+import { ImageCarousel } from './image-carousel';
 import styles from './index.less';
+import {
+  groupConsecutiveReferences,
+  shouldShowCarousel,
+} from './reference-utils';
 
 const getChunkIndex = (match: string) => Number(match);
+
 // TODO: The display of the table is inconsistent with the display previously placed in the MessageItem.
 const MarkdownContent = ({
   reference,
@@ -208,16 +212,40 @@ const MarkdownContent = ({
 
   const renderReference = useCallback(
     (text: string) => {
-      let replacedText = reactStringReplace(text, currentReg, (match, i) => {
+      const groups = groupConsecutiveReferences(text);
-        const chunkIndex = getChunkIndex(match);
+      const elements = [];
+      let lastIndex = 0;
 
-        const { documentUrl, fileExtension, imageId, chunkItem, documentId } =
+      groups.forEach((group, groupIndex) => {
-          getReferenceInfo(chunkIndex);
+        if (group[0].start > lastIndex) {
+          elements.push(text.substring(lastIndex, group[0].start));
+        }
 
+        if (shouldShowCarousel(group, reference)) {
+          elements.push(
+            <ImageCarousel
+              key={`carousel-${groupIndex}`}
+              group={group}
+              reference={reference}
+              fileThumbnails={fileThumbnails}
+              onImageClick={handleDocumentButtonClick}
+            />,
+          );
+        } else {
+          group.forEach((ref) => {
+            const chunkIndex = getChunkIndex(ref.id);
+            const {
+              documentUrl,
+              fileExtension,
+              imageId,
+              chunkItem,
+              documentId,
+            } = getReferenceInfo(chunkIndex);
             const docType = chunkItem?.doc_type;
 
-        return showImage(docType) ? (
+            if (showImage(docType)) {
-          <section>
+              elements.push(
+                <section key={ref.id}>
                   <Image
                     id={imageId}
                     className={styles.referenceInnerChunkImage}
@@ -231,28 +259,41 @@ const MarkdownContent = ({
                           )
                         : () => {}
                     }
-            ></Image>
+                  />
                   <span className="text-accent-primary"> {imageId}</span>
-          </section>
+                </section>,
-        ) : (
+              );
-          <HoverCard key={i}>
+            } else {
+              elements.push(
+                <HoverCard key={ref.id}>
                   <HoverCardTrigger>
                     <CircleAlert className="size-4 inline-block" />
                   </HoverCardTrigger>
                   <HoverCardContent className="max-w-3xl">
                     {getPopoverContent(chunkIndex)}
                   </HoverCardContent>
-          </HoverCard>
+                </HoverCard>,
               );
+            }
+          });
+        }
+
+        lastIndex = group[group.length - 1].end;
       });
 
-      // replacedText = reactStringReplace(replacedText, curReg, (match, i) => (
+      if (lastIndex < text.length) {
-      //   <span className={styles.cursor} key={i}></span>
+        elements.push(text.substring(lastIndex));
-      // ));
+      }
 
-      return replacedText;
+      return elements;
     },
-    [getPopoverContent, getReferenceInfo, handleDocumentButtonClick],
+    [
+      getPopoverContent,
+      getReferenceInfo,
+      handleDocumentButtonClick,
+      reference,
+      fileThumbnails,
+    ],
   );
 
   return (

web/src/components/markdown-content/reference-utils.ts

@@ -0,0 +1,67 @@
+import { IReference } from '@/interfaces/database/chat';
+import { currentReg, showImage } from '@/utils/chat';
+
+export interface ReferenceMatch {
+  id: string;
+  fullMatch: string;
+  start: number;
+  end: number;
+}
+
+export type ReferenceGroup = ReferenceMatch[];
+
+export const findAllReferenceMatches = (text: string): ReferenceMatch[] => {
+  const matches: ReferenceMatch[] = [];
+  let match;
+  while ((match = currentReg.exec(text)) !== null) {
+    matches.push({
+      id: match[1],
+      fullMatch: match[0],
+      start: match.index,
+      end: match.index + match[0].length,
+    });
+  }
+  return matches;
+};
+
+/**
+ * Helper to group consecutive references
+ */
+export const groupConsecutiveReferences = (text: string): ReferenceGroup[] => {
+  const matches = findAllReferenceMatches(text);
+  // Construct a two-dimensional array to distinguish whether images are continuous.
+  const groups: ReferenceGroup[] = [];
+
+  if (matches.length === 0) return groups;
+
+  let currentGroup: ReferenceGroup = [matches[0]];
+  // A group with only one element contains non-contiguous images,
+  // while a group with multiple elements contains contiguous images.
+  for (let i = 1; i < matches.length; i++) {
+    // If the end of the previous element equals the start of the current element,
+    // it means that they are consecutive images.
+    if (matches[i].start === currentGroup[currentGroup.length - 1].end) {
+      currentGroup.push(matches[i]);
+    } else {
+      // Save current group and start a new one
+      groups.push(currentGroup);
+      currentGroup = [matches[i]];
+    }
+  }
+  groups.push(currentGroup);
+
+  return groups;
+};
+
+export const shouldShowCarousel = (
+  group: ReferenceGroup,
+  reference: IReference,
+): boolean => {
+  if (group.length < 2) return false; // Need at least 2 images for carousel
+
+  return group.every((ref) => {
+    const chunkIndex = Number(ref.id);
+    const chunk = reference.chunks[chunkIndex];
+    return chunk && showImage(chunk.doc_type);
+  });
+};

web/src/components/ui/carousel.tsx

@@ -0,0 +1,241 @@
+'use client';
+
+import useEmblaCarousel, {
+  type UseEmblaCarouselType,
+} from 'embla-carousel-react';
+import { ArrowLeft, ArrowRight } from 'lucide-react';
+import * as React from 'react';
+
+import { Button } from '@/components/ui/button';
+import { cn } from '@/lib/utils';
+
+type CarouselApi = UseEmblaCarouselType[1];
+type UseCarouselParameters = Parameters<typeof useEmblaCarousel>;
+type CarouselOptions = UseCarouselParameters[0];
+type CarouselPlugin = UseCarouselParameters[1];
+
+type CarouselProps = {
+  opts?: CarouselOptions;
+  plugins?: CarouselPlugin;
+  orientation?: 'horizontal' | 'vertical';
+  setApi?: (api: CarouselApi) => void;
+};
+
+type CarouselContextProps = {
+  carouselRef: ReturnType<typeof useEmblaCarousel>[0];
+  api: ReturnType<typeof useEmblaCarousel>[1];
+  scrollPrev: () => void;
+  scrollNext: () => void;
+  canScrollPrev: boolean;
+  canScrollNext: boolean;
+} & CarouselProps;
+
+const CarouselContext = React.createContext<CarouselContextProps | null>(null);
+
+function useCarousel() {
+  const context = React.useContext(CarouselContext);
+
+  if (!context) {
+    throw new Error('useCarousel must be used within a <Carousel />');
+  }
+
+  return context;
+}
+
+function Carousel({
+  orientation = 'horizontal',
+  opts,
+  setApi,
+  plugins,
+  className,
+  children,
+  ...props
+}: React.ComponentProps<'div'> & CarouselProps) {
+  const [carouselRef, api] = useEmblaCarousel(
+    {
+      ...opts,
+      axis: orientation === 'horizontal' ? 'x' : 'y',
+    },
+    plugins,
+  );
+  const [canScrollPrev, setCanScrollPrev] = React.useState(false);
+  const [canScrollNext, setCanScrollNext] = React.useState(false);
+
+  const onSelect = React.useCallback((api: CarouselApi) => {
+    if (!api) return;
+    setCanScrollPrev(api.canScrollPrev());
+    setCanScrollNext(api.canScrollNext());
+  }, []);
+
+  const scrollPrev = React.useCallback(() => {
+    api?.scrollPrev();
+  }, [api]);
+
+  const scrollNext = React.useCallback(() => {
+    api?.scrollNext();
+  }, [api]);
+
+  const handleKeyDown = React.useCallback(
+    (event: React.KeyboardEvent<HTMLDivElement>) => {
+      if (event.key === 'ArrowLeft') {
+        event.preventDefault();
+        scrollPrev();
+      } else if (event.key === 'ArrowRight') {
+        event.preventDefault();
+        scrollNext();
+      }
+    },
+    [scrollPrev, scrollNext],
+  );
+
+  React.useEffect(() => {
+    if (!api || !setApi) return;
+    setApi(api);
+  }, [api, setApi]);
+
+  React.useEffect(() => {
+    if (!api) return;
+    onSelect(api);
+    api.on('reInit', onSelect);
+    api.on('select', onSelect);
+
+    return () => {
+      api?.off('select', onSelect);
+    };
+  }, [api, onSelect]);
+
+  return (
+    <CarouselContext.Provider
+      value={{
+        carouselRef,
+        api: api,
+        opts,
+        orientation:
+          orientation || (opts?.axis === 'y' ? 'vertical' : 'horizontal'),
+        scrollPrev,
+        scrollNext,
+        canScrollPrev,
+        canScrollNext,
+      }}
+    >
+      <div
+        onKeyDownCapture={handleKeyDown}
+        className={cn('relative', className)}
+        role="region"
+        aria-roledescription="carousel"
+        data-slot="carousel"
+        {...props}
+      >
+        {children}
+      </div>
+    </CarouselContext.Provider>
+  );
+}
+
+function CarouselContent({ className, ...props }: React.ComponentProps<'div'>) {
+  const { carouselRef, orientation } = useCarousel();
+
+  return (
+    <div
+      ref={carouselRef}
+      className="overflow-hidden"
+      data-slot="carousel-content"
+    >
+      <div
+        className={cn(
+          'flex',
+          orientation === 'horizontal' ? '-ml-4' : '-mt-4 flex-col',
+          className,
+        )}
+        {...props}
+      />
+    </div>
+  );
+}
+
+function CarouselItem({ className, ...props }: React.ComponentProps<'div'>) {
+  const { orientation } = useCarousel();
+
+  return (
+    <div
+      role="group"
+      aria-roledescription="slide"
+      data-slot="carousel-item"
+      className={cn(
+        'min-w-0 shrink-0 grow-0 basis-full',
+        orientation === 'horizontal' ? 'pl-4' : 'pt-4',
+        className,
+      )}
+      {...props}
+    />
+  );
+}
+
+function CarouselPrevious({
+  className,
+  variant = 'outline',
+  size = 'icon',
+  ...props
+}: React.ComponentProps<typeof Button>) {
+  const { orientation, scrollPrev, canScrollPrev } = useCarousel();
+
+  return (
+    <Button
+      data-slot="carousel-previous"
+      variant={variant}
+      size={size}
+      className={cn(
+        'absolute size-8 rounded-full',
+        orientation === 'horizontal'
+          ? 'top-1/2 -left-12 -translate-y-1/2'
+          : '-top-12 left-1/2 -translate-x-1/2 rotate-90',
+        className,
+      )}
+      disabled={!canScrollPrev}
+      onClick={scrollPrev}
+      {...props}
+    >
+      <ArrowLeft />
+      <span className="sr-only">Previous slide</span>
+    </Button>
+  );
+}
+
+function CarouselNext({
+  className,
+  variant = 'outline',
+  size = 'icon',
+  ...props
+}: React.ComponentProps<typeof Button>) {
+  const { orientation, scrollNext, canScrollNext } = useCarousel();
+
+  return (
+    <Button
+      data-slot="carousel-next"
+      variant={variant}
+      size={size}
+      className={cn(
+        'absolute size-8 rounded-full',
+        orientation === 'horizontal'
+          ? 'top-1/2 -right-12 -translate-y-1/2'
+          : '-bottom-12 left-1/2 -translate-x-1/2 rotate-90',
+        className,
+      )}
+      disabled={!canScrollNext}
+      onClick={scrollNext}
+      {...props}
+    >
+      <ArrowRight />
+      <span className="sr-only">Next slide</span>
+    </Button>
+  );
+}
+
+export {
+  Carousel,
+  CarouselContent,
+  CarouselItem,
+  CarouselNext,
+  CarouselPrevious,
+  type CarouselApi,
+};