Merge pull request #1419 from fadingNA/pagination

Pagination

README.md

@@ -128,7 +128,7 @@ docker compose -f docker-compose-dev.yaml up -d
 > Make sure you have Python 3.10 or 3.11 installed.
 
 1. Export required environment variables or prepare a `.env` file in the project folder:
-   - Copy [.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) and create `.env`.
+   - Copy [.env-template](https://github.com/arc53/DocsGPT/blob/main/application/.env-template) and create `.env`.
 
 (check out [`application/core/settings.py`](application/core/settings.py) if you want to see more config options.)
 

application/api/user/routes.py

@@ -2,11 +2,12 @@
 import os
 import shutil
 import uuid
+import math
 
 from bson.binary import Binary, UuidRepresentation
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
-from flask import Blueprint, jsonify, make_response, request
+from flask import Blueprint, jsonify, make_response, request, redirect
 from flask_restx import inputs, fields, Namespace, Resource
 from werkzeug.utils import secure_filename
 
@@ -429,12 +430,70 @@ def get(self):
 
 
 @user_ns.route("/api/combine")
+class RedirectToSources(Resource):
+    @api.doc(
+        description="Redirects /api/combine to /api/sources for backward compatibility"
+    )
+    def get(self):
+        return redirect("/api/sources", code=301)
+
+
+@user_ns.route("/api/sources/paginated")
+class PaginatedSources(Resource):
+    @api.doc(description="Get document with pagination, sorting and filtering")
+    def get(self):
+        user = "local"
+        sort_field = request.args.get("sort", "date")  # Default to 'date'
+        sort_order = request.args.get("order", "desc")  # Default to 'desc'
+        page = int(request.args.get("page", 1))  # Default to 1
+        rows_per_page = int(request.args.get("rows", 10))  # Default to 10
+
+        # Prepare
+        query = {"user": user}
+        total_documents = sources_collection.count_documents(query)
+        total_pages = max(1, math.ceil(total_documents / rows_per_page))
+        sort_order = 1 if sort_order == "asc" else -1
+        skip = (page - 1) * rows_per_page
+
+        try:
+            documents = (
+                sources_collection.find(query)
+                .sort(sort_field, sort_order)
+                .skip(skip)
+                .limit(rows_per_page)
+            )
+
+            paginated_docs = []
+            for doc in documents:
+                doc_data = {
+                    "id": str(doc["_id"]),
+                    "name": doc.get("name", ""),
+                    "date": doc.get("date", ""),
+                    "model": settings.EMBEDDINGS_NAME,
+                    "location": "local",
+                    "tokens": doc.get("tokens", ""),
+                    "retriever": doc.get("retriever", "classic"),
+                    "syncFrequency": doc.get("sync_frequency", ""),
+                }
+                paginated_docs.append(doc_data)
+
+            response = {
+                "total": total_documents,
+                "totalPages": total_pages,
+                "currentPage": page,
+                "paginated": paginated_docs,
+            }
+            return make_response(jsonify(response), 200)
+
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+
+@user_ns.route("/api/sources")
 class CombinedJson(Resource):
     @api.doc(description="Provide JSON file with combined available indexes")
     def get(self):
         user = "local"
-        sort_field = request.args.get('sort', 'date')  # Default to 'date'
-        sort_order = request.args.get('order', "desc")  # Default to 'desc'
         data = [
             {
                 "name": "default",
@@ -447,7 +506,7 @@ def get(self):
         ]
 
         try:
-            for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order=="asc" else -1):
+            for index in sources_collection.find({"user": user}).sort("date", -1):
                 data.append(
                     {
                         "id": str(index["_id"]),
@@ -485,6 +544,7 @@ def get(self):
                         "retriever": "brave_search",
                     }
                 )
+
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
@@ -1674,7 +1734,9 @@ class TextToSpeech(Resource):
     tts_model = api.model(
         "TextToSpeechModel",
         {
-            "text": fields.String(required=True, description="Text to be synthesized as audio"),
+            "text": fields.String(
+                required=True, description="Text to be synthesized as audio"
+            ),
         },
     )
 
@@ -1686,8 +1748,15 @@ def post(self):
         try:
             tts_instance = GoogleTTS()
             audio_base64, detected_language = tts_instance.text_to_speech(text)
-            return make_response(jsonify({"success": True,'audio_base64': audio_base64,'lang':detected_language}), 200)
+            return make_response(
+                jsonify(
+                    {
+                        "success": True,
+                        "audio_base64": audio_base64,
+                        "lang": detected_language,
+                    }
+                ),
+                200,
+            )
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
-
-

frontend/src/Navigation.tsx

@@ -34,10 +34,12 @@ import {
   selectSelectedDocs,
   selectSelectedDocsStatus,
   selectSourceDocs,
+  selectPaginatedDocuments,
   setConversations,
   setModalStateDeleteConv,
   setSelectedDocs,
   setSourceDocs,
+  setPaginatedDocuments,
 } from './preferences/preferenceSlice';
 import Spinner from './assets/spinner.svg';
 import SpinnerDark from './assets/spinner-dark.svg';
@@ -72,6 +74,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   const conversations = useSelector(selectConversations);
   const modalStateDeleteConv = useSelector(selectModalStateDeleteConv);
   const conversationId = useSelector(selectConversationId);
+  const paginatedDocuments = useSelector(selectPaginatedDocuments);
   const [isDeletingConversation, setIsDeletingConversation] = useState(false);
 
   const { isMobile } = useMediaQuery();
@@ -143,9 +146,18 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
       })
       .then((updatedDocs) => {
         dispatch(setSourceDocs(updatedDocs));
+        const updatedPaginatedDocs = paginatedDocuments?.filter(
+          (document) => document.id !== doc.id,
+        );
+        dispatch(
+          setPaginatedDocuments(updatedPaginatedDocs || paginatedDocuments),
+        );
         dispatch(
           setSelectedDocs(
-            updatedDocs?.find((doc) => doc.name.toLowerCase() === 'default'),
+            Array.isArray(updatedDocs) &&
+              updatedDocs?.find(
+                (doc: Doc) => doc.name.toLowerCase() === 'default',
+              ),
           ),
         );
       })

frontend/src/api/endpoints.ts

@@ -1,7 +1,8 @@
 const endpoints = {
   USER: {
-    DOCS: '/api/combine',
+    DOCS: '/api/sources',
     DOCS_CHECK: '/api/docs_check',
+    DOCS_PAGINATED: '/api/sources/paginated',
     API_KEYS: '/api/get_api_keys',
     CREATE_API_KEY: '/api/create_api_key',
     DELETE_API_KEY: '/api/delete_api_key',

frontend/src/api/services/userService.ts

@@ -2,8 +2,9 @@ import apiClient from '../client';
 import endpoints from '../endpoints';
 
 const userService = {
-  getDocs: (sort = 'date', order = 'desc'): Promise<any> =>
-    apiClient.get(`${endpoints.USER.DOCS}?sort=${sort}&order=${order}`),
+  getDocs: (): Promise<any> => apiClient.get(`${endpoints.USER.DOCS}`),
+  getDocsWithPagination: (query: string): Promise<any> =>
+    apiClient.get(`${endpoints.USER.DOCS_PAGINATED}?${query}`),
   checkDocs: (data: any): Promise<any> =>
     apiClient.post(endpoints.USER.DOCS_CHECK, data),
   getAPIKeys: (): Promise<any> => apiClient.get(endpoints.USER.API_KEYS),

frontend/src/components/DocumentPagination.tsx

@@ -0,0 +1,119 @@
+import React, { useState } from 'react';
+import SingleArrowLeft from '../assets/single-left-arrow.svg';
+import SingleArrowRight from '../assets/single-right-arrow.svg';
+import DoubleArrowLeft from '../assets/double-arrow-left.svg';
+import DoubleArrowRight from '../assets/double-arrow-right.svg';
+
+interface PaginationProps {
+  currentPage: number;
+  totalPages: number;
+  rowsPerPage: number;
+  onPageChange: (page: number) => void;
+  onRowsPerPageChange: (rows: number) => void;
+}
+
+const Pagination: React.FC<PaginationProps> = ({
+  currentPage,
+  totalPages,
+  rowsPerPage,
+  onPageChange,
+  onRowsPerPageChange,
+}) => {
+  const [rowsPerPageOptions] = useState([5, 10, 15, 20]);
+
+  const handlePreviousPage = () => {
+    if (currentPage > 1) {
+      onPageChange(currentPage - 1);
+    }
+  };
+
+  const handleNextPage = () => {
+    if (currentPage < totalPages) {
+      onPageChange(currentPage + 1);
+    }
+  };
+
+  const handleFirstPage = () => {
+    onPageChange(1);
+  };
+
+  const handleLastPage = () => {
+    onPageChange(totalPages);
+  };
+
+  return (
+    <div className="flex items-center text-xs justify-end gap-4 mt-2 p-2 border-gray-200">
+      <div className="flex items-center gap-2 ">
+        <span className="text-gray-900 dark:text-gray-50">Rows per page:</span>
+        <select
+          value={rowsPerPage}
+          onChange={(e) => onRowsPerPageChange(Number(e.target.value))}
+          className="border border-gray-300 rounded px-2 py-1 dark:bg-dark-charcoal dark:text-gray-50"
+        >
+          {rowsPerPageOptions.map((option) => (
+            <option
+              className="bg-white dark:bg-dark-charcoal dark:text-gray-50"
+              key={option}
+              value={option}
+            >
+              {option}
+            </option>
+          ))}
+        </select>
+      </div>
+
+      <div className="text-gray-900 dark:text-gray-50">
+        Page {currentPage} of {totalPages}
+      </div>
+
+      <div className="flex items-center gap-2 text-gray-900 dark:text-gray-50">
+        <button
+          onClick={handleFirstPage}
+          disabled={currentPage === 1}
+          className="px-2 py-1 border rounded disabled:opacity-50"
+        >
+          <img
+            src={DoubleArrowLeft}
+            alt="arrow"
+            className="dark:invert dark:sepia dark:brightness-200"
+          />
+        </button>
+        <button
+          onClick={handlePreviousPage}
+          disabled={currentPage === 1}
+          className="px-2 py-1 border rounded disabled:opacity-50"
+        >
+          <img
+            src={SingleArrowLeft}
+            alt="arrow"
+            className="dark:invert dark:sepia dark:brightness-200"
+          />
+        </button>
+        <button
+          onClick={handleNextPage}
+          disabled={currentPage === totalPages}
+          className="px-2 py-1 border rounded disabled:opacity-50"
+        >
+          <img
+            src={SingleArrowRight}
+            alt="arrow"
+            className="dark:invert dark:sepia dark:brightness-200"
+          />
+        </button>
+        <button
+          onClick={handleLastPage}
+          disabled={currentPage === totalPages}
+          className="px-2 py-1 border rounded disabled:opacity-50"
+        >
+          <img
+            src={DoubleArrowRight}
+            alt="arrow"
+            className="dark:invert dark:sepia dark:brightness-200"
+          />
+        </button>
+      </div>
+    </div>
+  );
+};
+
+export default Pagination;

frontend/src/hooks/useDefaultDocument.ts

@@ -17,11 +17,12 @@ export default function useDefaultDocument() {
     getDocs().then((data) => {
       dispatch(setSourceDocs(data));
       if (!selectedDoc)
-        data?.forEach((doc: Doc) => {
-          if (doc.model && doc.name === 'default') {
-            dispatch(setSelectedDocs(doc));
-          }
-        });
+        Array.isArray(data) &&
+          data?.forEach((doc: Doc) => {
+            if (doc.model && doc.name === 'default') {
+              dispatch(setSelectedDocs(doc));
+            }
+          });
     });
   };
 

frontend/src/index.css

@@ -50,11 +50,11 @@ body.dark {
 
 @layer components {
   .table-default {
-    @apply block w-max table-auto content-center justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray;
+    @apply block w-full mx-auto table-auto content-start justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray overflow-auto;
   }
 
   .table-default th {
-    @apply p-4 w-[244px] font-normal text-gray-400; /* Remove border-r */
+    @apply p-4 w-full font-normal text-gray-400 text-nowrap; /* Remove border-r */
   }
 
   .table-default th:last-child {