⬇️ refactor: Assistant File Downloads (danny-avila#2364)

* refactor(getFiledownload): explicit accept of `application/octet-stream` * chore: test compose file * chore: test compose file fix * chore(files/download): add more logs * Fix proxy_pass URLs in nginx.conf * fix: proxy_pass URLs in nginx.conf to fix file downloads from URL * chore: move test compose file to utils dir * refactor(useFileDownload): simplify API request by passing `file_id` instead of `filepath`
678720 · Apr 9, 2024 · cb64b84 · cb64b84
1 parent cc71125
commit cb64b84
Show file tree

Hide file tree

Showing 6 changed files with 92 additions and 20 deletions.
diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.js
@@ -66,17 +66,16 @@ router.delete('/', async (req, res) => {
   }
 });
 
-router.get('/download/:userId/:filepath', async (req, res) => {
+router.get('/download/:userId/:file_id', async (req, res) => {
   try {
-    const { userId, filepath } = req.params;
+    const { userId, file_id } = req.params;
+    logger.debug(`File download requested by user ${userId}: ${file_id}`);
 
     if (userId !== req.user.id) {
       logger.warn(`${errorPrefix} forbidden: ${file_id}`);
       return res.status(403).send('Forbidden');
     }
 
-    const parts = filepath.split('/');
-    const file_id = parts[2];
     const [file] = await getFiles({ file_id });
     const errorPrefix = `File download requested by user ${userId}`;
 
@@ -114,8 +113,10 @@ router.get('/download/:userId/:filepath', async (req, res) => {
     if (file.source === FileSources.openai) {
       req.body = { model: file.model };
       const { openai } = await initializeClient({ req, res });
+      logger.debug(`Downloading file ${file_id} from OpenAI`);
       passThrough = await getDownloadStream(file_id, openai);
       setHeaders();
+      logger.debug(`File ${file_id} downloaded from OpenAI`);
       passThrough.body.pipe(res);
     } else {
       fileStream = getDownloadStream(file_id);

diff --git a/client/nginx.conf b/client/nginx.conf
@@ -14,12 +14,12 @@ server {
     # The default limits for image uploads as of 11/22/23 is 20MB/file, and 25MB/request
     client_max_body_size 25M;
 
-    location /api {
-        proxy_pass http://api:3080/api;
+    location /api/ {
+        proxy_pass http://api:3080$request_uri;
     }
 
     location / {
-        proxy_pass http://api:3080;
+        proxy_pass http://api:3080/;
     }
 
     ########################################  SSL  ########################################

diff --git a/client/src/components/Chat/Messages/Content/Markdown.tsx b/client/src/components/Chat/Messages/Content/Markdown.tsx
@@ -44,21 +44,23 @@ export const a = memo(({ href, children }: { href: string; children: React.React
   const { showToast } = useToastContext();
   const localize = useLocalize();
 
-  const { filepath, filename } = useMemo(() => {
+  const { file_id, filename, filepath } = useMemo(() => {
     const pattern = new RegExp(`(?:files|outputs)/${user?.id}/([^\\s]+)`);
     const match = href.match(pattern);
     if (match && match[0]) {
       const path = match[0];
-      const name = path.split('/').pop();
-      return { filepath: path, filename: name };
+      const parts = path.split('/');
+      const name = parts.pop();
+      const file_id = parts.pop();
+      return { file_id, filename: name, filepath: path };
     }
-    return { filepath: '', filename: '' };
+    return { file_id: '', filename: '', filepath: '' };
   }, [user?.id, href]);
 
-  const { refetch: downloadFile } = useFileDownload(user?.id ?? '', filepath);
+  const { refetch: downloadFile } = useFileDownload(user?.id ?? '', file_id);
   const props: { target?: string; onClick?: React.MouseEventHandler } = { target: '_new' };
 
-  if (!filepath || !filename) {
+  if (!file_id || !filename) {
     return (
       <a href={href} {...props}>
         {children}

diff --git a/client/src/data-provider/queries.ts b/client/src/data-provider/queries.ts
@@ -325,15 +325,16 @@ export const useGetAssistantDocsQuery = (
   );
 };
 
-export const useFileDownload = (userId: string, filepath: string): QueryObserverResult<string> => {
+export const useFileDownload = (userId?: string, file_id?: string): QueryObserverResult<string> => {
   const queryClient = useQueryClient();
   return useQuery(
-    [QueryKeys.fileDownload, filepath],
+    [QueryKeys.fileDownload, file_id],
     async () => {
-      if (!userId) {
+      if (!userId || !file_id) {
         console.warn('No user ID provided for file download');
+        return;
       }
-      const response = await dataService.getFileDownload(userId, filepath);
+      const response = await dataService.getFileDownload(userId, file_id);
       const blob = response.data;
       const downloadURL = window.URL.createObjectURL(blob);
       try {

diff --git a/packages/data-provider/src/data-service.ts b/packages/data-provider/src/data-service.ts
@@ -202,10 +202,12 @@ export const uploadAssistantAvatar = (data: m.AssistantAvatarVariables): Promise
   );
 };
 
-export const getFileDownload = async (userId: string, filepath: string): Promise<AxiosResponse> => {
-  const encodedFilePath = encodeURIComponent(filepath);
-  return request.getResponse(`${endpoints.files()}/download/${userId}/${encodedFilePath}`, {
+export const getFileDownload = async (userId: string, file_id: string): Promise<AxiosResponse> => {
+  return request.getResponse(`${endpoints.files()}/download/${userId}/${file_id}`, {
     responseType: 'blob',
+    headers: {
+      Accept: 'application/octet-stream',
+    },
   });
 };
 

diff --git a/utils/docker/test-compose.yml b/utils/docker/test-compose.yml
@@ -0,0 +1,66 @@
+version: "3.8"
+services:
+  # api:
+  #     - HOST=0.0.0.0
+  #     - NODE_ENV=production
+  #     - MONGO_URI=mongodb://mongodb:27017/LibreChat
+  #     - MEILI_HOST=http://meilisearch:7700
+  #     - RAG_PORT=${RAG_PORT:-8000}
+  #     - RAG_API_URL=http://rag_api:${RAG_PORT:-8000}
+  client:
+    build:
+      context: .
+      dockerfile: Dockerfile.multi
+      target: prod-stage
+    container_name: LibreChat-NGINX
+    ports:
+      - 80:80
+      - 443:443
+    restart: always
+    volumes:
+      - ./client/nginx.conf:/etc/nginx/conf.d/default.conf
+  mongodb:
+    container_name: chat-mongodb
+    ports:  # Uncomment this to access mongodb from outside docker, not safe in deployment
+      - 27018:27017
+    image: mongo
+    restart: always
+    volumes:
+      - ./data-node:/data/db
+    command: mongod --noauth
+  meilisearch:
+    container_name: chat-meilisearch
+    image: getmeili/meilisearch:v1.7.3
+    ports: # Uncomment this to access meilisearch from outside docker
+      - 7700:7700 # if exposing these ports, make sure your master key is not the default value
+    env_file:
+      - .env
+    environment:
+      - MEILI_HOST=http://meilisearch:7700
+      - MEILI_NO_ANALYTICS=true
+    volumes:
+      - ./meili_data_v1.7:/meili_data
+  vectordb:
+    image: ankane/pgvector:latest
+    environment:
+      POSTGRES_DB: mydatabase
+      POSTGRES_USER: myuser
+      POSTGRES_PASSWORD: mypassword
+    restart: always
+    volumes:
+      - pgdata2:/var/lib/postgresql/data
+  rag_api:
+    image: ghcr.io/danny-avila/librechat-rag-api-dev-lite:latest
+    environment:
+      - DB_HOST=vectordb
+      - RAG_PORT=${RAG_PORT:-8000}
+    restart: always
+    ports:
+      - 8000:8000
+    depends_on:
+      - vectordb
+    env_file:
+      - .env
+
+volumes:
+  pgdata2: