File Processing

ws://<host>/ws/upload-health-report?token=<auth_token>

{
    "type": "upload_start",
    "messageId": "unique-message-id",
    "sessionId": "session-id",
    "query": "user notes",
    "isFirstMessage": false,
    "query_user_id": "target-user-id",
    "files": [
        {
            "filename": "report.pdf",
            "contentType": "application/pdf",
            "size": 1024000
        }
    ]
}

{
    "type": "upload_start",
    "messageId": "generated-message-id",
    "sessionId": "session-id",
    "status": "uploading",
    "progress": 0,
    "message": "Ready to receive 1 files",
    "files": [...]
}

{
    "type": "upload_chunk",
    "messageId": "message-id",
    "filename": "report.pdf",
    "chunk": "<base64-encoded-data>",
    "chunkIndex": 0,
    "totalChunks": 10,
    "contentType": "application/pdf",
    "fileSize": 1024000
}

{
    "type": "file_progress",
    "messageId": "message-id",
    "filename": "report.pdf",
    "progress": 10.0,
    "status": "uploading",
    "message": "Uploading report.pdf: 10.0%"
}

{
    "type": "file_received",
    "messageId": "message-id",
    "filename": "report.pdf",
    "status": "received",
    "message": "File report.pdf received successfully",
    "size": 1024000
}

{
    "type": "upload_progress",
    "messageId": "message-id",
    "status": "processing",
    "progress": 65,
    "message": "Starting to analyze health indicators in report.pdf...",
    "filename": "report.pdf",
    "timestamp": "2024-01-15T10:30:00.000Z"
}

{
    "type": "upload_completed",
    "messageId": "message-id",
    "status": "completed",
    "progress": 100,
    "message": "Processing completed: 1 files successful",
    "successful_files": 1,
    "failed_files": 0,
    "total_files": 1,
    "results": {
        "success": true,
        "message": "File processing completed",
        "type": "pdf",
        "url_thumb": ["https://..."],
        "url_full": ["https://..."],
        "files": [
            {
                "filename": "report.pdf",
                "type": "pdf",
                "url_thumb": "https://...",
                "url_full": "https://...",
                "file_key": "uploads/xxx.pdf",
                "file_size": 1024000,
                "file_abstract": "Health report summary...",
                "file_name": "2024 Annual Health Report",
                "success": true
            }
        ]
    }
}

{
    "type": "report_date_detected",
    "file_key": "uploads/xxx.pdf",
    "file_name": "2024 Annual Health Report",
    "report_date": "2024-03-29 00:00:00",
    "date_source": "document"
}

{
    "type": "extraction_completed",
    "file_key": "uploads/xxx.pdf",
    "file_name": "2024 Annual Health Report",
    "indicators_count": 145,
    "failed": false,
    "report_date": "2024-03-29 00:00:00",
    "date_source": "document"
}

{
    "type": "ping"
}

{
    "type": "pong",
    "timestamp": "2024-01-15T10:30:00.000Z"
}

POST /files/upload
Content-Type: multipart/form-data
Authorization: Bearer <token>

{
    "code": 0,
    "msg": "All 2 files uploaded successfully",
    "data": [
        {
            "file_url": "https://storage.example.com/uploads/20240115_xxx.pdf",
            "file_name": "report.pdf",
            "file_key": "uploads/20240115_xxx.pdf",
            "file_size": 1024000,
            "file_type": "application/pdf",
            "upload_time": "2024-01-15T10:30:00.000Z",
            "duration": null
        }
    ]
}

┌───────────────────────────────────────────────────────────────┐
│                         File Upload                           │
│  ┌─────────────┐    ┌─────────────┐    ┌─────────────┐        │
│  │  WebSocket  │    │  REST API   │    │   Direct    │        │
│  │   Upload    │    │   Upload    │    │   Upload    │        │
│  └──────┬──────┘    └──────┬──────┘    └──────┬──────┘        │
│         │                  │                  │                │
│         └──────────────────┼──────────────────┘                │
│                            ▼                                   │
│                   ┌─────────────────┐                          │
│                   │  FileProcessor  │                          │
│                   └────────┬────────┘                          │
│                            │                                   │
│         ┌──────────────────┼──────────────────┐                │
│         ▼                  ▼                  ▼                │
│  ┌─────────────┐   ┌─────────────┐   ┌─────────────┐          │
│  │ PDFHandler  │   │ImageHandler │   │ TextHandler │   ...    │
│  └──────┬──────┘   └──────┬──────┘   └──────┬──────┘          │
│         │                  │                  │                │
│         └──────────────────┼──────────────────┘                │
│                            ▼                                   │
│                  ┌───────────────────┐                         │
│                  │IndicatorExtractor │                         │
│                  └─────────┬─────────┘                         │
│                            ▼                                   │
│                   ┌─────────────────┐                          │
│                   │    Database     │                          │
│                   │  (th_messages,  │                          │
│                   │  th_observation)│                          │
│                   └─────────────────┘                          │
└───────────────────────────────────────────────────────────────┘

# Handler selection priority
1. GeneticHandler  - Genetic data files
2. ImageHandler    - Image files (image/*)
3. PDFHandler      - PDF documents

Single/few page PDF:
  35-65%: File upload and save
  65-90%: LLM indicator extraction

Multi-page PDF (>2 pages):
  35-50%: File upload and PDF splitting
  50-70%: Parallel page processing
  70-90%: Result merging and deduplication

35-55%: Image upload
55-90%: Recognition + indicator extraction

# config.yaml
ENABLE_INDICATOR_EXTRACTION: 1  # Set to 1 to enable indicator extraction

{
    "indicators": [
        {
            "original_indicator": "White Blood Cell Count",
            "value": "6.5",
            "unit": "10^9/L",
            "reference_range": "4.0-10.0",
            "status": "normal"
        },
        {
            "original_indicator": "Hemoglobin",
            "value": "145",
            "unit": "g/L",
            "reference_range": "120-160",
            "status": "normal"
        }
    ],
    "content_info": {
        "date_time": "2024-01-15",
        "institution": "XX Hospital"
    },
    "file_abstract": "January 2024 health report including CBC, comprehensive metabolic panel..."
}

POST /api/v1/health-indicators/file-date
Content-Type: application/json
Authorization: Bearer <token>

{"file_key": "20260903103000_ab12cd34", "report_date": "2025-08-15"}

POST /api/v1/data/delete-files
Content-Type: application/json
Authorization: Bearer <token>

{
    "message_id": "message-uuid",
    "file_keys": ["uploads/xxx.pdf", "uploads/yyy.jpg"]
}

{
    "code": 0,
    "msg": "Successfully deleted 2 file(s)",
    "data": {
        "success": true,
        "message_id": "message-uuid",
        "deleted_files": [
            {
                "file_key": "uploads/xxx.pdf",
                "filename": "report.pdf",
                "type": "pdf",
                "status": "deleted"
            }
        ],
        "failed_deletions": [],
        "remaining_files_count": 0,
        "message_deleted": true
    }
}

interface FileUploadData {
    file_url: string;      // File access URL
    file_name: string;     // Original filename
    file_key: string;      // Storage key
    file_size: number;     // File size in bytes
    file_type: string;     // MIME type
    upload_time: string;   // Upload timestamp
}

interface FileDeleteRequest {
    message_id: string;        // Message ID
    file_keys?: string[];      // List of file keys to delete
}

interface FileProcessingResult {
    success: boolean;
    message: string;
    type: string;              // File type: pdf, image, etc.
    filename: string;          // Original filename
    full_url: string;          // Full access URL
    file_key: string;          // Storage key
    file_abstract: string;     // File summary
    file_name: string;         // AI-generated file name
    raw?: string;              // Formatted markdown content from extracted data
}

{
    "type": "upload_error",
    "messageId": "message-id",
    "status": "failed",
    "message": "Upload start failed: <error_details>"
}

{
    "code": 1,
    "msg": "File upload failed: <error_details>",
    "data": null
}

{
    "type": "connection_timeout",
    "message": "Connection closed due to 5 minutes of inactivity",
    "idle_seconds": 300,
    "timeout_type": "idle",
    "active_uploads_count": 0
}

class FileUploader {
    constructor(token) {
        this.token = token;
        this.ws = null;
    }

    connect() {
        return new Promise((resolve, reject) => {
            this.ws = new WebSocket(
                `ws://localhost:18060/ws/upload-health-report?token=${this.token}`
            );

            this.ws.onopen = () => {
                console.log('WebSocket connected');
                resolve();
            };

            this.ws.onmessage = (event) => {
                const data = JSON.parse(event.data);
                this.handleMessage(data);
            };

            this.ws.onerror = reject;
        });
    }

    async uploadFile(file, messageId, sessionId) {
        // 1. Send upload start message
        this.ws.send(JSON.stringify({
            type: 'upload_start',
            messageId,
            sessionId,
            files: [{
                filename: file.name,
                contentType: file.type,
                size: file.size
            }]
        }));

        // 2. Chunked upload
        const chunkSize = 1024 * 1024; // 1MB
        const totalChunks = Math.ceil(file.size / chunkSize);

        for (let i = 0; i < totalChunks; i++) {
            const start = i * chunkSize;
            const end = Math.min(start + chunkSize, file.size);
            const chunk = file.slice(start, end);
            const base64 = await this.readAsBase64(chunk);

            this.ws.send(JSON.stringify({
                type: 'upload_chunk',
                messageId,
                filename: file.name,
                chunk: base64,
                chunkIndex: i,
                totalChunks,
                contentType: file.type,
                fileSize: file.size
            }));
        }
    }

    readAsBase64(blob) {
        return new Promise((resolve) => {
            const reader = new FileReader();
            reader.onloadend = () => {
                const base64 = reader.result.split(',')[1];
                resolve(base64);
            };
            reader.readAsDataURL(blob);
        });
    }

    handleMessage(data) {
        switch (data.type) {
            case 'upload_progress':
                console.log(`Progress: ${data.progress}% - ${data.message}`);
                break;
            case 'upload_completed':
                console.log('Upload completed:', data.results);
                break;
            case 'upload_error':
                console.error('Upload error:', data.message);
                break;
        }
    }
}

// Usage
const uploader = new FileUploader('your-auth-token');
await uploader.connect();
await uploader.uploadFile(file, 'msg-123', 'session-456');

import requests
from pathlib import Path

def upload_files(file_paths: list, token: str, folder: str = None) -> dict:
    """
    Upload files to the server.
    
    Args:
        file_paths: List of file paths to upload
        token: Authentication token
        folder: Custom folder prefix (optional)
    
    Returns:
        Upload result dictionary
    """
    url = "http://localhost:18060/files/upload"
    headers = {"Authorization": f"Bearer {token}"}
    
    files = []
    for file_path in file_paths:
        path = Path(file_path)
        files.append(('files', (path.name, open(path, 'rb'))))
    
    params = {}
    if folder:
        params['folder'] = folder
    
    try:
        response = requests.post(
            url,
            headers=headers,
            files=files,
            params=params
        )
        return response.json()
    finally:
        # Close file handles
        for _, file_tuple in files:
            file_tuple[1].close()


def delete_files(message_id: str, file_keys: list, token: str) -> dict:
    """
    Delete files from a message.
    
    Args:
        message_id: Message ID containing the files
        file_keys: List of file keys to delete
        token: Authentication token
    
    Returns:
        Deletion result dictionary
    """
    url = "http://localhost:18060/api/v1/data/delete-files"
    headers = {
        "Authorization": f"Bearer {token}",
        "Content-Type": "application/json"
    }
    
    payload = {
        "message_id": message_id,
        "file_keys": file_keys
    }
    
    response = requests.post(url, headers=headers, json=payload)
    return response.json()


# Usage examples
if __name__ == "__main__":
    TOKEN = "your-auth-token"
    
    # Upload files
    result = upload_files(
        file_paths=["./report.pdf", "./lab_results.jpg"],
        token=TOKEN,
        folder="health-reports"
    )
    print("Upload result:", result)
    
    # Delete files
    if result["code"] == 0:
        message_id = result["data"][0]["file_key"].split("/")[0]
        delete_result = delete_files(
            message_id=message_id,
            file_keys=[item["file_key"] for item in result["data"]],
            token=TOKEN
        )
        print("Delete result:", delete_result)