diff --git a/.github/workflows/build-and-deploy-fallback.yml b/.github/workflows/build-and-deploy-fallback.yml deleted file mode 100644 index affe15e..0000000 --- a/.github/workflows/build-and-deploy-fallback.yml +++ /dev/null @@ -1,101 +0,0 @@ -# .github/workflows/build-and-deploy-fallback.yml - -name: Build and Deploy Fallback - -on: - push: - branches: ["gemini"] - -jobs: - build-and-deploy: - permissions: - contents: read - packages: write - - name: Build Images and Deploy to Server - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set repo name to lowercase - id: repo_name - run: echo "name=$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Create web-app .env file - run: echo 'GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}' > web-app/.env - - - name: Build and push web-app image πŸš€ - uses: docker/build-push-action@v6 - with: - context: ./web-app - push: true - tags: ghcr.io/${{ steps.repo_name.outputs.name }}/web-app:${{ github.sha }} - cache-from: type=gha,scope=web-app - cache-to: type=gha,mode=max,scope=web-app - - - name: Ensure remote deploy directory exists - uses: appleboy/ssh-action@v1.0.3 - with: - host: ${{ secrets.SERVER_HOST }} - username: ${{ secrets.SERVER_USERNAME }} - key: ${{ secrets.SSH_PRIVATE_KEY }} - script: | - mkdir -p /home/github-actions/codered-astra - - - name: Upload compose files to server - uses: appleboy/scp-action@v0.1.7 - with: - host: ${{ secrets.SERVER_HOST }} - username: ${{ secrets.SERVER_USERNAME }} - key: ${{ secrets.SSH_PRIVATE_KEY }} - source: "docker-compose.yml,docker-compose.prod.yml" - target: "/home/github-actions/codered-astra/" - - - name: Deploy to server via SSH ☁️ - uses: appleboy/ssh-action@v1.0.3 - env: - RUNNER_GH_ACTOR: ${{ github.actor }} - RUNNER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - host: ${{ secrets.SERVER_HOST }} - username: ${{ secrets.SERVER_USERNAME }} - key: ${{ secrets.SSH_PRIVATE_KEY }} - # pass selected env vars to the remote shell so docker login works - envs: RUNNER_GITHUB_TOKEN,RUNNER_GH_ACTOR - debug: true - script: | - cd /home/github-actions/codered-astra - chmod -R o+rX rust-engine/demo-data - # wrapper to support both Docker Compose v2 and legacy v1 - compose() { docker compose "$@" || docker-compose "$@"; } - # Log in to GHCR using the run's GITHUB_TOKEN so compose can pull images. - if [ -n "$RUNNER_GITHUB_TOKEN" ] && [ -n "$RUNNER_GH_ACTOR" ]; then - echo "$RUNNER_GITHUB_TOKEN" | docker login ghcr.io -u "$RUNNER_GH_ACTOR" --password-stdin || true - fi - export REPO_NAME_LOWER='${{ steps.repo_name.outputs.name }}' - export GEMINI_API_KEY='${{ secrets.GEMINI_API_KEY }}' - export MYSQL_DATABASE='${{ secrets.MYSQL_DATABASE }}' - export MYSQL_USER='${{ secrets.MYSQL_USER }}' - export MYSQL_PASSWORD='${{ secrets.MYSQL_PASSWORD }}' - export MYSQL_ROOT_PASSWORD='${{ secrets.MYSQL_ROOT_PASSWORD }}' - export IMAGE_TAG=${{ github.sha }} - # Stop and remove old containers before pulling new images - compose -f docker-compose.prod.yml down - # Clear previous logs for a clean deployment log - : > ~/astra-logs/astra-errors.log || true - compose -f docker-compose.prod.yml pull - compose -f docker-compose.prod.yml up -d - # Security hygiene: remove GHCR credentials after pulling - docker logout ghcr.io || true \ No newline at end of file diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index e5aa3c3..d609f17 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -69,7 +69,7 @@ jobs: host: ${{ secrets.SERVER_HOST }} username: ${{ secrets.SERVER_USERNAME }} key: ${{ secrets.SSH_PRIVATE_KEY }} - source: "docker-compose.yml,docker-compose.prod.yml" + source: "docker-compose.yml,docker-compose.prod.yml,rust-engine/demo-data" target: "/home/github-actions/codered-astra/" - name: Deploy to server via SSH ☁️ @@ -86,6 +86,7 @@ jobs: debug: true script: | cd /home/github-actions/codered-astra + chmod -R o+rX rust-engine/demo-data # wrapper to support both Docker Compose v2 and legacy v1 compose() { docker compose "$@" || docker-compose "$@"; } # Log in to GHCR using the run's GITHUB_TOKEN so compose can pull images. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md deleted file mode 100644 index b61318a..0000000 --- a/ARCHITECTURE.md +++ /dev/null @@ -1,268 +0,0 @@ -# CodeRED-Astra Architecture - -## Overview - -CodeRED-Astra is a Retrieval-Augmented Generation (RAG) system for querying ISS technical documentation using vector search, MySQL metadata storage, and Gemini AI for analysis and response generation. - -## System Components - -### 1. **Rust Backend** (`rust-engine/`) -High-performance Rust backend using Warp for HTTP, SQLx for MySQL, and Reqwest for external API calls. - -#### Modules - -**`main.rs`** - Entry point -- Initializes tracing, database, storage -- Spawns FileWorker and QueryWorker background tasks -- Serves API routes on port 8000 - -**`db.rs`** - Database initialization -- Connects to MySQL -- Creates `files` table (id, filename, path, description, pending_analysis, analysis_status) -- Creates `queries` table (id, status, payload, result, timestamps) - -**`api.rs`** - HTTP endpoints -- `POST /api/files` - Upload file (multipart/form-data) -- `POST /api/files/import-demo` - Bulk import from demo-data directory -- `GET /api/files/list` - List all files with status -- `GET /api/files/delete?id=` - Delete file and remove from Qdrant -- `POST /api/query/create` - Create new query (returns query ID) -- `GET /api/query/status?id=` - Check query status -- `GET /api/query/result?id=` - Get query result -- `GET /api/query/cancel?id=` - Cancel in-progress query - -**`file_worker.rs`** - File analysis pipeline -- **Background worker** that processes files with `pending_analysis = TRUE` -- Claims stale/queued files (requeues if stuck >10 min) -- **Stage 1**: Call Gemini 1.5 Flash for initial description -- **Stage 2**: Call Gemini 1.5 Pro for deep vector graph data (keywords, relationships) -- **Stage 3**: Generate embedding and upsert to Qdrant -- **Stage 4**: Mark file as ready (`pending_analysis = FALSE`, `analysis_status = 'Completed'`) -- Resumable: Can recover from crashes/restarts - -**`worker.rs`** - Query processing pipeline -- **Background worker** that processes queries with `status = 'Queued'` -- Requeues stale InProgress jobs (>10 min) -- **Stage 1**: Embed query text -- **Stage 2**: Search top-K similar vectors in Qdrant -- **Stage 3**: Fetch file metadata from MySQL (only completed files) -- **Stage 4**: Call Gemini to analyze relationships between files -- **Stage 5**: Call Gemini for final answer synthesis (strict: no speculation) -- **Stage 6**: Save results to database -- Supports cancellation checks between stages - -**`gemini_client.rs`** - Gemini API integration -- `generate_text(prompt)` - Text generation with model switching via GEMINI_MODEL env var -- `demo_text_embedding(text)` - Demo 64-dim embeddings (replace with real Gemini embeddings) -- Falls back to demo responses if GEMINI_API_KEY not set - -**`vector_db.rs`** - Qdrant client -- `ensure_files_collection(dim)` - Create 'files' collection with Cosine distance -- `upsert_point(id, vector)` - Store file embedding -- `search_top_k(vector, k)` - Find k nearest neighbors -- `delete_point(id)` - Remove file from index - -**`storage.rs`** - File storage utilities -- `storage_dir()` - Get storage path from ASTRA_STORAGE env or default `/app/storage` -- `ensure_storage_dir()` - Create storage directory if missing -- `save_file(filename, contents)` - Save file to storage -- `delete_file(path)` - Remove file from storage - -**`models.rs`** - Data structures -- `FileRecord` - File metadata (mirrors files table) -- `QueryRecord` - Query metadata (mirrors queries table) -- `QueryStatus` enum - Queued, InProgress, Completed, Cancelled, Failed - -### 2. **Web App** (`web-app/`) -React + Vite frontend with Express backend for API proxying. - -#### Backend (`server.mjs`) -- Express server that proxies API calls to rust-engine:8000 -- Serves React static build from `/dist` -- **Why needed**: Docker networking - React can't call rust-engine directly from browser - -#### Frontend (`src/`) -- `App.jsx` - Main chat interface component -- `components/ui/chat/chat-header.jsx` - Header with debug-only "Seed Demo Data" button (visible with `?debug=1`) -- Calls `/api/files/import-demo` endpoint to bulk-load ISS PDFs - -### 3. **MySQL Database** -Two tables for metadata storage: - -**`files` table** -```sql -id VARCHAR(36) PRIMARY KEY -filename TEXT NOT NULL -path TEXT NOT NULL -description TEXT -created_at DATETIME DEFAULT CURRENT_TIMESTAMP -pending_analysis BOOLEAN DEFAULT TRUE -analysis_status VARCHAR(32) DEFAULT 'Queued' -``` - -**`queries` table** -```sql -id VARCHAR(36) PRIMARY KEY -status VARCHAR(32) NOT NULL -payload JSON -result JSON -created_at DATETIME DEFAULT CURRENT_TIMESTAMP -updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -``` - -### 4. **Qdrant Vector Database** -- Collection: `files` -- Dimension: 64 (demo) - replace with real Gemini embedding dimension -- Distance: Cosine similarity -- Stores file embeddings for semantic search - -### 5. **Demo Data** (`rust-engine/demo-data/`) -~20 ISS technical PDFs organized by subsystem: -- Electrical Power System (EPS) -- Environmental Control & Life Support (ECLSS) -- Command & Data Handling (C&DH) -- Structures & Mechanisms - -## Data Flow - -### File Upload & Analysis -``` -1. User uploads PDF β†’ POST /api/files -2. API saves file to storage, inserts DB record (pending_analysis=true) -3. FileWorker claims pending file -4. Gemini 1.5 Flash generates description -5. Gemini 1.5 Pro generates vector graph data -6. Embed text β†’ upsert to Qdrant -7. Mark file as ready (pending_analysis=false) -``` - -### Query Processing -``` -1. User submits query β†’ POST /api/query/create -2. API inserts query record (status='Queued') -3. QueryWorker claims queued query -4. Embed query text -5. Search Qdrant for top-K similar files -6. Fetch file metadata from MySQL -7. Gemini analyzes relationships between files -8. Gemini synthesizes final answer (no speculation) -9. Save results to database -``` - -## Deployment - -### Development (`docker-compose.yml`) -- Local testing with hot-reload -- Bind mounts for code - -### Production (`docker-compose.prod.yml`) -- Used by GitHub Actions for deployment -- Runs rust-engine as user "1004" (github-actions) -- Docker volume: `rust-storage` β†’ `/app/storage` -- Bind mount: `/var/www/codered-astra/rust-engine/demo-data` β†’ `/app/demo-data:ro` -- Environment variables: - - `ASTRA_STORAGE=/app/storage` - - `DEMO_DATA_DIR=/app/demo-data` - - `QDRANT_URL=http://qdrant:6333` - - `GEMINI_API_KEY=` - - `DATABASE_URL=mysql://astraadmin:password@mysql:3306/astra` - -## Key Design Decisions - -### 1. **Two-Stage Analysis (Flash β†’ Pro)** -- Flash is faster/cheaper for initial description -- Pro is better for deep analysis and relationship extraction -- Enables cost-effective scaling - -### 2. **Resumable Workers** -- Workers requeue stale jobs (>10 min in InProgress) -- Survives container restarts without data loss -- Atomic state transitions via SQL - -### 3. **Separation of Concerns** -- FileWorker: Makes files searchable -- QueryWorker: Answers user queries -- Independent scaling and failure isolation - -### 4. **Strict Answer Generation** -- Gemini prompted to not speculate -- Must state uncertainty when info is insufficient -- Prevents hallucination in critical ISS documentation - -### 5. **Demo Embeddings** -- Current: 64-dim deterministic embeddings from text hash -- Production: Replace with real Gemini text embeddings API -- Allows development/testing without embedding API credits - -## API Usage Examples - -### Upload File -```bash -curl -F "file=@document.pdf" http://localhost:3001/api/files -``` - -### Import Demo Data -```bash -curl -X POST http://localhost:3001/api/files/import-demo -``` - -### Create Query -```bash -curl -X POST http://localhost:3001/api/query/create \ - -H "Content-Type: application/json" \ - -d '{"q": "What is the voltage of the ISS main bus?", "top_k": 5}' -``` - -### Check Status -```bash -curl http://localhost:3001/api/query/status?id= -``` - -### Get Result -```bash -curl http://localhost:3001/api/query/result?id= -``` - -## Future Enhancements - -### High Priority -1. Real Gemini text embeddings (replace demo embeddings) -2. File status UI panel (show processing progress) -3. Health check endpoint (`/health`) -4. Data purge endpoint (clear all files/queries) - -### Medium Priority -1. Streaming query responses (SSE/WebSocket) -2. Query result caching -3. File chunking for large PDFs -4. User authentication - -### Low Priority -1. Multi-collection support (different document types) -2. Query history UI -3. File preview in chat -4. Export results to PDF - -## Troubleshooting - -### Storage Permission Errors -- Ensure `/app/storage` is owned by container user -- Docker volume must be writable by user 1004 in production - -### SQL Syntax Errors -- MySQL requires separate `CREATE TABLE` statements -- Cannot combine multiple DDL statements in one `sqlx::query()` - -### Qdrant Connection Issues -- Check QDRANT_URL environment variable -- Ensure qdrant service is running and healthy -- Verify network connectivity between containers - -### Worker Not Processing -- Check logs: `docker logs rust-engine` -- Verify database connectivity -- Look for stale InProgress jobs in queries/files tables - -## Demo Presentation (3 minutes) - -See `rust-engine/DEMODETAILS.md` for curated demo script with example queries. diff --git a/QUICK_REFERENCE.md b/QUICK_REFERENCE.md deleted file mode 100644 index c694cfa..0000000 --- a/QUICK_REFERENCE.md +++ /dev/null @@ -1,219 +0,0 @@ -# CodeRED-Astra Quick Reference - -## System Overview - -**Two-worker architecture for ISS document RAG:** - -1. **FileWorker**: Analyzes uploaded files (Flash β†’ Pro β†’ Embed β†’ Qdrant) -2. **QueryWorker**: Answers queries (Embed β†’ Search β†’ Relationships β†’ Answer) - -Both workers are **resumable** and automatically recover from crashes. - -## Core Data Flow - -``` -Upload PDF β†’ Storage β†’ MySQL (pending) β†’ FileWorker β†’ Qdrant β†’ MySQL (ready) - ↓ -User Query β†’ MySQL (queued) β†’ QueryWorker β†’ Search Qdrant β†’ Gemini β†’ Result -``` - -## Module Map - -| Module | Purpose | Key Functions | -|--------|---------|---------------| -| `main.rs` | Entry point | Spawns workers, serves API | -| `db.rs` | Database init | Creates files/queries tables | -| `api.rs` | HTTP endpoints | Upload, list, delete, query CRUD | -| `file_worker.rs` | File analysis | Flashβ†’Proβ†’embedβ†’upsert | -| `worker.rs` | Query processing | Searchβ†’relationshipsβ†’answer | -| `gemini_client.rs` | AI integration | Text generation, embeddings | -| `vector_db.rs` | Qdrant client | Upsert, search, delete | -| `storage.rs` | File management | Save/delete files | -| `models.rs` | Data structures | FileRecord, QueryRecord | - -## API Endpoints - -### Files -- `POST /api/files` - Upload file -- `POST /api/files/import-demo?force=1` - Bulk import demo PDFs -- `GET /api/files/list` - List all files with status -- `GET /api/files/delete?id=` - Delete file - -### Queries -- `POST /api/query/create` - Create query -- `GET /api/query/status?id=` - Check status -- `GET /api/query/result?id=` - Get result -- `GET /api/query/cancel?id=` - Cancel query - -## Database Schema - -### files -- `id` - UUID primary key -- `filename` - Original filename -- `path` - Storage path -- `description` - Gemini Flash description -- `pending_analysis` - FALSE when ready for search -- `analysis_status` - Queued/InProgress/Completed/Failed - -### queries -- `id` - UUID primary key -- `status` - Queued/InProgress/Completed/Cancelled/Failed -- `payload` - JSON query params `{"q": "...", "top_k": 5}` -- `result` - JSON result `{"summary": "...", "related_files": [...], "relationships": "...", "final_answer": "..."}` - -## Environment Variables - -### Required -- `GEMINI_API_KEY` - Gemini API key -- `DATABASE_URL` - MySQL connection string -- `QDRANT_URL` - Qdrant URL (default: http://qdrant:6333) - -### Optional -- `ASTRA_STORAGE` - Storage directory (default: /app/storage) -- `DEMO_DATA_DIR` - Demo data directory (default: /app/demo-data) -- `GEMINI_MODEL` - Override Gemini model (default: gemini-1.5-pro) - -## Worker States - -### FileWorker -1. **Queued** - File uploaded, awaiting processing -2. **InProgress** - Currently being analyzed -3. **Completed** - Ready for search (pending_analysis=FALSE) -4. **Failed** - Error during processing - -### QueryWorker -1. **Queued** - Query created, awaiting processing -2. **InProgress** - Currently searching/analyzing -3. **Completed** - Result available -4. **Cancelled** - User cancelled -5. **Failed** - Error during processing - -## Gemini Prompts - -### FileWorker Stage 1 (Flash) -``` -Describe the file '{filename}' and extract all key components, keywords, -and details for later vectorization. Be comprehensive and factual. -``` - -### FileWorker Stage 2 (Pro) -``` -Given the file '{filename}' and its description: {desc} -Generate a set of vector graph data (keywords, use cases, relationships) -that can be used for broad and precise search. Only include what is -directly supported by the file. -``` - -### QueryWorker Stage 4 (Relationships) -``` -You are an assistant analyzing relationships STRICTLY within the provided files. -Query: {query} -Files: {file_list} -Tasks: -1) Summarize key details from the files relevant to the query. -2) Describe relationships and linkages strictly supported by these files. -3) List important follow-up questions that could be answered only using the provided files. -Rules: Do NOT guess or invent. If information is insufficient in the files, explicitly state that. -``` - -### QueryWorker Stage 5 (Final Answer) -``` -You are to compose a final answer to the user query using only the information from the files. -Query: {query} -Files considered: {file_list} -Relationship analysis: {relationships} -Requirements: -- Use only information present in the files and analysis above. -- If the answer is uncertain or cannot be determined from the files, clearly state that limitation. -- Avoid speculation or assumptions. -Provide a concise, structured answer. -``` - -## Docker Architecture - -### Services -- **rust-engine** - Warp API + workers (port 8000) -- **web-app** - Express + React (port 3001) -- **mysql** - MySQL 9.1 (port 3306) -- **qdrant** - Qdrant vector DB (port 6333) -- **phpmyadmin** - DB admin UI (port 8080) - -### Volumes (Production) -- `rust-storage:/app/storage` - File storage (writable) -- `/var/www/codered-astra/rust-engine/demo-data:/app/demo-data:ro` - Demo PDFs (read-only) -- `~/astra-logs:/var/log` - Log files - -## Common Issues - -### 1. SQL Syntax Error -**Problem**: `error near 'CREATE TABLE'` -**Cause**: Multiple CREATE TABLE in one query -**Fix**: Split into separate `sqlx::query()` calls - -### 2. Permission Denied -**Problem**: `Permission denied (os error 13)` -**Cause**: Container user can't write to storage -**Fix**: Use Docker volume, ensure ownership matches container user - -### 3. Worker Not Processing -**Problem**: Files/queries stuck in Queued -**Cause**: Worker crashed or not started -**Fix**: Check logs, ensure workers spawned in main.rs - -### 4. Qdrant Connection Failed -**Problem**: `qdrant upsert/search failed` -**Cause**: Qdrant not running or wrong URL -**Fix**: Verify QDRANT_URL, check qdrant container health - -## Development Commands - -```bash -# Build and run locally -cd rust-engine -cargo build -cargo run - -# Check code -cargo check - -# Run with logs -RUST_LOG=info cargo run - -# Docker compose (dev) -docker-compose up --build - -# Docker compose (production) -docker-compose -f docker-compose.prod.yml up -d - -# View logs -docker logs rust-engine -f - -# Rebuild single service -docker-compose build rust-engine -docker-compose up -d rust-engine -``` - -## Testing Flow - -1. Start services: `docker-compose up -d` -2. Import demo data: `curl -X POST http://localhost:3001/api/files/import-demo` -3. Wait for FileWorker to complete (~30 seconds for 20 files) -4. Check file status: `curl http://localhost:3001/api/files/list` -5. Create query: `curl -X POST http://localhost:3001/api/query/create -H "Content-Type: application/json" -d '{"q": "ISS main bus voltage", "top_k": 5}'` -6. Check status: `curl http://localhost:3001/api/query/status?id=` -7. Get result: `curl http://localhost:3001/api/query/result?id=` - -## Performance Notes - -- FileWorker: ~1-2 sec per file (demo embeddings) -- QueryWorker: ~3-5 sec per query (search + 2 Gemini calls) -- Qdrant search: <100ms for 1000s of vectors -- MySQL queries: <10ms for simple selects - -## Security Considerations - -- Store GEMINI_API_KEY in GitHub Secrets (production) -- Use environment variables for all credentials -- Don't commit `.env` files -- Restrict phpmyadmin to internal network only -- Use HTTPS in production deployment diff --git a/README.md b/README.md deleted file mode 100644 index b0b6190..0000000 --- a/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# CodeRED-Astra πŸš€ - -A hackathon-ready project with React frontend and Rust backend engine. - -## Quick Start - -```bash -# 1. Setup environment -cp .env.example .env -# Edit .env with your credentials - -# 2. Start everything with Docker -docker-compose up --build - -# 3. Access your app -# Frontend: http://localhost -# API: http://localhost:8000 -# Database Admin: http://127.0.0.1:8080 -``` - -## Development - -**Frontend (React + Vite)**: -```bash -cd web-app -npm install -npm run dev # http://localhost:5173 -``` - -**Backend (Rust)**: -```bash -cd rust-engine -cargo run # http://localhost:8000 -``` - -## Architecture - -- **Frontend**: React 18 + Vite + Tailwind CSS -- **Backend**: Rust + Warp + SQLx -- **Database**: MySQL 8.0 + phpMyAdmin -- **API**: RESTful endpoints with CORS enabled -- **Docker**: Full containerization for easy deployment - -## Project Structure - -``` -β”œβ”€β”€ web-app/ # React frontend -β”‚ β”œβ”€β”€ src/ -β”‚ β”‚ β”œβ”€β”€ App.jsx # Main component -β”‚ β”‚ └── main.jsx # Entry point -β”‚ └── Dockerfile -β”œβ”€β”€ rust-engine/ # Rust backend -β”‚ β”œβ”€β”€ src/ -β”‚ β”‚ └── main.rs # API server -β”‚ └── Dockerfile -β”œβ”€β”€ docker-compose.yml # Full stack orchestration -└── .env.example # Environment template -``` - -## Team Workflow - -- **Frontend devs**: Work in `web-app/src/`, use `/api/*` for backend calls -- **Backend devs**: Work in `rust-engine/src/`, add endpoints to main.rs -- **Database**: Access phpMyAdmin at http://127.0.0.1:8080 - -## Features - -βœ… Hot reload for both frontend and backend -βœ… Automatic API proxying from React to Rust -βœ… Database connection with graceful fallback -βœ… CORS configured for cross-origin requests -βœ… Production-ready Docker containers -βœ… Health monitoring and status dashboard - -Ready for your hackathon! See `DEVELOPMENT.md` for detailed setup instructions. diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml deleted file mode 100644 index 5a7e19d..0000000 --- a/docker-compose.prod.yml +++ /dev/null @@ -1,8 +0,0 @@ -services: - fallback-web-app: - image: ghcr.io/${REPO_NAME_LOWER}/web-app:${IMAGE_TAG} - restart: always - ports: - - "127.0.0.1:3034:3000" - environment: - - GEMINI_API_KEY=${GEMINI_API_KEY} diff --git a/docker-compose.yml b/docker-compose.yml index 78e4c61..393e5d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,8 @@ services: - DATABASE_URL=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@mysql:3306/${MYSQL_DATABASE} - RUST_ENGINE_URL=http://rust-engine:8000 - GEMINI_API_KEY=${GEMINI_API_KEY} + volumes: + - rust-storage:/app/storage:ro depends_on: - mysql # <-- Updated dependency - rust-engine diff --git a/web-app/Dockerfile b/frontend/Dockerfile similarity index 92% rename from web-app/Dockerfile rename to frontend/Dockerfile index 9552c3f..5a98079 100644 --- a/web-app/Dockerfile +++ b/frontend/Dockerfile @@ -2,7 +2,7 @@ FROM node:23-alpine WORKDIR /app COPY package*.json ./ -RUN npm ci +RUN npm i COPY . . RUN npm run format && npm run build EXPOSE 3000 diff --git a/frontend/README.md b/frontend/README.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/frontend/README.md @@ -0,0 +1 @@ + diff --git a/web-app/eslint.config.js b/frontend/eslint.config.js similarity index 51% rename from web-app/eslint.config.js rename to frontend/eslint.config.js index 53a87c7..050e89d 100644 --- a/web-app/eslint.config.js +++ b/frontend/eslint.config.js @@ -1,18 +1,15 @@ import js from "@eslint/js"; import globals from "globals"; -import reactHooks from "eslint-plugin-react-hooks"; // Or import { configs as reactHooks } from "eslint-plugin-react-hooks"; +import reactHooks from "eslint-plugin-react-hooks"; import reactRefresh from "eslint-plugin-react-refresh"; -import { defineConfig, globalIgnores } from "eslint/config"; -export default defineConfig([ - globalIgnores(["dist"]), +export default [ { - files: ["**/*{js,jsx}"], - extends: [ - js.configs.recommended, - reactHooks.configs["recommended-latest"], - reactRefresh.configs.vite, - ], + ignores: ["dist/**"], + }, + js.configs.recommended, + { + files: ["**/*.{js,jsx}"], languageOptions: { ecmaVersion: 2020, globals: globals.browser, @@ -22,8 +19,17 @@ export default defineConfig([ sourceType: "module", }, }, + plugins: { + "react-hooks": reactHooks, + "react-refresh": reactRefresh, + }, rules: { + ...reactHooks.configs.recommended.rules, + "react-refresh/only-export-components": [ + "warn", + { allowConstantExport: true }, + ], "no-unused-vars": ["error", { varsIgnorePattern: "^[A-Z_]" }], }, }, -]); +]; diff --git a/web-app/index.html b/frontend/index.html similarity index 100% rename from web-app/index.html rename to frontend/index.html diff --git a/web-app/package-lock.json b/frontend/package-lock.json similarity index 99% rename from web-app/package-lock.json rename to frontend/package-lock.json index aa53229..7dca15a 100644 --- a/web-app/package-lock.json +++ b/frontend/package-lock.json @@ -33,6 +33,7 @@ }, "devDependencies": { "@eslint/js": "^9.38.0", + "daisyui": "^5.4.7", "eslint": "^9.38.0", "eslint-plugin-import": "^2.32.0", "eslint-plugin-react": "^7.37.5", @@ -2906,6 +2907,16 @@ "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", "license": "MIT" }, + "node_modules/daisyui": { + "version": "5.4.7", + "resolved": "https://registry.npmjs.org/daisyui/-/daisyui-5.4.7.tgz", + "integrity": "sha512-2wYO61vTPCXk7xEBgnzLZAYoE0xS5IRLu/GSq0vORpB+cTrtubdx69NnA0loc0exvCY1s2fYL4lGZtFHe2ohNQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/saadeghi/daisyui?sponsor=1" + } + }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", diff --git a/web-app/package.json b/frontend/package.json similarity index 98% rename from web-app/package.json rename to frontend/package.json index 5f5becb..b1476ae 100644 --- a/web-app/package.json +++ b/frontend/package.json @@ -40,6 +40,7 @@ "packageManager": ">=npm@10.9.0", "devDependencies": { "@eslint/js": "^9.38.0", + "daisyui": "^5.4.7", "eslint": "^9.38.0", "eslint-plugin-import": "^2.32.0", "eslint-plugin-react": "^7.37.5", diff --git a/web-app/src/app/index.jsx b/frontend/src/app/index.jsx similarity index 93% rename from web-app/src/app/index.jsx rename to frontend/src/app/index.jsx index 86a06e6..ece1d51 100644 --- a/web-app/src/app/index.jsx +++ b/frontend/src/app/index.jsx @@ -5,6 +5,7 @@ function App() { return (
+
); } diff --git a/frontend/src/components/layouts/chat-layout.jsx b/frontend/src/components/layouts/chat-layout.jsx new file mode 100644 index 0000000..951d54c --- /dev/null +++ b/frontend/src/components/layouts/chat-layout.jsx @@ -0,0 +1,244 @@ +import React, { + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import ChatHeader from "src/components/ui/chat/chat-header"; +import ChatWindow from "src/components/ui/chat/chat-window"; +import MessageInput from "src/components/ui/chat/message-input"; +import { + createQuery, + getQueryResult, + getQueryStatus, + listFiles, +} from "src/lib/api"; + +const createId = () => + globalThis.crypto?.randomUUID?.() ?? `id-${Date.now()}-${Math.random()}`; + +const INTRO_MESSAGE = { + id: "intro", + role: "assistant", + content: + "Ask me about the demo PDFs and I'll respond with the best matches pulled from the processed files.", +}; + +export default function ChatLayout() { + const [messages, setMessages] = useState([INTRO_MESSAGE]); + const [isProcessing, setIsProcessing] = useState(false); + const [files, setFiles] = useState([]); + const [errorToast, setErrorToast] = useState(""); + const pollAbortRef = useRef(null); + + const showError = useCallback((message) => { + setErrorToast(message); + window.setTimeout(() => setErrorToast(""), 5000); + }, []); + + const refreshFiles = useCallback(async () => { + try { + const latest = await listFiles(); + setFiles(latest); + } catch (error) { + showError(error.message ?? "Failed to load files"); + } + }, [showError]); + + useEffect(() => { + refreshFiles(); + }, [refreshFiles]); + + useEffect(() => { + return () => { + if (pollAbortRef.current) { + pollAbortRef.current.aborted = true; + } + }; + }, []); + + const buildAssistantMarkdown = useCallback((result) => { + if (!result || typeof result !== "object") { + return "I could not find a response for that request."; + } + + const finalAnswer = result.final_answer?.trim(); + const relationships = result.relationships?.trim(); + const relatedFiles = Array.isArray(result.related_files) + ? result.related_files + : []; + + const fileLines = relatedFiles + .filter((f) => f && typeof f === "object") + .map((file) => { + const filename = file.filename || file.id || "download"; + const storageUrl = file.storage_url || `/storage/${filename}`; + const linkTarget = storageUrl.startsWith("/storage/") + ? `/storage/${encodeURIComponent(storageUrl.replace("/storage/", ""))}` + : storageUrl; + const description = file.description?.trim(); + const score = + typeof file.score === "number" + ? ` _(score: ${file.score.toFixed(3)})_` + : ""; + const detail = description ? ` β€” ${description}` : ""; + return `- [${filename}](${linkTarget})${detail}${score}`; + }); + + let content = + finalAnswer || + "I could not determine an answer from the indexed documents yet."; + + if (fileLines.length) { + content += `\n\n**Related Files**\n${fileLines.join("\n")}`; + } + + if (relationships && relationships !== finalAnswer) { + content += `\n\n---\n${relationships}`; + } + + if (!fileLines.length && (!finalAnswer || finalAnswer.length < 10)) { + content += + "\n\n_No analyzed documents matched yet. Try seeding demo data or wait for processing to finish._"; + } + + return content; + }, []); + + const waitForResult = useCallback(async (id) => { + const abortState = { aborted: false }; + pollAbortRef.current = abortState; + const timeoutMs = 120_000; + const intervalMs = 1_500; + const started = Date.now(); + + while (!abortState.aborted) { + if (Date.now() - started > timeoutMs) { + throw new Error("Timed out waiting for the query to finish"); + } + + const statusPayload = await getQueryStatus(id); + const status = statusPayload?.status; + + if (status === "Completed") { + const resultPayload = await getQueryResult(id); + return resultPayload?.result; + } + + if (status === "Failed") { + const resultPayload = await getQueryResult(id); + const reason = resultPayload?.result?.error || "Query failed"; + throw new Error(reason); + } + + if (status === "Cancelled") { + throw new Error("Query was cancelled"); + } + + if (status === "not_found") { + throw new Error("Query was not found"); + } + + await new Promise((resolve) => window.setTimeout(resolve, intervalMs)); + } + + throw new Error("Query polling was aborted"); + }, []); + + const handleSend = useCallback( + async (text) => { + if (isProcessing) { + showError("Please wait for the current response to finish."); + return; + } + + const userEntry = { + id: createId(), + role: "user", + content: text, + }; + setMessages((prev) => [...prev, userEntry]); + + const placeholderId = createId(); + setMessages((prev) => [ + ...prev, + { + id: placeholderId, + role: "assistant", + content: "_Analyzing indexed documents..._", + pending: true, + }, + ]); + + setIsProcessing(true); + + try { + const payload = { q: text, top_k: 5 }; + const created = await createQuery(payload); + const result = await waitForResult(created.id); + const content = buildAssistantMarkdown(result); + setMessages((prev) => + prev.map((message) => + message.id === placeholderId + ? { ...message, content, pending: false } + : message, + ), + ); + } catch (error) { + const message = error?.message || "Something went wrong."; + setMessages((prev) => + prev.map((entry) => + entry.id === placeholderId + ? { + ...entry, + content: `⚠️ ${message}`, + pending: false, + error: true, + } + : entry, + ), + ); + showError(message); + } finally { + pollAbortRef.current = null; + setIsProcessing(false); + refreshFiles(); + } + }, + [ + isProcessing, + showError, + refreshFiles, + waitForResult, + buildAssistantMarkdown, + ], + ); + + const handleDeleteAll = useCallback(() => { + if (!window.confirm("Delete all messages?")) { + return; + } + setMessages([INTRO_MESSAGE]); + }, []); + + const latestFileSummary = useMemo(() => { + if (!files.length) return "No files indexed yet."; + const pending = files.filter((f) => f.pending_analysis).length; + const ready = files.length - pending; + return `${ready} ready β€’ ${pending} processing`; + }, [files]); + + return ( +
+ + + +
+ ); +} diff --git a/frontend/src/components/ui/button/delete-button.jsx b/frontend/src/components/ui/button/delete-button.jsx new file mode 100644 index 0000000..37e008e --- /dev/null +++ b/frontend/src/components/ui/button/delete-button.jsx @@ -0,0 +1,19 @@ +import { Flame } from "lucide-react"; +import { motion } from "motion/react"; + +export default function FlameButton({ onClick, disabled = false }) { + return ( + + + + ); +} diff --git a/web-app/src/components/ui/button/down-button.jsx b/frontend/src/components/ui/button/down-button.jsx similarity index 100% rename from web-app/src/components/ui/button/down-button.jsx rename to frontend/src/components/ui/button/down-button.jsx diff --git a/web-app/src/components/ui/button/schematic-button.jsx b/frontend/src/components/ui/button/schematic-button.jsx similarity index 100% rename from web-app/src/components/ui/button/schematic-button.jsx rename to frontend/src/components/ui/button/schematic-button.jsx diff --git a/frontend/src/components/ui/chat/chat-header.jsx b/frontend/src/components/ui/chat/chat-header.jsx new file mode 100644 index 0000000..b5d9a63 --- /dev/null +++ b/frontend/src/components/ui/chat/chat-header.jsx @@ -0,0 +1,89 @@ +import React, { useEffect, useMemo, useState } from "react"; +import { motion } from "motion/react"; +import { Rocket } from "lucide-react"; +import DeleteButton from "src/components/ui/button/delete-button"; +import SchematicButton from "src/components/ui/button/schematic-button"; + +export default function ChatHeader({ + title = "Title of Chat", + onClear, + busy = false, + fileSummary, + errorMessage, +}) { + const isDebug = useMemo(() => { + const p = new URLSearchParams(window.location.search); + return p.get("debug") === "1"; + }, []); + const [ingesting, setIngesting] = useState(false); + const [toast, setToast] = useState(""); + const [externalToast, setExternalToast] = useState(""); + + useEffect(() => { + if (!errorMessage) return; + setExternalToast(errorMessage); + const timer = window.setTimeout(() => setExternalToast(""), 5000); + return () => window.clearTimeout(timer); + }, [errorMessage]); + + async function triggerDemoIngest() { + try { + setIngesting(true); + const res = await fetch("/api/files/import-demo", { method: "POST" }); + const json = await res.json().catch(() => ({})); + const imported = json.imported ?? "?"; + const skipped = json.skipped ?? "?"; + const summary = `Imported: ${imported}, Skipped: ${skipped}`; + setToast(json.error ? `${summary} - ${json.error}` : summary); + setTimeout(() => setToast(""), 4000); + } catch (e) { + setToast("Import failed"); + setTimeout(() => setToast(""), 4000); + } finally { + setIngesting(false); + } + } + + return ( +
+
+
+ +
+

+ {title} +

+ {fileSummary && ( +
+ {fileSummary} +
+ )} + + {isDebug && ( + + + {ingesting ? "Seeding…" : "Seed Demo Data"} + + )} +
+
+ {toast && ( +
+ {toast} +
+ )} + {externalToast && ( +
+ {externalToast} +
+ )} +
+
+ ); +} diff --git a/web-app/src/components/ui/chat/chat-window.jsx b/frontend/src/components/ui/chat/chat-window.jsx similarity index 62% rename from web-app/src/components/ui/chat/chat-window.jsx rename to frontend/src/components/ui/chat/chat-window.jsx index 2b78ce2..1252d54 100644 --- a/web-app/src/components/ui/chat/chat-window.jsx +++ b/frontend/src/components/ui/chat/chat-window.jsx @@ -4,10 +4,11 @@ import { MARKDOWN_COMPONENTS } from "src/config/markdown"; function MessageBubble({ message }) { const isUser = message.role === "user"; + const isError = !!message.error; return (
{isUser ? (
{message.content}
@@ -22,12 +23,21 @@ function MessageBubble({ message }) { } export default function ChatWindow({ messages }) { + const bottomRef = useRef(null); + + useEffect(() => { + if (bottomRef.current) { + bottomRef.current.scrollIntoView({ behavior: "smooth" }); + } + }, [messages]); + return (
{messages.map((m, i) => ( - + ))} +
); diff --git a/frontend/src/components/ui/chat/message-input.jsx b/frontend/src/components/ui/chat/message-input.jsx new file mode 100644 index 0000000..0f49a7f --- /dev/null +++ b/frontend/src/components/ui/chat/message-input.jsx @@ -0,0 +1,119 @@ +import React, { useState, useRef, useEffect } from "react"; +import DownButton from "src/components/ui/button/down-button"; +import { motion } from "motion/react"; +import { BotMessageSquare } from "lucide-react"; + +export default function MessageInput({ onSend, disabled = false }) { + const [text, setText] = useState(""); + const textareaRef = useRef(null); + + useEffect(() => { + // ensure correct initial height + if (textareaRef.current) textareaRef.current.style.height = "auto"; + }, []); + + async function handleSubmit(e) { + e.preventDefault(); + if (!text.trim() || disabled) return; + onSend(text.trim()); + + // create query on backend + try { + if (onMessage) + onMessage("assistant", "Queued: sending request to server..."); + const createRes = await fetch(`/api/query/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ q: text, top_k: 5 }), + }); + const createJson = await createRes.json(); + const id = createJson.id; + if (!id) throw new Error("no id returned"); + + // poll status + let status = "Queued"; + if (onMessage) onMessage("assistant", `Status: ${status}`); + while (status !== "Completed" && status !== "Failed") { + await new Promise((r) => setTimeout(r, 1000)); + const sRes = await fetch(`/api/query/status?id=${id}`); + const sJson = await sRes.json(); + status = sJson.status; + if (onMessage) onMessage("assistant", `Status: ${status}`); + if (status === "Cancelled") break; + } + + if (status === "Completed") { + const resultRes = await fetch(`/api/query/result?id=${id}`); + const resultJson = await resultRes.json(); + const final = + resultJson?.result?.final_answer || + JSON.stringify(resultJson?.result || {}); + if (onMessage) onMessage("assistant", final); + } else { + if (onMessage) + onMessage("assistant", `Query status ended as: ${status}`); + } + } catch (err) { + console.error(err); + if (onMessage) onMessage("assistant", `Error: ${err.message}`); + } + + setText(""); + } + + return ( +
+