diff --git a/.github/workflows/build-and-deploy-fallback.yml b/.github/workflows/build-and-deploy-fallback.yml new file mode 100644 index 0000000..affe15e --- /dev/null +++ b/.github/workflows/build-and-deploy-fallback.yml @@ -0,0 +1,101 @@ +# .github/workflows/build-and-deploy-fallback.yml + +name: Build and Deploy Fallback + +on: + push: + branches: ["gemini"] + +jobs: + build-and-deploy: + permissions: + contents: read + packages: write + + name: Build Images and Deploy to Server + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set repo name to lowercase + id: repo_name + run: echo "name=$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Create web-app .env file + run: echo 'GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}' > web-app/.env + + - name: Build and push web-app image πŸš€ + uses: docker/build-push-action@v6 + with: + context: ./web-app + push: true + tags: ghcr.io/${{ steps.repo_name.outputs.name }}/web-app:${{ github.sha }} + cache-from: type=gha,scope=web-app + cache-to: type=gha,mode=max,scope=web-app + + - name: Ensure remote deploy directory exists + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ secrets.SERVER_HOST }} + username: ${{ secrets.SERVER_USERNAME }} + key: ${{ secrets.SSH_PRIVATE_KEY }} + script: | + mkdir -p /home/github-actions/codered-astra + + - name: Upload compose files to server + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.SERVER_HOST }} + username: ${{ secrets.SERVER_USERNAME }} + key: ${{ secrets.SSH_PRIVATE_KEY }} + source: "docker-compose.yml,docker-compose.prod.yml" + target: "/home/github-actions/codered-astra/" + + - name: Deploy to server via SSH ☁️ + uses: appleboy/ssh-action@v1.0.3 + env: + RUNNER_GH_ACTOR: ${{ github.actor }} + RUNNER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + host: ${{ secrets.SERVER_HOST }} + username: ${{ secrets.SERVER_USERNAME }} + key: ${{ secrets.SSH_PRIVATE_KEY }} + # pass selected env vars to the remote shell so docker login works + envs: RUNNER_GITHUB_TOKEN,RUNNER_GH_ACTOR + debug: true + script: | + cd /home/github-actions/codered-astra + chmod -R o+rX rust-engine/demo-data + # wrapper to support both Docker Compose v2 and legacy v1 + compose() { docker compose "$@" || docker-compose "$@"; } + # Log in to GHCR using the run's GITHUB_TOKEN so compose can pull images. + if [ -n "$RUNNER_GITHUB_TOKEN" ] && [ -n "$RUNNER_GH_ACTOR" ]; then + echo "$RUNNER_GITHUB_TOKEN" | docker login ghcr.io -u "$RUNNER_GH_ACTOR" --password-stdin || true + fi + export REPO_NAME_LOWER='${{ steps.repo_name.outputs.name }}' + export GEMINI_API_KEY='${{ secrets.GEMINI_API_KEY }}' + export MYSQL_DATABASE='${{ secrets.MYSQL_DATABASE }}' + export MYSQL_USER='${{ secrets.MYSQL_USER }}' + export MYSQL_PASSWORD='${{ secrets.MYSQL_PASSWORD }}' + export MYSQL_ROOT_PASSWORD='${{ secrets.MYSQL_ROOT_PASSWORD }}' + export IMAGE_TAG=${{ github.sha }} + # Stop and remove old containers before pulling new images + compose -f docker-compose.prod.yml down + # Clear previous logs for a clean deployment log + : > ~/astra-logs/astra-errors.log || true + compose -f docker-compose.prod.yml pull + compose -f docker-compose.prod.yml up -d + # Security hygiene: remove GHCR credentials after pulling + docker logout ghcr.io || true \ No newline at end of file diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index d609f17..e5aa3c3 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -69,7 +69,7 @@ jobs: host: ${{ secrets.SERVER_HOST }} username: ${{ secrets.SERVER_USERNAME }} key: ${{ secrets.SSH_PRIVATE_KEY }} - source: "docker-compose.yml,docker-compose.prod.yml,rust-engine/demo-data" + source: "docker-compose.yml,docker-compose.prod.yml" target: "/home/github-actions/codered-astra/" - name: Deploy to server via SSH ☁️ @@ -86,7 +86,6 @@ jobs: debug: true script: | cd /home/github-actions/codered-astra - chmod -R o+rX rust-engine/demo-data # wrapper to support both Docker Compose v2 and legacy v1 compose() { docker compose "$@" || docker-compose "$@"; } # Log in to GHCR using the run's GITHUB_TOKEN so compose can pull images. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..b61318a --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,268 @@ +# CodeRED-Astra Architecture + +## Overview + +CodeRED-Astra is a Retrieval-Augmented Generation (RAG) system for querying ISS technical documentation using vector search, MySQL metadata storage, and Gemini AI for analysis and response generation. + +## System Components + +### 1. **Rust Backend** (`rust-engine/`) +High-performance Rust backend using Warp for HTTP, SQLx for MySQL, and Reqwest for external API calls. + +#### Modules + +**`main.rs`** - Entry point +- Initializes tracing, database, storage +- Spawns FileWorker and QueryWorker background tasks +- Serves API routes on port 8000 + +**`db.rs`** - Database initialization +- Connects to MySQL +- Creates `files` table (id, filename, path, description, pending_analysis, analysis_status) +- Creates `queries` table (id, status, payload, result, timestamps) + +**`api.rs`** - HTTP endpoints +- `POST /api/files` - Upload file (multipart/form-data) +- `POST /api/files/import-demo` - Bulk import from demo-data directory +- `GET /api/files/list` - List all files with status +- `GET /api/files/delete?id=` - Delete file and remove from Qdrant +- `POST /api/query/create` - Create new query (returns query ID) +- `GET /api/query/status?id=` - Check query status +- `GET /api/query/result?id=` - Get query result +- `GET /api/query/cancel?id=` - Cancel in-progress query + +**`file_worker.rs`** - File analysis pipeline +- **Background worker** that processes files with `pending_analysis = TRUE` +- Claims stale/queued files (requeues if stuck >10 min) +- **Stage 1**: Call Gemini 1.5 Flash for initial description +- **Stage 2**: Call Gemini 1.5 Pro for deep vector graph data (keywords, relationships) +- **Stage 3**: Generate embedding and upsert to Qdrant +- **Stage 4**: Mark file as ready (`pending_analysis = FALSE`, `analysis_status = 'Completed'`) +- Resumable: Can recover from crashes/restarts + +**`worker.rs`** - Query processing pipeline +- **Background worker** that processes queries with `status = 'Queued'` +- Requeues stale InProgress jobs (>10 min) +- **Stage 1**: Embed query text +- **Stage 2**: Search top-K similar vectors in Qdrant +- **Stage 3**: Fetch file metadata from MySQL (only completed files) +- **Stage 4**: Call Gemini to analyze relationships between files +- **Stage 5**: Call Gemini for final answer synthesis (strict: no speculation) +- **Stage 6**: Save results to database +- Supports cancellation checks between stages + +**`gemini_client.rs`** - Gemini API integration +- `generate_text(prompt)` - Text generation with model switching via GEMINI_MODEL env var +- `demo_text_embedding(text)` - Demo 64-dim embeddings (replace with real Gemini embeddings) +- Falls back to demo responses if GEMINI_API_KEY not set + +**`vector_db.rs`** - Qdrant client +- `ensure_files_collection(dim)` - Create 'files' collection with Cosine distance +- `upsert_point(id, vector)` - Store file embedding +- `search_top_k(vector, k)` - Find k nearest neighbors +- `delete_point(id)` - Remove file from index + +**`storage.rs`** - File storage utilities +- `storage_dir()` - Get storage path from ASTRA_STORAGE env or default `/app/storage` +- `ensure_storage_dir()` - Create storage directory if missing +- `save_file(filename, contents)` - Save file to storage +- `delete_file(path)` - Remove file from storage + +**`models.rs`** - Data structures +- `FileRecord` - File metadata (mirrors files table) +- `QueryRecord` - Query metadata (mirrors queries table) +- `QueryStatus` enum - Queued, InProgress, Completed, Cancelled, Failed + +### 2. **Web App** (`web-app/`) +React + Vite frontend with Express backend for API proxying. + +#### Backend (`server.mjs`) +- Express server that proxies API calls to rust-engine:8000 +- Serves React static build from `/dist` +- **Why needed**: Docker networking - React can't call rust-engine directly from browser + +#### Frontend (`src/`) +- `App.jsx` - Main chat interface component +- `components/ui/chat/chat-header.jsx` - Header with debug-only "Seed Demo Data" button (visible with `?debug=1`) +- Calls `/api/files/import-demo` endpoint to bulk-load ISS PDFs + +### 3. **MySQL Database** +Two tables for metadata storage: + +**`files` table** +```sql +id VARCHAR(36) PRIMARY KEY +filename TEXT NOT NULL +path TEXT NOT NULL +description TEXT +created_at DATETIME DEFAULT CURRENT_TIMESTAMP +pending_analysis BOOLEAN DEFAULT TRUE +analysis_status VARCHAR(32) DEFAULT 'Queued' +``` + +**`queries` table** +```sql +id VARCHAR(36) PRIMARY KEY +status VARCHAR(32) NOT NULL +payload JSON +result JSON +created_at DATETIME DEFAULT CURRENT_TIMESTAMP +updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +``` + +### 4. **Qdrant Vector Database** +- Collection: `files` +- Dimension: 64 (demo) - replace with real Gemini embedding dimension +- Distance: Cosine similarity +- Stores file embeddings for semantic search + +### 5. **Demo Data** (`rust-engine/demo-data/`) +~20 ISS technical PDFs organized by subsystem: +- Electrical Power System (EPS) +- Environmental Control & Life Support (ECLSS) +- Command & Data Handling (C&DH) +- Structures & Mechanisms + +## Data Flow + +### File Upload & Analysis +``` +1. User uploads PDF β†’ POST /api/files +2. API saves file to storage, inserts DB record (pending_analysis=true) +3. FileWorker claims pending file +4. Gemini 1.5 Flash generates description +5. Gemini 1.5 Pro generates vector graph data +6. Embed text β†’ upsert to Qdrant +7. Mark file as ready (pending_analysis=false) +``` + +### Query Processing +``` +1. User submits query β†’ POST /api/query/create +2. API inserts query record (status='Queued') +3. QueryWorker claims queued query +4. Embed query text +5. Search Qdrant for top-K similar files +6. Fetch file metadata from MySQL +7. Gemini analyzes relationships between files +8. Gemini synthesizes final answer (no speculation) +9. Save results to database +``` + +## Deployment + +### Development (`docker-compose.yml`) +- Local testing with hot-reload +- Bind mounts for code + +### Production (`docker-compose.prod.yml`) +- Used by GitHub Actions for deployment +- Runs rust-engine as user "1004" (github-actions) +- Docker volume: `rust-storage` β†’ `/app/storage` +- Bind mount: `/var/www/codered-astra/rust-engine/demo-data` β†’ `/app/demo-data:ro` +- Environment variables: + - `ASTRA_STORAGE=/app/storage` + - `DEMO_DATA_DIR=/app/demo-data` + - `QDRANT_URL=http://qdrant:6333` + - `GEMINI_API_KEY=` + - `DATABASE_URL=mysql://astraadmin:password@mysql:3306/astra` + +## Key Design Decisions + +### 1. **Two-Stage Analysis (Flash β†’ Pro)** +- Flash is faster/cheaper for initial description +- Pro is better for deep analysis and relationship extraction +- Enables cost-effective scaling + +### 2. **Resumable Workers** +- Workers requeue stale jobs (>10 min in InProgress) +- Survives container restarts without data loss +- Atomic state transitions via SQL + +### 3. **Separation of Concerns** +- FileWorker: Makes files searchable +- QueryWorker: Answers user queries +- Independent scaling and failure isolation + +### 4. **Strict Answer Generation** +- Gemini prompted to not speculate +- Must state uncertainty when info is insufficient +- Prevents hallucination in critical ISS documentation + +### 5. **Demo Embeddings** +- Current: 64-dim deterministic embeddings from text hash +- Production: Replace with real Gemini text embeddings API +- Allows development/testing without embedding API credits + +## API Usage Examples + +### Upload File +```bash +curl -F "file=@document.pdf" http://localhost:3001/api/files +``` + +### Import Demo Data +```bash +curl -X POST http://localhost:3001/api/files/import-demo +``` + +### Create Query +```bash +curl -X POST http://localhost:3001/api/query/create \ + -H "Content-Type: application/json" \ + -d '{"q": "What is the voltage of the ISS main bus?", "top_k": 5}' +``` + +### Check Status +```bash +curl http://localhost:3001/api/query/status?id= +``` + +### Get Result +```bash +curl http://localhost:3001/api/query/result?id= +``` + +## Future Enhancements + +### High Priority +1. Real Gemini text embeddings (replace demo embeddings) +2. File status UI panel (show processing progress) +3. Health check endpoint (`/health`) +4. Data purge endpoint (clear all files/queries) + +### Medium Priority +1. Streaming query responses (SSE/WebSocket) +2. Query result caching +3. File chunking for large PDFs +4. User authentication + +### Low Priority +1. Multi-collection support (different document types) +2. Query history UI +3. File preview in chat +4. Export results to PDF + +## Troubleshooting + +### Storage Permission Errors +- Ensure `/app/storage` is owned by container user +- Docker volume must be writable by user 1004 in production + +### SQL Syntax Errors +- MySQL requires separate `CREATE TABLE` statements +- Cannot combine multiple DDL statements in one `sqlx::query()` + +### Qdrant Connection Issues +- Check QDRANT_URL environment variable +- Ensure qdrant service is running and healthy +- Verify network connectivity between containers + +### Worker Not Processing +- Check logs: `docker logs rust-engine` +- Verify database connectivity +- Look for stale InProgress jobs in queries/files tables + +## Demo Presentation (3 minutes) + +See `rust-engine/DEMODETAILS.md` for curated demo script with example queries. diff --git a/QUICK_REFERENCE.md b/QUICK_REFERENCE.md new file mode 100644 index 0000000..c694cfa --- /dev/null +++ b/QUICK_REFERENCE.md @@ -0,0 +1,219 @@ +# CodeRED-Astra Quick Reference + +## System Overview + +**Two-worker architecture for ISS document RAG:** + +1. **FileWorker**: Analyzes uploaded files (Flash β†’ Pro β†’ Embed β†’ Qdrant) +2. **QueryWorker**: Answers queries (Embed β†’ Search β†’ Relationships β†’ Answer) + +Both workers are **resumable** and automatically recover from crashes. + +## Core Data Flow + +``` +Upload PDF β†’ Storage β†’ MySQL (pending) β†’ FileWorker β†’ Qdrant β†’ MySQL (ready) + ↓ +User Query β†’ MySQL (queued) β†’ QueryWorker β†’ Search Qdrant β†’ Gemini β†’ Result +``` + +## Module Map + +| Module | Purpose | Key Functions | +|--------|---------|---------------| +| `main.rs` | Entry point | Spawns workers, serves API | +| `db.rs` | Database init | Creates files/queries tables | +| `api.rs` | HTTP endpoints | Upload, list, delete, query CRUD | +| `file_worker.rs` | File analysis | Flashβ†’Proβ†’embedβ†’upsert | +| `worker.rs` | Query processing | Searchβ†’relationshipsβ†’answer | +| `gemini_client.rs` | AI integration | Text generation, embeddings | +| `vector_db.rs` | Qdrant client | Upsert, search, delete | +| `storage.rs` | File management | Save/delete files | +| `models.rs` | Data structures | FileRecord, QueryRecord | + +## API Endpoints + +### Files +- `POST /api/files` - Upload file +- `POST /api/files/import-demo?force=1` - Bulk import demo PDFs +- `GET /api/files/list` - List all files with status +- `GET /api/files/delete?id=` - Delete file + +### Queries +- `POST /api/query/create` - Create query +- `GET /api/query/status?id=` - Check status +- `GET /api/query/result?id=` - Get result +- `GET /api/query/cancel?id=` - Cancel query + +## Database Schema + +### files +- `id` - UUID primary key +- `filename` - Original filename +- `path` - Storage path +- `description` - Gemini Flash description +- `pending_analysis` - FALSE when ready for search +- `analysis_status` - Queued/InProgress/Completed/Failed + +### queries +- `id` - UUID primary key +- `status` - Queued/InProgress/Completed/Cancelled/Failed +- `payload` - JSON query params `{"q": "...", "top_k": 5}` +- `result` - JSON result `{"summary": "...", "related_files": [...], "relationships": "...", "final_answer": "..."}` + +## Environment Variables + +### Required +- `GEMINI_API_KEY` - Gemini API key +- `DATABASE_URL` - MySQL connection string +- `QDRANT_URL` - Qdrant URL (default: http://qdrant:6333) + +### Optional +- `ASTRA_STORAGE` - Storage directory (default: /app/storage) +- `DEMO_DATA_DIR` - Demo data directory (default: /app/demo-data) +- `GEMINI_MODEL` - Override Gemini model (default: gemini-1.5-pro) + +## Worker States + +### FileWorker +1. **Queued** - File uploaded, awaiting processing +2. **InProgress** - Currently being analyzed +3. **Completed** - Ready for search (pending_analysis=FALSE) +4. **Failed** - Error during processing + +### QueryWorker +1. **Queued** - Query created, awaiting processing +2. **InProgress** - Currently searching/analyzing +3. **Completed** - Result available +4. **Cancelled** - User cancelled +5. **Failed** - Error during processing + +## Gemini Prompts + +### FileWorker Stage 1 (Flash) +``` +Describe the file '{filename}' and extract all key components, keywords, +and details for later vectorization. Be comprehensive and factual. +``` + +### FileWorker Stage 2 (Pro) +``` +Given the file '{filename}' and its description: {desc} +Generate a set of vector graph data (keywords, use cases, relationships) +that can be used for broad and precise search. Only include what is +directly supported by the file. +``` + +### QueryWorker Stage 4 (Relationships) +``` +You are an assistant analyzing relationships STRICTLY within the provided files. +Query: {query} +Files: {file_list} +Tasks: +1) Summarize key details from the files relevant to the query. +2) Describe relationships and linkages strictly supported by these files. +3) List important follow-up questions that could be answered only using the provided files. +Rules: Do NOT guess or invent. If information is insufficient in the files, explicitly state that. +``` + +### QueryWorker Stage 5 (Final Answer) +``` +You are to compose a final answer to the user query using only the information from the files. +Query: {query} +Files considered: {file_list} +Relationship analysis: {relationships} +Requirements: +- Use only information present in the files and analysis above. +- If the answer is uncertain or cannot be determined from the files, clearly state that limitation. +- Avoid speculation or assumptions. +Provide a concise, structured answer. +``` + +## Docker Architecture + +### Services +- **rust-engine** - Warp API + workers (port 8000) +- **web-app** - Express + React (port 3001) +- **mysql** - MySQL 9.1 (port 3306) +- **qdrant** - Qdrant vector DB (port 6333) +- **phpmyadmin** - DB admin UI (port 8080) + +### Volumes (Production) +- `rust-storage:/app/storage` - File storage (writable) +- `/var/www/codered-astra/rust-engine/demo-data:/app/demo-data:ro` - Demo PDFs (read-only) +- `~/astra-logs:/var/log` - Log files + +## Common Issues + +### 1. SQL Syntax Error +**Problem**: `error near 'CREATE TABLE'` +**Cause**: Multiple CREATE TABLE in one query +**Fix**: Split into separate `sqlx::query()` calls + +### 2. Permission Denied +**Problem**: `Permission denied (os error 13)` +**Cause**: Container user can't write to storage +**Fix**: Use Docker volume, ensure ownership matches container user + +### 3. Worker Not Processing +**Problem**: Files/queries stuck in Queued +**Cause**: Worker crashed or not started +**Fix**: Check logs, ensure workers spawned in main.rs + +### 4. Qdrant Connection Failed +**Problem**: `qdrant upsert/search failed` +**Cause**: Qdrant not running or wrong URL +**Fix**: Verify QDRANT_URL, check qdrant container health + +## Development Commands + +```bash +# Build and run locally +cd rust-engine +cargo build +cargo run + +# Check code +cargo check + +# Run with logs +RUST_LOG=info cargo run + +# Docker compose (dev) +docker-compose up --build + +# Docker compose (production) +docker-compose -f docker-compose.prod.yml up -d + +# View logs +docker logs rust-engine -f + +# Rebuild single service +docker-compose build rust-engine +docker-compose up -d rust-engine +``` + +## Testing Flow + +1. Start services: `docker-compose up -d` +2. Import demo data: `curl -X POST http://localhost:3001/api/files/import-demo` +3. Wait for FileWorker to complete (~30 seconds for 20 files) +4. Check file status: `curl http://localhost:3001/api/files/list` +5. Create query: `curl -X POST http://localhost:3001/api/query/create -H "Content-Type: application/json" -d '{"q": "ISS main bus voltage", "top_k": 5}'` +6. Check status: `curl http://localhost:3001/api/query/status?id=` +7. Get result: `curl http://localhost:3001/api/query/result?id=` + +## Performance Notes + +- FileWorker: ~1-2 sec per file (demo embeddings) +- QueryWorker: ~3-5 sec per query (search + 2 Gemini calls) +- Qdrant search: <100ms for 1000s of vectors +- MySQL queries: <10ms for simple selects + +## Security Considerations + +- Store GEMINI_API_KEY in GitHub Secrets (production) +- Use environment variables for all credentials +- Don't commit `.env` files +- Restrict phpmyadmin to internal network only +- Use HTTPS in production deployment diff --git a/README.md b/README.md new file mode 100644 index 0000000..b0b6190 --- /dev/null +++ b/README.md @@ -0,0 +1,75 @@ +# CodeRED-Astra πŸš€ + +A hackathon-ready project with React frontend and Rust backend engine. + +## Quick Start + +```bash +# 1. Setup environment +cp .env.example .env +# Edit .env with your credentials + +# 2. Start everything with Docker +docker-compose up --build + +# 3. Access your app +# Frontend: http://localhost +# API: http://localhost:8000 +# Database Admin: http://127.0.0.1:8080 +``` + +## Development + +**Frontend (React + Vite)**: +```bash +cd web-app +npm install +npm run dev # http://localhost:5173 +``` + +**Backend (Rust)**: +```bash +cd rust-engine +cargo run # http://localhost:8000 +``` + +## Architecture + +- **Frontend**: React 18 + Vite + Tailwind CSS +- **Backend**: Rust + Warp + SQLx +- **Database**: MySQL 8.0 + phpMyAdmin +- **API**: RESTful endpoints with CORS enabled +- **Docker**: Full containerization for easy deployment + +## Project Structure + +``` +β”œβ”€β”€ web-app/ # React frontend +β”‚ β”œβ”€β”€ src/ +β”‚ β”‚ β”œβ”€β”€ App.jsx # Main component +β”‚ β”‚ └── main.jsx # Entry point +β”‚ └── Dockerfile +β”œβ”€β”€ rust-engine/ # Rust backend +β”‚ β”œβ”€β”€ src/ +β”‚ β”‚ └── main.rs # API server +β”‚ └── Dockerfile +β”œβ”€β”€ docker-compose.yml # Full stack orchestration +└── .env.example # Environment template +``` + +## Team Workflow + +- **Frontend devs**: Work in `web-app/src/`, use `/api/*` for backend calls +- **Backend devs**: Work in `rust-engine/src/`, add endpoints to main.rs +- **Database**: Access phpMyAdmin at http://127.0.0.1:8080 + +## Features + +βœ… Hot reload for both frontend and backend +βœ… Automatic API proxying from React to Rust +βœ… Database connection with graceful fallback +βœ… CORS configured for cross-origin requests +βœ… Production-ready Docker containers +βœ… Health monitoring and status dashboard + +Ready for your hackathon! See `DEVELOPMENT.md` for detailed setup instructions. diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..5a7e19d --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,8 @@ +services: + fallback-web-app: + image: ghcr.io/${REPO_NAME_LOWER}/web-app:${IMAGE_TAG} + restart: always + ports: + - "127.0.0.1:3034:3000" + environment: + - GEMINI_API_KEY=${GEMINI_API_KEY} diff --git a/docker-compose.yml b/docker-compose.yml index 393e5d9..78e4c61 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,8 +13,6 @@ services: - DATABASE_URL=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@mysql:3306/${MYSQL_DATABASE} - RUST_ENGINE_URL=http://rust-engine:8000 - GEMINI_API_KEY=${GEMINI_API_KEY} - volumes: - - rust-storage:/app/storage:ro depends_on: - mysql # <-- Updated dependency - rust-engine diff --git a/frontend/README.md b/frontend/README.md deleted file mode 100644 index 8b13789..0000000 --- a/frontend/README.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/frontend/src/components/layouts/chat-layout.jsx b/frontend/src/components/layouts/chat-layout.jsx deleted file mode 100644 index 951d54c..0000000 --- a/frontend/src/components/layouts/chat-layout.jsx +++ /dev/null @@ -1,244 +0,0 @@ -import React, { - useCallback, - useEffect, - useMemo, - useRef, - useState, -} from "react"; -import ChatHeader from "src/components/ui/chat/chat-header"; -import ChatWindow from "src/components/ui/chat/chat-window"; -import MessageInput from "src/components/ui/chat/message-input"; -import { - createQuery, - getQueryResult, - getQueryStatus, - listFiles, -} from "src/lib/api"; - -const createId = () => - globalThis.crypto?.randomUUID?.() ?? `id-${Date.now()}-${Math.random()}`; - -const INTRO_MESSAGE = { - id: "intro", - role: "assistant", - content: - "Ask me about the demo PDFs and I'll respond with the best matches pulled from the processed files.", -}; - -export default function ChatLayout() { - const [messages, setMessages] = useState([INTRO_MESSAGE]); - const [isProcessing, setIsProcessing] = useState(false); - const [files, setFiles] = useState([]); - const [errorToast, setErrorToast] = useState(""); - const pollAbortRef = useRef(null); - - const showError = useCallback((message) => { - setErrorToast(message); - window.setTimeout(() => setErrorToast(""), 5000); - }, []); - - const refreshFiles = useCallback(async () => { - try { - const latest = await listFiles(); - setFiles(latest); - } catch (error) { - showError(error.message ?? "Failed to load files"); - } - }, [showError]); - - useEffect(() => { - refreshFiles(); - }, [refreshFiles]); - - useEffect(() => { - return () => { - if (pollAbortRef.current) { - pollAbortRef.current.aborted = true; - } - }; - }, []); - - const buildAssistantMarkdown = useCallback((result) => { - if (!result || typeof result !== "object") { - return "I could not find a response for that request."; - } - - const finalAnswer = result.final_answer?.trim(); - const relationships = result.relationships?.trim(); - const relatedFiles = Array.isArray(result.related_files) - ? result.related_files - : []; - - const fileLines = relatedFiles - .filter((f) => f && typeof f === "object") - .map((file) => { - const filename = file.filename || file.id || "download"; - const storageUrl = file.storage_url || `/storage/${filename}`; - const linkTarget = storageUrl.startsWith("/storage/") - ? `/storage/${encodeURIComponent(storageUrl.replace("/storage/", ""))}` - : storageUrl; - const description = file.description?.trim(); - const score = - typeof file.score === "number" - ? ` _(score: ${file.score.toFixed(3)})_` - : ""; - const detail = description ? ` β€” ${description}` : ""; - return `- [${filename}](${linkTarget})${detail}${score}`; - }); - - let content = - finalAnswer || - "I could not determine an answer from the indexed documents yet."; - - if (fileLines.length) { - content += `\n\n**Related Files**\n${fileLines.join("\n")}`; - } - - if (relationships && relationships !== finalAnswer) { - content += `\n\n---\n${relationships}`; - } - - if (!fileLines.length && (!finalAnswer || finalAnswer.length < 10)) { - content += - "\n\n_No analyzed documents matched yet. Try seeding demo data or wait for processing to finish._"; - } - - return content; - }, []); - - const waitForResult = useCallback(async (id) => { - const abortState = { aborted: false }; - pollAbortRef.current = abortState; - const timeoutMs = 120_000; - const intervalMs = 1_500; - const started = Date.now(); - - while (!abortState.aborted) { - if (Date.now() - started > timeoutMs) { - throw new Error("Timed out waiting for the query to finish"); - } - - const statusPayload = await getQueryStatus(id); - const status = statusPayload?.status; - - if (status === "Completed") { - const resultPayload = await getQueryResult(id); - return resultPayload?.result; - } - - if (status === "Failed") { - const resultPayload = await getQueryResult(id); - const reason = resultPayload?.result?.error || "Query failed"; - throw new Error(reason); - } - - if (status === "Cancelled") { - throw new Error("Query was cancelled"); - } - - if (status === "not_found") { - throw new Error("Query was not found"); - } - - await new Promise((resolve) => window.setTimeout(resolve, intervalMs)); - } - - throw new Error("Query polling was aborted"); - }, []); - - const handleSend = useCallback( - async (text) => { - if (isProcessing) { - showError("Please wait for the current response to finish."); - return; - } - - const userEntry = { - id: createId(), - role: "user", - content: text, - }; - setMessages((prev) => [...prev, userEntry]); - - const placeholderId = createId(); - setMessages((prev) => [ - ...prev, - { - id: placeholderId, - role: "assistant", - content: "_Analyzing indexed documents..._", - pending: true, - }, - ]); - - setIsProcessing(true); - - try { - const payload = { q: text, top_k: 5 }; - const created = await createQuery(payload); - const result = await waitForResult(created.id); - const content = buildAssistantMarkdown(result); - setMessages((prev) => - prev.map((message) => - message.id === placeholderId - ? { ...message, content, pending: false } - : message, - ), - ); - } catch (error) { - const message = error?.message || "Something went wrong."; - setMessages((prev) => - prev.map((entry) => - entry.id === placeholderId - ? { - ...entry, - content: `⚠️ ${message}`, - pending: false, - error: true, - } - : entry, - ), - ); - showError(message); - } finally { - pollAbortRef.current = null; - setIsProcessing(false); - refreshFiles(); - } - }, - [ - isProcessing, - showError, - refreshFiles, - waitForResult, - buildAssistantMarkdown, - ], - ); - - const handleDeleteAll = useCallback(() => { - if (!window.confirm("Delete all messages?")) { - return; - } - setMessages([INTRO_MESSAGE]); - }, []); - - const latestFileSummary = useMemo(() => { - if (!files.length) return "No files indexed yet."; - const pending = files.filter((f) => f.pending_analysis).length; - const ready = files.length - pending; - return `${ready} ready β€’ ${pending} processing`; - }, [files]); - - return ( -
- - - -
- ); -} diff --git a/frontend/src/components/ui/button/delete-button.jsx b/frontend/src/components/ui/button/delete-button.jsx deleted file mode 100644 index 37e008e..0000000 --- a/frontend/src/components/ui/button/delete-button.jsx +++ /dev/null @@ -1,19 +0,0 @@ -import { Flame } from "lucide-react"; -import { motion } from "motion/react"; - -export default function FlameButton({ onClick, disabled = false }) { - return ( - - - - ); -} diff --git a/frontend/src/components/ui/chat/chat-header.jsx b/frontend/src/components/ui/chat/chat-header.jsx deleted file mode 100644 index b5d9a63..0000000 --- a/frontend/src/components/ui/chat/chat-header.jsx +++ /dev/null @@ -1,89 +0,0 @@ -import React, { useEffect, useMemo, useState } from "react"; -import { motion } from "motion/react"; -import { Rocket } from "lucide-react"; -import DeleteButton from "src/components/ui/button/delete-button"; -import SchematicButton from "src/components/ui/button/schematic-button"; - -export default function ChatHeader({ - title = "Title of Chat", - onClear, - busy = false, - fileSummary, - errorMessage, -}) { - const isDebug = useMemo(() => { - const p = new URLSearchParams(window.location.search); - return p.get("debug") === "1"; - }, []); - const [ingesting, setIngesting] = useState(false); - const [toast, setToast] = useState(""); - const [externalToast, setExternalToast] = useState(""); - - useEffect(() => { - if (!errorMessage) return; - setExternalToast(errorMessage); - const timer = window.setTimeout(() => setExternalToast(""), 5000); - return () => window.clearTimeout(timer); - }, [errorMessage]); - - async function triggerDemoIngest() { - try { - setIngesting(true); - const res = await fetch("/api/files/import-demo", { method: "POST" }); - const json = await res.json().catch(() => ({})); - const imported = json.imported ?? "?"; - const skipped = json.skipped ?? "?"; - const summary = `Imported: ${imported}, Skipped: ${skipped}`; - setToast(json.error ? `${summary} - ${json.error}` : summary); - setTimeout(() => setToast(""), 4000); - } catch (e) { - setToast("Import failed"); - setTimeout(() => setToast(""), 4000); - } finally { - setIngesting(false); - } - } - - return ( -
-
-
- -
-

- {title} -

- {fileSummary && ( -
- {fileSummary} -
- )} - - {isDebug && ( - - - {ingesting ? "Seeding…" : "Seed Demo Data"} - - )} -
-
- {toast && ( -
- {toast} -
- )} - {externalToast && ( -
- {externalToast} -
- )} -
-
- ); -} diff --git a/frontend/src/components/ui/chat/message-input.jsx b/frontend/src/components/ui/chat/message-input.jsx deleted file mode 100644 index 0f49a7f..0000000 --- a/frontend/src/components/ui/chat/message-input.jsx +++ /dev/null @@ -1,119 +0,0 @@ -import React, { useState, useRef, useEffect } from "react"; -import DownButton from "src/components/ui/button/down-button"; -import { motion } from "motion/react"; -import { BotMessageSquare } from "lucide-react"; - -export default function MessageInput({ onSend, disabled = false }) { - const [text, setText] = useState(""); - const textareaRef = useRef(null); - - useEffect(() => { - // ensure correct initial height - if (textareaRef.current) textareaRef.current.style.height = "auto"; - }, []); - - async function handleSubmit(e) { - e.preventDefault(); - if (!text.trim() || disabled) return; - onSend(text.trim()); - - // create query on backend - try { - if (onMessage) - onMessage("assistant", "Queued: sending request to server..."); - const createRes = await fetch(`/api/query/create`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ q: text, top_k: 5 }), - }); - const createJson = await createRes.json(); - const id = createJson.id; - if (!id) throw new Error("no id returned"); - - // poll status - let status = "Queued"; - if (onMessage) onMessage("assistant", `Status: ${status}`); - while (status !== "Completed" && status !== "Failed") { - await new Promise((r) => setTimeout(r, 1000)); - const sRes = await fetch(`/api/query/status?id=${id}`); - const sJson = await sRes.json(); - status = sJson.status; - if (onMessage) onMessage("assistant", `Status: ${status}`); - if (status === "Cancelled") break; - } - - if (status === "Completed") { - const resultRes = await fetch(`/api/query/result?id=${id}`); - const resultJson = await resultRes.json(); - const final = - resultJson?.result?.final_answer || - JSON.stringify(resultJson?.result || {}); - if (onMessage) onMessage("assistant", final); - } else { - if (onMessage) - onMessage("assistant", `Query status ended as: ${status}`); - } - } catch (err) { - console.error(err); - if (onMessage) onMessage("assistant", `Error: ${err.message}`); - } - - setText(""); - } - - return ( -
-