From da6ab3a782a70b16bf0237a7becb7ca6cd7d756d Mon Sep 17 00:00:00 2001 From: Christbru Date: Sun, 19 Oct 2025 03:53:02 -0500 Subject: [PATCH 1/2] Add and prepare rust worker management system for file information processing and knowledge base framework --- docker-compose.yml | 14 +- rust-engine/Cargo.lock | 839 +++++++++++++++++++++++++------ rust-engine/Cargo.toml | 13 +- rust-engine/src/api.rs | 226 +++++++++ rust-engine/src/db.rs | 33 ++ rust-engine/src/gemini_client.rs | 37 ++ rust-engine/src/main.rs | 130 +---- rust-engine/src/models.rs | 56 +++ rust-engine/src/storage.rs | 34 ++ rust-engine/src/vector.rs | 24 + rust-engine/src/vector_db.rs | 87 ++++ rust-engine/src/worker.rs | 160 ++++++ 12 files changed, 1402 insertions(+), 251 deletions(-) create mode 100644 rust-engine/src/api.rs create mode 100644 rust-engine/src/db.rs create mode 100644 rust-engine/src/gemini_client.rs create mode 100644 rust-engine/src/models.rs create mode 100644 rust-engine/src/storage.rs create mode 100644 rust-engine/src/vector.rs create mode 100644 rust-engine/src/vector_db.rs create mode 100644 rust-engine/src/worker.rs diff --git a/docker-compose.yml b/docker-compose.yml index f61275f..c9b1ab4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,5 +49,17 @@ services: depends_on: - mysql + qdrant: + image: qdrant/qdrant:latest + restart: unless-stopped + ports: + - "127.0.0.1:6333:6333" + volumes: + - qdrant-data:/qdrant/storage + environment: + - QDRANT__SERVICE__GRPC_PORT=6334 + # expose to rust-engine via service name 'qdrant' + volumes: - mysql-data: # Renamed volume for clarity (optional but good practice) \ No newline at end of file + mysql-data: # Renamed volume for clarity (optional but good practice) + qdrant-data: \ No newline at end of file diff --git a/rust-engine/Cargo.lock b/rust-engine/Cargo.lock index 86cece2..57988d6 100644 --- a/rust-engine/Cargo.lock +++ b/rust-engine/Cargo.lock @@ -23,6 +23,17 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atoi" version = "2.0.0" @@ -32,18 +43,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -108,6 +119,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.42" @@ -119,7 +136,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -137,6 +154,16 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -192,12 +219,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "data-encoding" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" - [[package]] name = "der" version = "0.7.10" @@ -262,6 +283,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -284,6 +315,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "find-msvc-tools" version = "0.1.4" @@ -313,6 +350,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -366,6 +418,17 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -386,6 +449,7 @@ checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -411,22 +475,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", ] [[package]] name = "h2" -version = "0.3.27" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", - "http 0.2.12", + "http", "indexmap", "slab", "tokio", @@ -462,14 +542,14 @@ dependencies = [ [[package]] name = "headers" -version = "0.3.9" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" +checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64 0.21.7", + "base64", "bytes", "headers-core", - "http 0.2.12", + "http", "httpdate", "mime", "sha1", @@ -477,11 +557,11 @@ dependencies = [ [[package]] name = "headers-core" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 0.2.12", + "http", ] [[package]] @@ -523,17 +603,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.3.1" @@ -547,12 +616,24 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.6" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 0.2.12", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", "pin-project-lite", ] @@ -570,26 +651,84 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.32" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" dependencies = [ + "atomic-waker", "bytes", "futures-channel", "futures-core", - "futures-util", "h2", - "http 0.2.12", + "http", "http-body", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.10", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.3", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", "tokio", "tower-service", "tracing", - "want", + "windows-registry", ] [[package]] @@ -733,6 +872,22 @@ dependencies = [ "hashbrown 0.16.0", ] +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "itoa" version = "1.0.15" @@ -791,6 +946,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + [[package]] name = "litemap" version = "0.8.0" @@ -812,6 +973,12 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "md-5" version = "0.10.6" @@ -857,22 +1024,38 @@ dependencies = [ [[package]] name = "multer" -version = "2.1.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" dependencies = [ "bytes", "encoding_rs", "futures-util", - "http 0.2.12", + "http", "httparse", - "log", "memchr", "mime", "spin", "version_check", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -894,7 +1077,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.5", "smallvec", "zeroize", ] @@ -935,6 +1118,50 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "openssl" +version = "0.10.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ad14dd45412269e1a30f52ad8f0664f0f4f4a89ee8fe28c3b3527021ebb654" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.110" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a9f0075ba3c21b09f8e8b2026584b1d18d49388648f2fbbf3c97ea8deced8e2" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking" version = "2.2.1" @@ -961,7 +1188,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -1065,6 +1292,61 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" version = "1.0.41" @@ -1074,6 +1356,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "rand" version = "0.8.5" @@ -1081,8 +1369,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", ] [[package]] @@ -1092,7 +1390,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -1101,7 +1409,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.16", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.4", ] [[package]] @@ -1113,6 +1430,50 @@ dependencies = [ "bitflags", ] +[[package]] +name = "reqwest" +version = "0.12.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 1.0.3", +] + [[package]] name = "ring" version = "0.17.14" @@ -1121,7 +1482,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.16", "libc", "untrusted", "windows-sys 0.52.0", @@ -1140,7 +1501,7 @@ dependencies = [ "num-traits", "pkcs1", "pkcs8", - "rand_core", + "rand_core 0.6.4", "signature", "spki", "subtle", @@ -1152,17 +1513,43 @@ name = "rust-engine" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", + "bytes", "chrono", "dotenvy", + "futures-util", + "lazy_static", + "reqwest", "serde", "serde_json", "sqlx", "tokio", + "tokio-util", "tracing", "tracing-subscriber", + "uuid", "warp", ] +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.33" @@ -1183,6 +1570,7 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ + "web-time", "zeroize", ] @@ -1209,6 +1597,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -1221,6 +1618,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.228" @@ -1329,7 +1749,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -1347,16 +1767,6 @@ dependencies = [ "serde", ] -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.1" @@ -1405,7 +1815,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "chrono", "crc", @@ -1428,11 +1838,12 @@ dependencies = [ "serde_json", "sha2", "smallvec", - "thiserror 2.0.17", + "thiserror", "tokio", "tokio-stream", "tracing", "url", + "uuid", "webpki-roots 0.26.11", ] @@ -1481,7 +1892,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", - "base64 0.22.1", + "base64", "bitflags", "byteorder", "bytes", @@ -1504,7 +1915,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rand", + "rand 0.8.5", "rsa", "serde", "sha1", @@ -1512,8 +1923,9 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror", "tracing", + "uuid", "whoami", ] @@ -1524,7 +1936,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", - "base64 0.22.1", + "base64", "bitflags", "byteorder", "chrono", @@ -1543,15 +1955,16 @@ dependencies = [ "md-5", "memchr", "once_cell", - "rand", + "rand 0.8.5", "serde", "serde_json", "sha2", "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror", "tracing", + "uuid", "whoami", ] @@ -1575,9 +1988,10 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.17", + "thiserror", "tracing", "url", + "uuid", ] [[package]] @@ -1614,6 +2028,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1626,12 +2049,37 @@ dependencies = [ ] [[package]] -name = "thiserror" -version = "1.0.69" +name = "system-configuration" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "thiserror-impl 1.0.69", + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", ] [[package]] @@ -1640,18 +2088,7 @@ version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl 2.0.17", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "thiserror-impl", ] [[package]] @@ -1711,7 +2148,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.1", + "socket2", "tokio-macros", "windows-sys 0.61.2", ] @@ -1727,6 +2164,26 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.17" @@ -1738,18 +2195,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-tungstenite" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite", -] - [[package]] name = "tokio-util" version = "0.7.16" @@ -1763,6 +2208,45 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -1833,25 +2317,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "tungstenite" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http 1.3.1", - "httparse", - "log", - "rand", - "sha1", - "thiserror 1.0.69", - "url", - "utf-8", -] - [[package]] name = "typenum" version = "1.19.0" @@ -1909,18 +2374,24 @@ dependencies = [ "serde", ] -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf8_iter" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -1950,16 +2421,18 @@ dependencies = [ [[package]] name = "warp" -version = "0.3.7" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4378d202ff965b011c64817db11d5829506d3404edeadb61f190d111da3f231c" +checksum = "51d06d9202adc1f15d709c4f4a2069be5428aa912cc025d6f268ac441ab066b0" dependencies = [ "bytes", - "futures-channel", "futures-util", "headers", - "http 0.2.12", + "http", + "http-body", + "http-body-util", "hyper", + "hyper-util", "log", "mime", "mime_guess", @@ -1971,7 +2444,6 @@ dependencies = [ "serde_json", "serde_urlencoded", "tokio", - "tokio-tungstenite", "tokio-util", "tower-service", "tracing", @@ -1983,6 +2455,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasite" version = "0.1.0" @@ -2016,6 +2497,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e038d41e478cc73bae0ff9b36c60cff1c98b8f38f8d7e8061e79ee63608ac5c" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.104" @@ -2048,6 +2542,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.26.11" @@ -2084,9 +2598,9 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link", - "windows-result", - "windows-strings", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", ] [[package]] @@ -2111,19 +2625,54 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e" +dependencies = [ + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-result" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", ] [[package]] @@ -2132,7 +2681,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -2177,7 +2726,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -2217,7 +2766,7 @@ version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link", + "windows-link 0.2.1", "windows_aarch64_gnullvm 0.53.1", "windows_aarch64_msvc 0.53.1", "windows_i686_gnu 0.53.1", @@ -2366,6 +2915,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + [[package]] name = "writeable" version = "0.6.1" diff --git a/rust-engine/Cargo.toml b/rust-engine/Cargo.toml index a93f36e..8613870 100644 --- a/rust-engine/Cargo.toml +++ b/rust-engine/Cargo.toml @@ -7,12 +7,19 @@ edition = "2021" [dependencies] tokio = { version = "1.38.0", features = ["full"] } -warp = "0.3.7" +warp = { version = "0.4.2", features = ["server", "multipart"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -sqlx = { version = "0.8.6", features = ["runtime-tokio-rustls", "mysql", "chrono"] } +sqlx = { version = "0.8.6", features = ["runtime-tokio-rustls", "mysql", "chrono", "uuid", "macros"] } chrono = { version = "0.4", features = ["serde"] } tracing = "0.1" tracing-subscriber = "0.3" dotenvy = "0.15.7" # Switched from unmaintained 'dotenv' -anyhow = "1.0" \ No newline at end of file +anyhow = "1.0" +uuid = { version = "1", features = ["serde", "v4"] } +reqwest = { version = "0.12.24", features = ["json", "rustls-tls"] } +async-trait = "0.1" +tokio-util = "0.7" +futures-util = "0.3" +lazy_static = "1.4" +bytes = "1.4" diff --git a/rust-engine/src/api.rs b/rust-engine/src/api.rs new file mode 100644 index 0000000..cae23d7 --- /dev/null +++ b/rust-engine/src/api.rs @@ -0,0 +1,226 @@ +use crate::gemini_client; +use crate::vector_db::QdrantClient; +use crate::storage; +use anyhow::Result; +use bytes::Buf; +use futures_util::{StreamExt, TryStreamExt}; +use serde::Deserialize; +use sqlx::{MySqlPool, Row}; +use warp::{multipart::FormData, Filter, Rejection, Reply}; + +#[derive(Debug, Deserialize)] +struct DeleteQuery { + id: String, +} + +pub fn routes(pool: MySqlPool) -> impl Filter + Clone { + let pool_filter = warp::any().map(move || pool.clone()); + + // Upload file + let upload = warp::path("files") + .and(warp::post()) + .and(warp::multipart::form().max_length(50_000_000)) // 50MB per part default; storage is filesystem-backed + .and(pool_filter.clone()) + .and_then(handle_upload); + + // Delete file + let delete = warp::path!("files" / "delete") + .and(warp::get()) + .and(warp::query::()) + .and(pool_filter.clone()) + .and_then(handle_delete); + + // List files + let list = warp::path!("files" / "list") + .and(warp::get()) + .and(pool_filter.clone()) + .and_then(handle_list); + + // Create query + let create_q = warp::path!("query" / "create") + .and(warp::post()) + .and(warp::body::json()) + .and(pool_filter.clone()) + .and_then(handle_create_query); + + // Query status + let status = warp::path!("query" / "status") + .and(warp::get()) + .and(warp::query::()) + .and(pool_filter.clone()) + .and_then(handle_query_status); + + // Query result + let result = warp::path!("query" / "result") + .and(warp::get()) + .and(warp::query::()) + .and(pool_filter.clone()) + .and_then(handle_query_result); + + // Cancel + let cancel = warp::path!("query" / "cancel") + .and(warp::get()) + .and(warp::query::()) + .and(pool_filter.clone()) + .and_then(handle_cancel_query); + + upload.or(delete).or(list).or(create_q).or(status).or(result).or(cancel) +} + +async fn handle_upload(mut form: FormData, pool: MySqlPool) -> Result { + // qdrant client + let qdrant_url = std::env::var("QDRANT_URL").unwrap_or_else(|_| "http://qdrant:6333".to_string()); + let qdrant = QdrantClient::new(&qdrant_url); + + while let Some(field) = form.try_next().await.map_err(|_| warp::reject())? { + let name = field.name().to_string(); + let filename = field + .filename() + .map(|s| s.to_string()) + .unwrap_or_else(|| format!("upload-{}", uuid::Uuid::new_v4())); + + // Read stream of Buf into a Vec + let data = field + .stream() + .map_ok(|mut buf| { + let mut v = Vec::new(); + while buf.has_remaining() { + let chunk = buf.chunk(); + v.extend_from_slice(chunk); + let n = chunk.len(); + buf.advance(n); + } + v + }) + .try_fold(Vec::new(), |mut acc, chunk_vec| async move { + acc.extend_from_slice(&chunk_vec); + Ok(acc) + }) + .await + .map_err(|_| warp::reject())?; + + // Save file + let path = storage::save_file(&filename, &data).map_err(|_| warp::reject())?; + + // Generate gemini token/description (stub) + let token = gemini_client::generate_token_for_file(path.to_str().unwrap()).await.map_err(|_| warp::reject())?; + + // Insert file record + let id = uuid::Uuid::new_v4().to_string(); + let desc = Some(format!("token:{}", token)); + sqlx::query("INSERT INTO files (id, filename, path, description) VALUES (?, ?, ?, ?)") + .bind(&id) + .bind(&filename) + .bind(path.to_str().unwrap()) + .bind(desc) + .execute(&pool) + .await + .map_err(|e| { + tracing::error!("DB insert error: {}", e); + warp::reject() + })?; + + // generate demo embedding and upsert to Qdrant (async best-effort) + let emb = crate::gemini_client::demo_embedding_from_path(path.to_str().unwrap()); + let qdrant_clone = qdrant.clone(); + let id_clone = id.clone(); + tokio::spawn(async move { + if let Err(e) = qdrant_clone.upsert_point(&id_clone, emb).await { + tracing::error!("qdrant upsert failed: {}", e); + } + }); + } + + Ok(warp::reply::json(&serde_json::json!({"success": true}))) +} + +async fn handle_delete(q: DeleteQuery, pool: MySqlPool) -> Result { + if let Some(row) = sqlx::query("SELECT path FROM files WHERE id = ?") + .bind(&q.id) + .fetch_optional(&pool) + .await + .map_err(|_| warp::reject())? + { + let path: String = row.get("path"); + let _ = storage::delete_file(std::path::Path::new(&path)); + let _ = sqlx::query("DELETE FROM files WHERE id = ?").bind(&q.id).execute(&pool).await; + return Ok(warp::reply::json(&serde_json::json!({"deleted": true}))); + } + Ok(warp::reply::json(&serde_json::json!({"deleted": false}))) +} + +async fn handle_list(pool: MySqlPool) -> Result { + let rows = sqlx::query("SELECT id, filename, path, description FROM files ORDER BY created_at DESC LIMIT 500") + .fetch_all(&pool) + .await + .map_err(|e| { + tracing::error!("DB list error: {}", e); + warp::reject() + })?; + + let files: Vec = rows + .into_iter() + .map(|r| { + let id: String = r.get("id"); + let filename: String = r.get("filename"); + let path: String = r.get("path"); + let description: Option = r.get("description"); + serde_json::json!({"id": id, "filename": filename, "path": path, "description": description}) + }) + .collect(); + + Ok(warp::reply::json(&serde_json::json!({"files": files}))) +} + +async fn handle_create_query(body: serde_json::Value, pool: MySqlPool) -> Result { + // Insert query as queued, worker will pick it up + let id = uuid::Uuid::new_v4().to_string(); + let payload = body; + sqlx::query("INSERT INTO queries (id, status, payload) VALUES (?, 'Queued', ?)") + .bind(&id) + .bind(payload) + .execute(&pool) + .await + .map_err(|e| { + tracing::error!("DB insert query error: {}", e); + warp::reject() + })?; + + Ok(warp::reply::json(&serde_json::json!({"id": id}))) +} + +async fn handle_query_status(q: DeleteQuery, pool: MySqlPool) -> Result { + if let Some(row) = sqlx::query("SELECT status FROM queries WHERE id = ?") + .bind(&q.id) + .fetch_optional(&pool) + .await + .map_err(|_| warp::reject())? + { + let status: String = row.get("status"); + return Ok(warp::reply::json(&serde_json::json!({"status": status}))); + } + Ok(warp::reply::json(&serde_json::json!({"status": "not_found"}))) +} + +async fn handle_query_result(q: DeleteQuery, pool: MySqlPool) -> Result { + if let Some(row) = sqlx::query("SELECT result FROM queries WHERE id = ?") + .bind(&q.id) + .fetch_optional(&pool) + .await + .map_err(|_| warp::reject())? + { + let result: Option = row.get("result"); + return Ok(warp::reply::json(&serde_json::json!({"result": result}))); + } + Ok(warp::reply::json(&serde_json::json!({"result": null}))) +} + +async fn handle_cancel_query(q: DeleteQuery, pool: MySqlPool) -> Result { + // Mark as cancelled; worker must check status before heavy steps + sqlx::query("UPDATE queries SET status = 'Cancelled' WHERE id = ?") + .bind(&q.id) + .execute(&pool) + .await + .map_err(|_| warp::reject())?; + Ok(warp::reply::json(&serde_json::json!({"cancelled": true}))) +} diff --git a/rust-engine/src/db.rs b/rust-engine/src/db.rs new file mode 100644 index 0000000..19565a1 --- /dev/null +++ b/rust-engine/src/db.rs @@ -0,0 +1,33 @@ +use sqlx::{MySql, MySqlPool}; +use tracing::info; + +pub async fn init_db(database_url: &str) -> Result { + let pool = MySqlPool::connect(database_url).await?; + + // Create tables if they don't exist. Simple schema for demo/hackathon use. + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS files ( + id VARCHAR(36) PRIMARY KEY, + filename TEXT NOT NULL, + path TEXT NOT NULL, + description TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS queries ( + id VARCHAR(36) PRIMARY KEY, + status VARCHAR(32) NOT NULL, + payload JSON, + result JSON, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ); + "#, + ) + .execute(&pool) + .await?; + + info!("Database initialized"); + Ok(pool) +} diff --git a/rust-engine/src/gemini_client.rs b/rust-engine/src/gemini_client.rs new file mode 100644 index 0000000..e6ef798 --- /dev/null +++ b/rust-engine/src/gemini_client.rs @@ -0,0 +1,37 @@ +use anyhow::Result; +use serde::Deserialize; + +// NOTE: This is a small stub to represent where you'd call the Gemini API. +// Replace with real API call and proper auth handling for production. + +#[derive(Debug, Deserialize)] +pub struct GeminiTokenResponse { + pub token: String, +} + +pub async fn generate_token_for_file(_path: &str) -> Result { + Ok("gemini-token-placeholder".to_string()) +} + +/// Demo embedding generator - deterministic pseudo-embedding from filename/path +pub fn demo_embedding_from_path(path: &str) -> Vec { + // Very simple: hash bytes into a small vector + let mut v = vec![0f32; 64]; + for (i, b) in path.as_bytes().iter().enumerate() { + let idx = i % v.len(); + v[idx] += (*b as f32) / 255.0; + } + v +} + +pub const DEMO_EMBED_DIM: usize = 64; + +/// Demo text embedding (replace with real Gemini text embedding API) +pub async fn demo_text_embedding(text: &str) -> Result> { + let mut v = vec![0f32; DEMO_EMBED_DIM]; + for (i, b) in text.as_bytes().iter().enumerate() { + let idx = i % v.len(); + v[idx] += (*b as f32) / 255.0; + } + Ok(v) +} diff --git a/rust-engine/src/main.rs b/rust-engine/src/main.rs index 6a15672..30c022a 100644 --- a/rust-engine/src/main.rs +++ b/rust-engine/src/main.rs @@ -1,21 +1,16 @@ +mod api; +mod db; +mod gemini_client; +mod models; +mod storage; +mod vector; +mod worker; +mod vector_db; + use std::env; +use std::error::Error; +use tracing::info; use warp::Filter; -use sqlx::mysql::MySqlPool; -use serde::{Deserialize, Serialize}; -use tracing::{info, warn}; - -#[derive(Debug, Serialize, Deserialize)] -struct HealthResponse { - status: String, - timestamp: String, -} - -#[derive(Debug, Serialize, Deserialize)] -struct ApiResponse { - success: bool, - data: Option, - message: Option, -} #[tokio::main] async fn main() -> Result<(), Box> { @@ -29,103 +24,28 @@ async fn main() -> Result<(), Box> { .unwrap_or_else(|_| "mysql://astraadmin:password@mysql:3306/astra".to_string()); info!("Starting Rust Engine..."); - // info!("Connecting to database: {}", database_url); - // Connect to database - let pool = match MySqlPool::connect(&database_url).await { - Ok(pool) => { - info!("Successfully connected to database"); - pool - } - Err(e) => { - warn!("Failed to connect to database: {}. Starting without DB connection.", e); - // In a hackathon setting, we might want to continue without DB for initial testing - return start_server_without_db().await; - } - }; + // Ensure storage dir + storage::ensure_storage_dir().expect("storage dir"); - // CORS configuration - let cors = warp::cors() - .allow_any_origin() - .allow_headers(vec!["content-type", "authorization"]) - .allow_methods(vec!["GET", "POST", "PUT", "DELETE", "OPTIONS"]); + // Initialize DB + let pool = db::init_db(&database_url).await.map_err(|e| -> Box { Box::new(e) })?; - // Health check endpoint - let health = warp::path("health") - .and(warp::get()) - .map(|| { - let response = HealthResponse { - status: "healthy".to_string(), - timestamp: chrono::Utc::now().to_rfc3339(), - }; - warp::reply::json(&ApiResponse { - success: true, - data: Some(response), - message: None, - }) - }); + // Spawn worker + let worker = worker::Worker::new(pool.clone()); + tokio::spawn(async move { worker.run().await }); - // API routes - you'll expand these for your hackathon needs - let api = warp::path("api") - .and( - health.or( - // Add more routes here as needed - warp::path("version") - .and(warp::get()) - .map(|| { - warp::reply::json(&ApiResponse { - success: true, - data: Some("1.0.0"), - message: Some("Rust Engine API".to_string()), - }) - }) - ) - ); - - let routes = api - .with(cors) + // API routes + let api_routes = api::routes(pool.clone()) + .with(warp::cors() + .allow_any_origin() + .allow_headers(vec!["content-type", "authorization"]) + .allow_methods(vec!["GET", "POST", "PUT", "DELETE", "OPTIONS"])) .with(warp::log("rust_engine")); info!("Rust Engine started on http://0.0.0.0:8000"); - warp::serve(routes) - .run(([0, 0, 0, 0], 8000)) - .await; - - Ok(()) -} - -async fn start_server_without_db() -> Result<(), Box> { - info!("Starting server in DB-less mode for development"); - - let cors = warp::cors() - .allow_any_origin() - .allow_headers(vec!["content-type", "authorization"]) - .allow_methods(vec!["GET", "POST", "PUT", "DELETE", "OPTIONS"]); - - let health = warp::path("health") - .and(warp::get()) - .map(|| { - let response = HealthResponse { - status: "healthy (no db)".to_string(), - timestamp: chrono::Utc::now().to_rfc3339(), - }; - warp::reply::json(&ApiResponse { - success: true, - data: Some(response), - message: Some("Running without database connection".to_string()), - }) - }); - - let routes = warp::path("api") - .and(health) - .with(cors) - .with(warp::log("rust_engine")); - - info!("Rust Engine started on http://0.0.0.0:8000 (DB-less mode)"); - info!("Rust Engine prepared!"); - - warp::serve(routes) + warp::serve(api_routes) .run(([0, 0, 0, 0], 8000)) .await; diff --git a/rust-engine/src/models.rs b/rust-engine/src/models.rs new file mode 100644 index 0000000..0ecad7c --- /dev/null +++ b/rust-engine/src/models.rs @@ -0,0 +1,56 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct FileRecord { + pub id: String, + pub filename: String, + pub path: String, + pub description: Option, + pub created_at: Option>, +} + +impl FileRecord { + pub fn new(filename: impl Into, path: impl Into, description: Option) -> Self { + Self { + id: Uuid::new_v4().to_string(), + filename: filename.into(), + path: path.into(), + description, + created_at: None, + } + } +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub enum QueryStatus { + Queued, + InProgress, + Completed, + Cancelled, + Failed, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct QueryRecord { + pub id: String, + pub status: QueryStatus, + pub payload: serde_json::Value, + pub result: Option, + pub created_at: Option>, + pub updated_at: Option>, +} + +impl QueryRecord { + pub fn new(payload: serde_json::Value) -> Self { + Self { + id: Uuid::new_v4().to_string(), + status: QueryStatus::Queued, + payload, + result: None, + created_at: None, + updated_at: None, + } + } +} diff --git a/rust-engine/src/storage.rs b/rust-engine/src/storage.rs new file mode 100644 index 0000000..95d6415 --- /dev/null +++ b/rust-engine/src/storage.rs @@ -0,0 +1,34 @@ +use anyhow::Result; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +pub fn storage_dir() -> PathBuf { + std::env::var("ASTRA_STORAGE") + .map(PathBuf::from) + .unwrap_or_else(|_| std::env::current_dir().unwrap().join("storage")) +} + +pub fn ensure_storage_dir() -> Result<()> { + let dir = storage_dir(); + if !dir.exists() { + fs::create_dir_all(&dir)?; + } + Ok(()) +} + +pub fn save_file(filename: &str, contents: &[u8]) -> Result { + ensure_storage_dir()?; + let mut path = storage_dir(); + path.push(filename); + let mut f = fs::File::create(&path)?; + f.write_all(contents)?; + Ok(path) +} + +pub fn delete_file(path: &Path) -> Result<()> { + if path.exists() { + fs::remove_file(path)?; + } + Ok(()) +} diff --git a/rust-engine/src/vector.rs b/rust-engine/src/vector.rs new file mode 100644 index 0000000..c34b0a3 --- /dev/null +++ b/rust-engine/src/vector.rs @@ -0,0 +1,24 @@ +use anyhow::Result; +use lazy_static::lazy_static; +use std::collections::HashMap; +use std::sync::Mutex; + +lazy_static! { + static ref VECTOR_STORE: Mutex>> = Mutex::new(HashMap::new()); +} + +pub fn store_embedding(id: &str, emb: Vec) -> Result<()> { + let mut s = VECTOR_STORE.lock().unwrap(); + s.insert(id.to_string(), emb); + Ok(()) +} + +pub fn query_top_k(_query_emb: &[f32], k: usize) -> Result> { + // Very naive: return up to k ids from the store. + let s = VECTOR_STORE.lock().unwrap(); + let mut out = Vec::new(); + for key in s.keys().take(k) { + out.push(key.clone()); + } + Ok(out) +} diff --git a/rust-engine/src/vector_db.rs b/rust-engine/src/vector_db.rs new file mode 100644 index 0000000..1ce45f5 --- /dev/null +++ b/rust-engine/src/vector_db.rs @@ -0,0 +1,87 @@ +use anyhow::Result; +use reqwest::Client; +use serde_json::json; +use serde::Deserialize; + +#[derive(Clone)] +pub struct QdrantClient { + base: String, + client: Client, +} + +impl QdrantClient { + pub fn new(base: &str) -> Self { + Self { + base: base.trim_end_matches('/').to_string(), + client: Client::new(), + } + } + + /// Upsert a point into collection `files` with id and vector + pub async fn upsert_point(&self, id: &str, vector: Vec) -> Result<()> { + let url = format!("{}/collections/files/points", self.base); + let body = json!({ + "points": [{ + "id": id, + "vector": vector, + "payload": {"type": "file"} + }] + }); + + let resp = self.client.post(&url).json(&body).send().await?; + let status = resp.status(); + if status.is_success() { + Ok(()) + } else { + let t = resp.text().await.unwrap_or_default(); + Err(anyhow::anyhow!("qdrant upsert failed: {} - {}", status, t)) + } + } + + /// Ensure the 'files' collection exists with the given dimension and distance metric + pub async fn ensure_files_collection(&self, dim: usize) -> Result<()> { + let url = format!("{}/collections/files", self.base); + let body = json!({ + "vectors": {"size": dim, "distance": "Cosine"} + }); + let resp = self.client.put(&url).json(&body).send().await?; + // 200 OK or 201 Created means ready; 409 Conflict means already exists + if resp.status().is_success() || resp.status().as_u16() == 409 { + Ok(()) + } else { + let status = resp.status(); + let t = resp.text().await.unwrap_or_default(); + Err(anyhow::anyhow!("qdrant ensure collection failed: {} - {}", status, t)) + } + } + + /// Search top-k nearest points from 'files' + pub async fn search_top_k(&self, vector: Vec, k: usize) -> Result> { + let url = format!("{}/collections/files/points/search", self.base); + let body = json!({ + "vector": vector, + "limit": k + }); + let resp = self.client.post(&url).json(&body).send().await?; + let status = resp.status(); + if !status.is_success() { + let t = resp.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!("qdrant search failed: {} - {}", status, t)); + } + #[derive(Deserialize)] + struct Hit { id: serde_json::Value } + #[derive(Deserialize)] + struct Data { result: Vec } + let data: Data = resp.json().await?; + let mut ids = Vec::new(); + for h in data.result { + // id can be string or number; handle string + if let Some(s) = h.id.as_str() { + ids.push(s.to_string()); + } else { + ids.push(h.id.to_string()); + } + } + Ok(ids) + } +} diff --git a/rust-engine/src/worker.rs b/rust-engine/src/worker.rs new file mode 100644 index 0000000..5b455a0 --- /dev/null +++ b/rust-engine/src/worker.rs @@ -0,0 +1,160 @@ +use crate::gemini_client::{demo_text_embedding, DEMO_EMBED_DIM}; +use crate::models::{QueryRecord, QueryStatus}; +use crate::vector_db::QdrantClient; +use anyhow::Result; +use sqlx::MySqlPool; +use std::time::Duration; +use tracing::{error, info}; + +pub struct Worker { + pool: MySqlPool, + qdrant: QdrantClient, +} + +impl Worker { + pub fn new(pool: MySqlPool) -> Self { + let qdrant_url = std::env::var("QDRANT_URL").unwrap_or_else(|_| "http://qdrant:6333".to_string()); + let qdrant = QdrantClient::new(&qdrant_url); + Self { pool, qdrant } + } + + pub async fn run(&self) { + info!("Worker starting"); + + // Ensure qdrant collection exists + if let Err(e) = self.qdrant.ensure_files_collection(DEMO_EMBED_DIM).await { + error!("Failed to ensure Qdrant collection: {}", e); + } + + // Requeue stale InProgress jobs older than cutoff (e.g., 10 minutes) + if let Err(e) = self.requeue_stale_inprogress(10 * 60).await { + error!("Failed to requeue stale jobs: {}", e); + } + + loop { + // Claim next queued query + match self.fetch_and_claim().await { + Ok(Some(mut q)) => { + info!("Processing query {}", q.id); + if let Err(e) = self.process_query(&mut q).await { + error!("Error processing {}: {}", q.id, e); + let _ = self.mark_failed(&q.id, &format!("{}", e)).await; + } + } + Ok(None) => { + tokio::time::sleep(Duration::from_secs(2)).await; + } + Err(e) => { + error!("Worker fetch error: {}", e); + tokio::time::sleep(Duration::from_secs(5)).await; + } + } + } + } + + async fn fetch_and_claim(&self) -> Result> { + // Note: MySQL transactional SELECT FOR UPDATE handling is more complex; for this hackathon scaffold + // we do a simple two-step: select one queued id, then update it to InProgress if it is still queued. + if let Some(row) = sqlx::query("SELECT id, payload FROM queries WHERE status = 'Queued' ORDER BY created_at LIMIT 1") + .fetch_optional(&self.pool) + .await? + { + use sqlx::Row; + let id: String = row.get("id"); + let payload: serde_json::Value = row.get("payload"); + + let updated = sqlx::query("UPDATE queries SET status = 'InProgress' WHERE id = ? AND status = 'Queued'") + .bind(&id) + .execute(&self.pool) + .await?; + + if updated.rows_affected() == 1 { + let mut q = QueryRecord::new(payload); + q.id = id; + q.status = QueryStatus::InProgress; + return Ok(Some(q)); + } + } + Ok(None) + } + + async fn process_query(&self, q: &mut QueryRecord) -> Result<()> { + // Stage 1: set InProgress (idempotent) + self.update_status(&q.id, QueryStatus::InProgress).await?; + + // Stage 2: embed query text + let text = q.payload.get("q").and_then(|v| v.as_str()).unwrap_or(""); + let emb = demo_text_embedding(text).await?; + + // Check cancellation + if self.is_cancelled(&q.id).await? { return Ok(()); } + + // Stage 3: search top-K in Qdrant + let top_ids = self.qdrant.search_top_k(emb, 5).await.unwrap_or_default(); + + // Check cancellation + if self.is_cancelled(&q.id).await? { return Ok(()); } + + // Stage 4: persist results + let result = serde_json::json!({ + "summary": format!("Found {} related files", top_ids.len()), + "related_file_ids": top_ids, + }); + sqlx::query("UPDATE queries SET status = 'Completed', result = ? WHERE id = ?") + .bind(result) + .bind(&q.id) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn update_status(&self, id: &str, status: QueryStatus) -> Result<()> { + let s = match status { + QueryStatus::Queued => "Queued", + QueryStatus::InProgress => "InProgress", + QueryStatus::Completed => "Completed", + QueryStatus::Cancelled => "Cancelled", + QueryStatus::Failed => "Failed", + }; + sqlx::query("UPDATE queries SET status = ? WHERE id = ?") + .bind(s) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn mark_failed(&self, id: &str, message: &str) -> Result<()> { + let result = serde_json::json!({"error": message}); + sqlx::query("UPDATE queries SET status = 'Failed', result = ? WHERE id = ?") + .bind(result) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn requeue_stale_inprogress(&self, age_secs: i64) -> Result<()> { + // MySQL: requeue items updated_at < now()-age and status = InProgress + sqlx::query( + "UPDATE queries SET status = 'Queued' WHERE status = 'InProgress' AND updated_at < (NOW() - INTERVAL ? SECOND)" + ) + .bind(age_secs) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn is_cancelled(&self, id: &str) -> Result { + if let Some(row) = sqlx::query("SELECT status FROM queries WHERE id = ?") + .bind(id) + .fetch_optional(&self.pool) + .await? + { + use sqlx::Row; + let s: String = row.get("status"); + return Ok(s == "Cancelled"); + } + Ok(false) + } +} From 1912ab2e53ba12ca6c3b19340664ddb991328d28 Mon Sep 17 00:00:00 2001 From: Christbru Date: Sun, 19 Oct 2025 03:54:11 -0500 Subject: [PATCH 2/2] Add qdrant docker to server build file --- docker-compose.prod.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 1f773d7..4a9c3f9 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -46,5 +46,17 @@ services: depends_on: - mysql + qdrant: + image: qdrant/qdrant:latest + restart: unless-stopped + ports: + - "127.0.0.1:6333:6333" + volumes: + - qdrant-data:/qdrant/storage + environment: + - QDRANT__SERVICE__GRPC_PORT=6334 + # expose to rust-engine via service name 'qdrant' + volumes: mysql-data: + qdrant-data: