Added uploading file functionality

Cargo.toml

@@ -68,6 +68,8 @@ env_logger = "0.11"
 lettre = { version = "0.11", features = ["tokio1", "smtp-transport", "tokio1-native-tls", "pool"]}
 unicase = "2.8.0"
 sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "sqlite", "macros"] }
+lopdf = "0.34.0"
+docx-rs = "0.4.17"
 # triple_accel = "0.4.0"
 
 [build-dependencies]

src/kb/crud.rs

@@ -1,20 +1,29 @@
 use std::path::Path;
 
-use axum::{extract::{Multipart, Query}, response::IntoResponse};
+use axum::{
+    extract::{Multipart, Query},
+    response::IntoResponse,
+};
 
+use super::doc;
 use crate::result::{Error, Result};
 use crate::robot::dto::RobotQuery;
 use crate::web::server::to_res;
 
-pub(crate) async fn upload(Query(q):Query<RobotQuery>,multipart: Multipart) -> impl IntoResponse {
-    if let Err(e) = do_upload(&q.robot_id,multipart).await {
+pub(crate) async fn upload(Query(q): Query<RobotQuery>, multipart: Multipart) -> impl IntoResponse {
+    if let Err(e) = do_uploading(&q.robot_id, multipart).await {
         return to_res(Err(e));
     }
     to_res(Ok(()))
 }
 
-async fn do_upload(robot_id:&str,mut multipart: Multipart) -> Result<()> {
-    let p = Path::new(".").join("data").join("kb").join("docs").join("upload").join(robot_id);
+async fn do_uploading(robot_id: &str, mut multipart: Multipart) -> Result<()> {
+    let p = Path::new(".")
+        .join("data")
+        .join(robot_id)
+        .join("kb")
+        .join("docs")
+        .join("upload");
     if !p.exists() {
         std::fs::create_dir_all(&p)?;
     }
@@ -29,11 +38,15 @@ async fn do_upload(robot_id:&str,mut multipart: Multipart) -> Result<()> {
         };
         let name = name.to_string();
         let Some(file_name) = field.file_name() else {
-            return Err(Error::ErrorWithMessage(String::from("File name is missing.")))
+            return Err(Error::ErrorWithMessage(String::from(
+                "File name is missing.",
+            )));
         };
         let file_name = file_name.to_string();
         let Some(content_type) = field.content_type() else {
-            return Err(Error::ErrorWithMessage(String::from("Content type is missing.")))
+            return Err(Error::ErrorWithMessage(String::from(
+                "Content type is missing.",
+            )));
         };
         let content_type = content_type.to_string();
         let data = field.bytes().await?;
@@ -42,7 +55,10 @@ async fn do_upload(robot_id:&str,mut multipart: Multipart) -> Result<()> {
             "Length of `{name}` (`{file_name}`: `{content_type}`) is {} bytes",
             data.len()
         );
+
+        let text = doc::parse_docx(data.to_vec())?;
+        log::info!("Extract text: {text}");
     }
 }
 
-pub(crate) async fn new_qa() -> impl IntoResponse {}
+pub(crate) async fn new_qa() -> impl IntoResponse {}

src/kb/doc.rs

@@ -0,0 +1,56 @@
+// use std::fs::File;
+// use std::io::Read;
+// use std::path::Path;
+use std::vec::Vec;
+
+use docx_rs::read_docx;
+
+use crate::result::Result;
+
+pub(super) fn parse_docx(buf: Vec<u8>) -> Result<String> {
+    // let mut file = File::open("./numbering.docx")?;
+    // let mut buf = Vec::with_capacity(3096);
+    // file.read_to_end(&mut buf)?;
+    let mut doc_text = String::with_capacity(3096);
+    let docx = read_docx(&buf)?;
+    let doc = docx.document;
+    for d in doc.children.iter() {
+        match d {
+            docx_rs::DocumentChild::Paragraph(paragraph) => {
+                for p in paragraph.children() {
+                    match p {
+                        docx_rs::ParagraphChild::Run(run) => {
+                            for r in run.children.iter() {
+                                match r {
+                                    docx_rs::RunChild::Text(text) => {
+                                        // log::info!("Docx text={}", text.text);
+                                        doc_text.push_str(&text.text);
+                                        doc_text.push('\n');
+                                        doc_text.push('\n');
+                                    }
+                                    docx_rs::RunChild::Sym(sym) => {
+                                        doc_text.push_str(&sym.char);
+                                    }
+                                    docx_rs::RunChild::Break(_) => {
+                                        doc_text.push('\n');
+                                    }
+                                    _ => {}
+                                }
+                            }
+                        }
+                        docx_rs::ParagraphChild::Hyperlink(hyperlink) => {
+                            log::info!("hyperlink: {:?}", hyperlink.link)
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            docx_rs::DocumentChild::Table(_table) => {}
+            docx_rs::DocumentChild::TableOfContents(_table_of_contents) => {}
+            _ => {}
+        }
+    }
+    Ok(doc_text)
+}
+
+fn parse_pdf() {}

src/kb/dto.rs

@@ -7,4 +7,4 @@ pub(super) struct QuestionAnswer {
     pub(super) question: String,
     pub(super) similar_questions: Option<Vec<String>>,
     pub(super) answer: String,
-}
+}

src/kb/mod.rs

@@ -1,2 +1,3 @@
 pub(crate) mod crud;
-pub(crate) mod dto;
+pub(crate) mod doc;
+pub(crate) mod dto;

src/result/mod.rs

@@ -194,3 +194,9 @@ impl From<axum::extract::multipart::MultipartError> for Error {
         Error::ErrorWithMessage(format!("Multipart error: {:?}", err))
     }
 }
+
+impl From<docx_rs::ReaderError> for Error {
+    fn from(err: docx_rs::ReaderError) -> Self {
+        Error::ErrorWithMessage(format!("Read docx file failed: {:?}", err))
+    }
+}

src/web/server.rs

@@ -233,10 +233,7 @@ fn gen_router() -> Router {
             "/management/settings/model/check/embedding",
             get(settings::check_embedding_model),
         )
-        .route(
-            "/kb/doc/upload",
-            post(kb::upload),
-        )
+        .route("/kb/doc/upload", post(kb::upload))
         .route("/management/settings/smtp/test", post(settings::smtp_test))
         .route("/flow/answer", post(rt::answer))
         .route("/flow/answer/sse", post(rt::answer_sse))