Merge pull request #56 from EETagent/finish_cli

Finish cli

Cargo.lock

@@ -354,6 +354,18 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "bstr"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
+dependencies = [
+ "lazy_static",
+ "memchr",
+ "regex-automata",
+ "serde",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.11.1"
@@ -645,6 +657,28 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "csv"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1"
+dependencies = [
+ "bstr",
+ "csv-core",
+ "itoa 0.4.8",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "ctr"
 version = "0.9.2"
@@ -1226,7 +1260,7 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
 dependencies = [
  "bytes",
  "fnv",
- "itoa",
+ "itoa 1.0.4",
 ]
 
 [[package]]
@@ -1267,7 +1301,7 @@ dependencies = [
  "http-body",
  "httparse",
  "httpdate",
- "itoa",
+ "itoa 1.0.4",
  "pin-project-lite",
  "socket2",
  "tokio",
@@ -1456,6 +1490,12 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itoa"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
+
 [[package]]
 name = "itoa"
 version = "1.0.4"
@@ -2019,6 +2059,7 @@ dependencies = [
  "async_zip",
  "base64",
  "chrono",
+ "csv",
  "dotenv",
  "futures",
  "infer",
@@ -2706,7 +2747,7 @@ version = "1.0.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ce777b7b150d76b9cf60d28b55f5847135a003f7d7350c6be7a773508ce7d45"
 dependencies = [
- "itoa",
+ "itoa 1.0.4",
  "ryu",
  "serde",
 ]
@@ -2862,7 +2903,7 @@ dependencies = [
  "hkdf",
  "hmac",
  "indexmap",
- "itoa",
+ "itoa 1.0.4",
  "libc",
  "libsqlite3-sys",
  "log",
@@ -3060,7 +3101,7 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"
 dependencies = [
- "itoa",
+ "itoa 1.0.4",
  "serde",
  "time-core",
  "time-macros",

cli/Cargo.toml

@@ -14,6 +14,7 @@ portfolio-core = { path = "../core" }
 [dependencies.tokio]
 version = "^1.21"
 features = [
+    "process",
     "macros",
 ]
 

cli/src/main.rs

@@ -1,12 +1,47 @@
 use std::path::PathBuf;
+
+use clap::{arg, ArgAction, ArgMatches, command, Command, value_parser};
+use sea_orm::{Database, DatabaseConnection, DbConn};
 use url::Url;
 
-use clap::{arg, command, value_parser, ArgAction, Command};
+use portfolio_core::{crypto, Query};
-use sea_orm::{Database, DatabaseConnection};
+use portfolio_core::services::portfolio_service::{FileType};
 
-use ::entity::candidate::Entity as Candidate;
+async fn get_admin_private_key(db: &DbConn, sub_matches: &ArgMatches) -> Result<String, Box<dyn std::error::Error>> {
-use ::entity::parent::Entity as Parent;
+    Ok(match (sub_matches.get_one::<String>("key"), sub_matches.get_one::<String>("password")) {
-use sea_orm::*;
+        (Some(key), _) => {
+            key.to_string()
+        },
+        (_, Some(password)) => {
+            let admin_id = if let Some(s) = sub_matches.get_one::<String>("admin_id") {
+                s.parse::<i32>().unwrap()
+            } else {
+                return Err("Admin ID required")?;
+            };
+            let admin = Query::find_admin_by_id(&db, admin_id)
+                .await
+                .map_err(|e| format!("Admin {} not found: {}", admin_id, e))?
+                .ok_or("Admin not found")?;
+            crypto::decrypt_password(
+                admin.private_key,
+                password.to_string()
+            ).await?
+
+        },
+        _ => {
+            return Err("Either key or password must be provided")?;
+        }
+    })
+}
+
+async fn get_db_conn(sub_matches: &ArgMatches) -> Result<DbConn, Box<dyn std::error::Error>> {
+    let db_url = sub_matches.get_one::<Url>("database").unwrap();
+    if db_url.scheme() != "sqlite" && db_url.scheme() != "postgres" {
+        return Err("URL scheme postgres:// or sqlite:// required")?;
+    }
+    let db: DatabaseConnection = Database::connect(db_url.as_str()).await?;
+    Ok(db)
+}
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -15,8 +50,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .subcommand_required(true)
         .arg_required_else_help(true)
         .subcommand(
-            Command::new("portfolio")
+            Command::new("export")
-                .about("Database & Portfolio operations")
+                .about("Export all candidate data to a CSV file")
+                .arg(
+                    arg!(
+                        -o --output <PATH> "Output file path"
+                    )
+                    .required(true)
+                    .value_parser(value_parser!(PathBuf)),
+                )
                 .arg(
                     arg!(
                         -d --database <URL> "URL to the database or sql file with postgres:// or sqlite://"
@@ -27,16 +69,114 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 )
                 .arg(
                     arg!(
-                        -p --portfolio <PATH>  "Path to the portfolio root"
+                        -k --key <KEY> "AGE private key for decryption"
+                    )
+                    .required(false),
+                )
+                .arg(
+                    arg!(
+                        -p --password <PASSWORD> "Password for decryption"
+                    )
+                    .required(false),
+                )
+                .arg(
+                    arg!(
+                        -a --admin_id <ADMIN_ID> "Admin ID"
+                    )
+                    .required(false),
+                )
+        )
+        .subcommand(
+            Command::new("portfolio")
+                .about("Portfolio file operations")
+                .arg(
+                    arg!(
+                        -f --file <PATH>  "Age file path"
                     )
                     .required(true)
                     .value_parser(value_parser!(PathBuf)),
                 )
+                .arg(
+                    arg!(
+                        -o --output <PATH> "Output file path"
+                    )
+                        .required(true)
+                        .value_parser(value_parser!(PathBuf)),
+                )
+                .arg(
+                    arg!(
+                        -d --database <URL> "URL to the database or sql file with postgres:// or sqlite://"
+                    )
+                        .alias("url")
+                        .required(true)
+                        .value_parser(value_parser!(Url)),
+                )
                 .arg(
                     arg!(
                         -k --key <KEY> "AGE private key for decryption"
                     )
-                    .required(true),
+                        .required(false),
+                )
+                .arg(
+                    arg!(
+                        -p --password <PASSWORD> "Password for decryption"
+                    )
+                        .required(false),
+                )
+                .arg(
+                    arg!(
+                        -a --admin_id <ADMIN_ID> "Admin ID"
+                    )
+                        .required(false),
+                )
+        )
+        .subcommand(
+            Command::new("package")
+                .about("Package all data into one zip")
+                .arg(
+                    arg!(
+                        -s --pg_dump "Backup SQL database with pg_dump (PostgreSQL only)"
+                    )
+                )
+                .arg(
+                    arg!(
+                        -r --root_dir <PATH> "Portfolio root directory"
+                    )
+                        .required(true)
+                        .value_parser(value_parser!(PathBuf)),
+                )
+                .arg(
+                    arg!(
+                        -o --output <PATH> "Output file path"
+                    )
+                        .required(true)
+                        .value_parser(value_parser!(PathBuf)),
+                )
+                .arg(
+                    arg!(
+                        -d --database <URL> "URL to the database or sql file with postgres:// or sqlite://"
+                    )
+                        .alias("url")
+                        .required(true)
+                        .value_parser(value_parser!(Url)),
+                )
+                .arg(
+                    arg!(
+                        -k --key <KEY> "AGE private key for decryption"
+                    )
+                        .required(false),
+                )
+                .arg(
+                    arg!(
+                        -p --password <PASSWORD> "Password for decryption"
+                    )
+                        .required(false),
+                )
+                .arg(
+                    arg!(
+                        -a --admin_id <ADMIN_ID> "Admin ID"
+                    )
+                        .required(false),
                 )
         )
         .subcommand(
@@ -98,29 +238,64 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .get_matches();
 
     match clap.subcommand() {
+        Some(("export", sub_matches)) => {
+            let db = get_db_conn(sub_matches).await?;
+            let key = get_admin_private_key(&db, sub_matches).await?;
+
+            let output = sub_matches.get_one::<PathBuf>("output").unwrap();
+            let csv = portfolio_core::utils::csv::export(&db, key).await?;
+            tokio::fs::write(output, csv).await?;
+        },
         Some(("portfolio", sub_matches)) => {
-            let sqlite_url = sub_matches.get_one::<Url>("database").unwrap();
+            let db = get_db_conn(sub_matches).await?;
+            let key = get_admin_private_key(&db, sub_matches).await?;
 
-            println!("Connecting to {:?}", sqlite_url);
+            let age_file_path = sub_matches.get_one::<PathBuf>("file").unwrap();
 
-            if sqlite_url.scheme() != "sqlite" && sqlite_url.scheme() != "postgres" {
+            let decrypted = crypto::decrypt_file_with_private_key_as_buffer(age_file_path, &key).await?;
-                return Err("URL scheme postgres:// or sqlite:// required")?;
+
+            let output = sub_matches.get_one::<PathBuf>("output").unwrap();
+            tokio::fs::write(output, decrypted).await?;
+        },
+        Some(("package", sub_matches)) => { // TODO: compress the output directory into one file???
+            let db_url = sub_matches.get_one::<Url>("database").unwrap();
+            let db = get_db_conn(sub_matches).await?;
+            let key = get_admin_private_key(&db, sub_matches).await?;
+
+            let portfolio_root_dir = sub_matches.get_one::<PathBuf>("root_dir").unwrap();
+            let output = sub_matches.get_one::<PathBuf>("output").unwrap();
+            tokio::fs::create_dir_all(&output).await?;
+
+            let csv = portfolio_core::utils::csv::export(&db, key.to_string()).await?;
+            tokio::fs::write(output.join("personal_data.csv"), csv).await?;
+            println!("Exported personal data to personal_data.csv");
+
+            let ids: Vec<i32> = Query::list_all_candidate_ids(&db)
+                .await?
+                .iter()
+                .map(|application_id| application_id.to_i32())
+                .collect();
+            for id in ids {
+                let file_path = portfolio_root_dir.join(&id.to_string()).join(FileType::Age.as_str());
+                println!("{}", file_path.display());
+                let output_path = output.join(&id.to_string());
+                if let Ok(portfolio) = crypto::decrypt_file_with_private_key_as_buffer(file_path, &key).await {
+                    tokio::fs::create_dir_all(&output_path).await?;
+                    tokio::fs::write(&output_path.join(FileType::PortfolioZip.as_str()), portfolio).await?;
+                };
             }
+            println!("Exported all portfolios");
 
-            let db: DatabaseConnection = Database::connect(sqlite_url.as_str()).await?;
+            if *sub_matches.get_one::<bool>("pg_dump").unwrap_or(&false) {
+                let file = std::fs::File::create(&output.join("pg_dump.sql"))?;
+                tokio::process::Command::new("pg_dump")
+                    .args(&[db_url.as_str()])
+                    .stdout(file)
+                    .spawn()
+                    .expect("failed to start pg_dump");
+            }
+            println!("Exported database");
 
-            let entries = Candidate::find()
-                .join_rev(
-                    JoinType::InnerJoin,
-                    Parent::belongs_to(Candidate)
-                        .from(::entity::parent::Column::Application)
-                        .to(::entity::candidate::Column::Application)
-                        .into(),
-                )
-                .all(&db)
-                .await?;
-
-            println!("Found {} entries", entries.len());
         }
         Some(("hash", sub_matches)) => {
             let input = sub_matches.get_one::<String>("input").unwrap();

core/Cargo.toml

@@ -12,6 +12,9 @@ portfolio-entity = { path = "../entity" }
 # serde
 serde = { version = "^1.0", features = ["derive"] }
 
+# csv
+csv = "1.1"
+
 # error
 
 thiserror = "^1.0"

core/src/candidate_details.rs

@@ -1,8 +1,9 @@
 use chrono::NaiveDate;
-use entity::{candidate, parent};
 use serde::{Deserialize, Serialize};
 
-use crate::{crypto, error::ServiceError};
+use entity::{candidate, parent};
+
+use crate::{crypto, database::query::candidate::CandidateWithParent, error::ServiceError};
 
 pub const NAIVE_DATE_FMT: &str = "%Y-%m-%d";
 
@@ -184,6 +185,33 @@ impl TryFrom<(candidate::Model, parent::Model)> for EncryptedApplicationDetails
     }
 }
 
+impl TryFrom<CandidateWithParent> for EncryptedApplicationDetails {
+    type Error = ServiceError;
+
+    fn try_from(
+        cp: CandidateWithParent,
+    ) -> Result<Self, Self::Error> {
+        Ok(EncryptedApplicationDetails {
+            name: EncryptedString::try_from(cp.name)?,
+            surname: EncryptedString::try_from(cp.surname)?,
+            birthplace: EncryptedString::try_from(cp.birthplace)?,
+            birthdate: EncryptedString::try_from(cp.birthdate)?,
+            address: EncryptedString::try_from(cp.address)?,
+            telephone: EncryptedString::try_from(cp.telephone)?,
+            citizenship: EncryptedString::try_from(cp.citizenship)?,
+            email: EncryptedString::try_from(cp.email)?,
+            sex: EncryptedString::try_from(cp.sex)?,
+            study: cp.study.ok_or(ServiceError::CandidateDetailsNotSet)?,
+
+            parent_name: EncryptedString::try_from(cp.parent_name)?,
+            parent_surname: EncryptedString::try_from(cp.parent_surname)?,
+            parent_telephone: EncryptedString::try_from(cp.parent_telephone)?,
+            parent_email: EncryptedString::try_from(cp.parent_email)?,
+        })
+    }
+}
+
+
 
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct ApplicationDetails {

core/src/database/query/candidate.rs

@@ -1,9 +1,9 @@
-use crate::{Query};
-
-use ::entity::{candidate, candidate::Entity as Candidate, parent};
 use sea_orm::*;
 use serde::Serialize;
 
+use ::entity::{candidate, candidate::Entity as Candidate, parent};
+
+use crate::Query;
 
 pub const PAGE_SIZE: u64 = 20;
 
@@ -19,6 +19,38 @@ pub struct CandidateParentResult {
     pub parent_surname: Option<String>,
 }
 
+#[derive(FromQueryResult, Serialize, Default)]
+pub struct CandidateWithParent { // TODO: use this instead of (Candidate, Parent)???
+    pub application: i32,
+    pub name: Option<String>,
+    pub surname: Option<String>,
+    pub birthplace: Option<String>,
+    pub birthdate: Option<String>,
+    pub address: Option<String>,
+    pub telephone: Option<String>,
+    pub citizenship: Option<String>,
+    pub email: Option<String>,
+    pub sex: Option<String>,
+    pub study: Option<String>,
+    pub personal_identification_number: Option<String>,
+
+    pub parent_name: Option<String>,
+    pub parent_surname: Option<String>,
+    pub parent_telephone: Option<String>,
+    pub parent_email: Option<String>,
+}
+
+#[derive(FromQueryResult)]
+pub struct ApplicationId {
+    application: i32,
+}
+
+impl ApplicationId {
+    pub fn to_i32(&self) -> i32 {
+        self.application
+    }
+}
+
 impl Query {
     pub async fn find_candidate_by_id(
         db: &DbConn,
@@ -53,14 +85,42 @@ impl Query {
             query.fetch().await
         }
     }
+
+
+    pub async fn list_all_candidates_with_parents(
+        db: &DbConn,
+    ) -> Result<Vec<CandidateWithParent>, DbErr> {
+        Candidate::find()
+            .order_by(candidate::Column::Application, Order::Asc)
+            .join(JoinType::InnerJoin, candidate::Relation::Parent.def())
+            .column_as(parent::Column::Name, "parent_name")
+            .column_as(parent::Column::Surname, "parent_surname")
+            .column_as(parent::Column::Telephone, "parent_telephone")
+            .column_as(parent::Column::Email, "parent_email")
+            .into_model::<CandidateWithParent>()
+            .all(db)
+            .await
+    }
+
+    pub async fn list_all_candidate_ids(
+        db: &DbConn,
+    ) -> Result<Vec<ApplicationId>, DbErr> {
+        Candidate::find()
+            .order_by(candidate::Column::Application, Order::Asc)
+            .column(candidate::Column::Application)
+            .into_model::<ApplicationId>()
+            .all(db)
+            .await
+    }
     
 }
 
 #[cfg(test)]
 mod tests {
-    use entity::candidate;
     use sea_orm::{ActiveModelTrait, Set};
 
+    use entity::candidate;
+
     use crate::Query;
     use crate::util::get_memory_sqlite_connection;
 

core/src/error.rs

@@ -59,7 +59,11 @@ pub enum ServiceError {
     #[error("Portfolio is incomplete")]
     IncompletePortfolio,
     #[error("Zip error")]
-    ZipError(#[from] async_zip::error::ZipError)
+    ZipError(#[from] async_zip::error::ZipError),
+    #[error("Csv error")]
+    CsvError(#[from] csv::Error),
+    #[error("Csv into inner error")]
+    CsvIntoInnerError,
 }
 
 impl ServiceError {
@@ -94,6 +98,8 @@ impl ServiceError {
             //TODO: Correct code
             ServiceError::IncompletePortfolio => 406,
             ServiceError::ZipError(_) => 500,
+            ServiceError::CsvError(_) => 500,
+            ServiceError::CsvIntoInnerError => 500,
         }
     }
 }

core/src/lib.rs

@@ -1,3 +1,8 @@
+pub use sea_orm;
+
+pub use database::mutation::*;
+pub use database::query::*;
+
 pub mod database;
 pub mod crypto;
 pub mod filetype;
@@ -6,8 +11,5 @@ pub mod error;
 pub mod candidate_details;
 pub mod util;
 pub mod responses;
+pub mod utils;
 
-pub use database::mutation::*;
-pub use database::query::*;
-
-pub use sea_orm;

core/src/utils/csv.rs

@@ -0,0 +1,59 @@
+use sea_orm::{DbConn};
+use crate::{error::ServiceError, candidate_details::{EncryptedApplicationDetails, ApplicationDetails}, Query, database::query::candidate::CandidateWithParent};
+
+
+type Row = CandidateWithParent;
+
+impl From<(i32, ApplicationDetails)> for Row {
+    fn from((application, d): (i32, ApplicationDetails)) -> Self {
+        Self {
+            application,
+            name: Some(d.name),
+            surname: Some(d.surname),
+            birthplace: Some(d.birthplace),
+            birthdate: Some(d.birthdate.to_string()),
+            address: Some(d.address),
+            telephone: Some(d.telephone),
+            citizenship: Some(d.citizenship),
+            email: Some(d.email),
+            sex: Some(d.sex),
+            study: Some(d.study),
+            personal_identification_number: None, // TODO
+
+            parent_name: Some(d.parent_name),
+            parent_surname: Some(d.parent_surname),
+            parent_telephone: Some(d.parent_telephone),
+            parent_email: Some(d.parent_email),
+        }
+    }
+}
+
+pub async fn export(
+    db: &DbConn,
+    private_key: String,
+) -> Result<Vec<u8>, ServiceError> {
+    let mut wtr = csv::Writer::from_writer(vec![]);
+
+    let candidates_with_parents = Query::list_all_candidates_with_parents(&db).await?;
+    for candidate in candidates_with_parents {
+        let application = candidate.application;
+
+        let row: Row = match EncryptedApplicationDetails::try_from(candidate) {
+            Ok(d) => Row::from(
+                d
+                    .decrypt(private_key.to_string())
+                    .await
+                    .map(|d| (application, d))?
+            ),
+
+            Err(_) => Row {
+                application,
+                ..Default::default()
+            }
+        };
+        wtr.serialize(row)?;
+    }
+    wtr
+        .into_inner()
+        .map_err(|_| ServiceError::CsvIntoInnerError)
+}

core/src/utils/mod.rs

@@ -0,0 +1 @@
+pub mod csv;