From ce708d0c983323fcd2153a9a33f9030c328efa23 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sun, 5 Mar 2023 23:18:33 +0300 Subject: [PATCH 01/10] initial payload table impl --- Cargo.lock | 2 + Cargo.toml | 1 + bin/u_agent/src/lib.rs | 11 +- bin/u_panel/Cargo.toml | 2 +- bin/u_panel/src/argparse.rs | 18 +- bin/u_server/Cargo.toml | 1 + bin/u_server/src/db.rs | 131 ++++++----- bin/u_server/src/handlers.rs | 98 ++++---- bin/u_server/src/u_server.rs | 22 +- integration/Cargo.toml | 1 + integration/tests/fixtures/agent.rs | 53 +++-- integration/tests/fixtures/connections.rs | 28 +++ integration/tests/fixtures/env.rs | 16 ++ integration/tests/fixtures/mod.rs | 8 + integration/tests/helpers/mod.rs | 5 - integration/tests/integration/api.rs | 1 + integration/tests/integration/behaviour.rs | 11 +- integration/tests/integration/connection.rs | 7 +- lib/u_lib/src/api.rs | 18 +- lib/u_lib/src/cache.rs | 11 +- lib/u_lib/src/db.rs | 9 +- lib/u_lib/src/jobs.rs | 193 +++++++--------- .../src/{messaging/mod.rs => messaging.rs} | 10 +- lib/u_lib/src/messaging/files.rs | 8 - lib/u_lib/src/misc.rs | 2 +- lib/u_lib/src/models/jobs/assigned.rs | 10 +- lib/u_lib/src/models/jobs/meta.rs | 209 ++++++++++++------ lib/u_lib/src/models/payload.rs | 90 ++------ lib/u_lib/src/models/schema.rs | 27 +-- lib/u_lib/src/ufs/mod.rs | 88 +++++++- .../2020-10-24-111622_create_all/down.sql | 9 +- .../2020-10-24-111622_create_all/up.sql | 27 +-- 32 files changed, 608 insertions(+), 519 deletions(-) create mode 100644 integration/tests/fixtures/connections.rs create mode 100644 integration/tests/fixtures/env.rs create mode 100644 integration/tests/integration/api.rs rename lib/u_lib/src/{messaging/mod.rs => messaging.rs} (80%) delete mode 100644 lib/u_lib/src/messaging/files.rs diff --git a/Cargo.lock b/Cargo.lock index 8d0a08b..9e3b502 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1195,6 +1195,7 @@ dependencies = [ name = "integration" version = "0.1.0" dependencies = [ + "futures", "once_cell", "reqwest", "rstest", @@ -2690,6 +2691,7 @@ dependencies = [ "deadpool-diesel", "diesel", "hyper", + "mime_guess", "once_cell", "openssl", "rstest", diff --git a/Cargo.toml b/Cargo.toml index 224420d..850aad0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ anyhow = "=1.0.63" deadpool-diesel = "0.4.0" diesel = { version = "2", features = ["postgres", "uuid"] } +mime_guess = "2.0" openssl = "0.10" reqwest = { version = "0.11", features = ["json"] } serde = { version = "1.0", features = ["derive"] } diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 08cc6a3..7f08635 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -10,7 +10,7 @@ use u_lib::{ config::{get_self_id, EndpointsEnv, AGENT_ITERATION_INTERVAL}, error::ErrChan, executor::pop_completed, - jobs::{fat_meta_to_thin, AnonymousJobBatch}, + jobs::{split_payload, AnonymousJobBatch}, logging::init_logger, messaging::Reportable, models::AssignedJobById, @@ -30,8 +30,8 @@ pub async fn process_request(jobs: Vec, client: &ClientHandler) } } }; - match fat_meta_to_thin(fetched_job) { - Ok(thin_meta) => JobCache::insert(thin_meta), + match split_payload(fetched_job) { + Ok(job_payload_meta) => JobCache::insert(job_payload_meta), Err(e) => ErrChan::send(e, "pld").await, } } @@ -109,7 +109,7 @@ pub fn run_forever() -> ! { let env = EndpointsEnv::load(); if cfg!(debug_assertions) { - init_logger(Some(format!( + let logfile_uid = format!( "u_agent-{}", get_self_id() .hyphenated() @@ -117,7 +117,8 @@ pub fn run_forever() -> ! { .split("-") .next() .unwrap() - ))); + ); + init_logger(Some(logfile_uid)); } else { #[cfg(unix)] u_lib::unix::daemonize() diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index e6705f6..083c7e0 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -11,7 +11,7 @@ actix-cors = "0.6.1" actix-web = "4.1" anyhow = { workspace = true } futures-util = "0.3.21" -mime_guess = "2.0.4" +mime_guess = { workspace = true } once_cell = "1.8.0" rust-embed = { version = "6.3.0", features = ["debug-embed", "compression"] } serde = { workspace = true } diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 8a13e99..eeaf168 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -2,8 +2,9 @@ use serde_json::{from_str, to_value, Value}; use structopt::StructOpt; use u_lib::{ api::ClientHandler, + jobs::join_payload, messaging::AsMsg, - models::{Agent, AssignedJob, FatJobMeta}, + models::{Agent, AssignedJob, RawJob}, types::Id, types::PanelResult, UError, UResult, @@ -87,14 +88,11 @@ pub async fn process_cmd(client: ClientHandler, args: Args) -> PanelResult match action { JobCRUD::Create { job } => { - let raw_job = from_str::(&job)?; - let mut job = raw_job.validated()?; - - if let Some(payload) = &mut job.payload { - payload.read_into_self()?; - } + let raw_job = from_str::(&job)?; + let job = raw_job.validated()?; + let fat_job = join_payload(job)?; - into_value(client.upload_jobs(job).await?) + into_value(client.upload_jobs(fat_job).await?) } JobCRUD::RUD(RUD::Read { id }) => match id { //todo: use vec not to break frontend api, possibly refactor later @@ -102,9 +100,9 @@ pub async fn process_cmd(client: ClientHandler, args: Args) -> PanelResult into_value(client.get_jobs().await?), }, JobCRUD::RUD(RUD::Update { item }) => { - let raw_job = from_str::(&item)?; + let raw_job = from_str::(&item)?; let job = raw_job.validated()?; - into_value(client.update_job(job).await?) + into_value(client.update_job(join_payload(job)?).await?) } JobCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), }, diff --git a/bin/u_server/Cargo.toml b/bin/u_server/Cargo.toml index ee1e2a8..9b800b2 100644 --- a/bin/u_server/Cargo.toml +++ b/bin/u_server/Cargo.toml @@ -9,6 +9,7 @@ anyhow = { workspace = true } diesel = { workspace = true } deadpool-diesel = { workspace = true } hyper = "0.14" +mime_guess = { workspace = true } once_cell = "1.7.2" openssl = { workspace = true } serde = { workspace = true } diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index 23b1dc1..6d43f96 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -1,7 +1,8 @@ use crate::error::Error; use diesel::{pg::PgConnection, prelude::*, result::Error as DslError, Connection}; +use std::mem::drop; use u_lib::db::PgAsyncPool; -use u_lib::models::{schema, Agent, AssignedJob, JobState, ThinJobMeta}; +use u_lib::models::{schema, Agent, AssignedJob, JobModel, JobState, PayloadMeta, ThinJob}; use u_lib::platform::Platform; use u_lib::types::Id; @@ -47,27 +48,46 @@ pub struct UDB<'c> { } impl UDB<'_> { - pub fn insert_jobs(&mut self, job_metas: &[ThinJobMeta]) -> Result> { - use schema::jobs; + pub fn insert_jobs(&mut self, jobs: &[ThinJob]) -> Result<()> { + use schema::{jobs, payloads}; + + let (jobs, payloads_opt): (Vec<_>, Vec<_>) = jobs + .iter() + .map(|j| (&j.job, j.payload_meta.as_ref())) + .unzip(); + + let payloads = payloads_opt + .into_iter() + .filter_map(|p| p) + .collect::>(); + + diesel::insert_into(payloads::table) + .values(payloads) + .execute(self.conn) + .map(drop) + .map_err(with_err_ctx("Can't insert payloads"))?; diesel::insert_into(jobs::table) - .values(job_metas) - .get_results(self.conn) - .map(|rows| rows.iter().map(|job: &ThinJobMeta| job.id).collect()) + .values(jobs) + .execute(self.conn) + .map(drop) .map_err(with_err_ctx("Can't insert jobs")) } - pub fn get_job(&mut self, id: Id) -> Result> { - use schema::jobs; + pub fn get_job(&mut self, id: Id) -> Result> { + use schema::{jobs, payloads}; - jobs::table + let maybe_job_with_payload = jobs::table + .left_join(payloads::table) .filter(jobs::id.eq(id)) - .first(self.conn) + .first::<(JobModel, Option)>(self.conn) .optional() - .map_err(with_err_ctx(format!("Can't get job {id}"))) + .map_err(with_err_ctx(format!("Can't get job {id}")))?; + + Ok(maybe_job_with_payload.map(|(job, payload_meta)| ThinJob { job, payload_meta })) } - pub fn get_jobs(&mut self) -> Result> { + pub fn get_jobs(&mut self) -> Result> { use schema::jobs; jobs::table @@ -75,27 +95,17 @@ impl UDB<'_> { .map_err(with_err_ctx("Can't get jobs")) } - pub fn find_job_by_alias(&mut self, alias: &str) -> Result> { - use schema::jobs; + pub fn find_job_by_alias(&mut self, alias: &str) -> Result> { + use schema::{jobs, payloads}; - jobs::table + let maybe_job_with_payload = jobs::table + .left_join(payloads::table) .filter(jobs::alias.eq(alias)) - .first(self.conn) + .first::<(JobModel, Option)>(self.conn) .optional() - .map_err(with_err_ctx(format!("Can't find job by alias {alias}"))) - } - - pub fn insert_agent(&mut self, agent: &Agent) -> Result<()> { - use schema::agents; + .map_err(with_err_ctx(format!("Can't get job by alias {alias}")))?; - diesel::insert_into(agents::table) - .values(agent) - .on_conflict(agents::id) - .do_update() - .set(agent) - .execute(self.conn) - .map_err(with_err_ctx(format!("Can't insert agent {agent:?}")))?; - Ok(()) + Ok(maybe_job_with_payload.map(|(job, payload_meta)| ThinJob { job, payload_meta })) } pub fn insert_result(&mut self, result: &AssignedJob) -> Result<()> { @@ -163,7 +173,7 @@ impl UDB<'_> { Ok(result) } - pub fn set_jobs_for_agent(&mut self, agent_id: Id, job_ids: &[Id]) -> Result> { + pub fn set_jobs_for_agent(&mut self, agent_id: Id, job_ids: &[Id]) -> Result<()> { use schema::{jobs, results}; let agent_platform = match self.get_agent(agent_id)? { @@ -203,64 +213,51 @@ impl UDB<'_> { diesel::insert_into(results::table) .values(&job_requests) .execute(self.conn) + .map(drop) .map_err(with_err_ctx(format!( "Can't setup jobs {job_ids:?} for agent {agent_id:?}" - )))?; - - Ok(job_requests.iter().map(|aj| aj.id).collect()) + ))) } - pub fn del_jobs(&mut self, ids: &[Id]) -> Result { + pub fn del_jobs(&mut self, ids: &[Id]) -> Result<()> { use schema::jobs; - let mut affected = 0; - for id in ids { - let deleted = diesel::delete(jobs::table) - .filter(jobs::id.eq(id)) - .execute(self.conn) - .map_err(with_err_ctx("Can't delete jobs"))?; - affected += deleted; - } - Ok(affected) + diesel::delete(jobs::table) + .filter(jobs::id.eq_any(ids)) + .execute(self.conn) + .map(drop) + .map_err(with_err_ctx("Can't delete jobs")) } - pub fn del_results(&mut self, ids: &[Id]) -> Result { + pub fn del_results(&mut self, ids: &[Id]) -> Result<()> { use schema::results; - let mut affected = 0; - for id in ids { - let deleted = diesel::delete(results::table) - .filter(results::id.eq(id)) - .execute(self.conn) - .map_err(with_err_ctx("Can't delete results"))?; - affected += deleted; - } - Ok(affected) + diesel::delete(results::table) + .filter(results::id.eq_any(ids)) + .execute(self.conn) + .map(drop) + .map_err(with_err_ctx("Can't delete results")) } - pub fn del_agents(&mut self, ids: &[Id]) -> Result { + pub fn del_agents(&mut self, ids: &[Id]) -> Result<()> { use schema::agents; - let mut affected = 0; - for id in ids { - let deleted = diesel::delete(agents::table) - .filter(agents::id.eq(id)) - .execute(self.conn) - .map_err(with_err_ctx("Can't delete agents"))?; - affected += deleted; - } - Ok(affected) + diesel::delete(agents::table) + .filter(agents::id.eq_any(ids)) + .execute(self.conn) + .map(drop) + .map_err(with_err_ctx("Can't delete agents")) } - pub fn update_agent(&mut self, agent: &Agent) -> Result<()> { + pub fn upsert_agent(&mut self, agent: &Agent) -> Result<()> { agent .save_changes::(self.conn) - .map_err(with_err_ctx(format!("Can't update agent {agent:?}")))?; + .map_err(with_err_ctx(format!("Can't upsert agent {agent:?}")))?; Ok(()) } - pub fn update_job(&mut self, job: &ThinJobMeta) -> Result<()> { - job.save_changes::(self.conn) + pub fn update_job(&mut self, job: &JobModel) -> Result<()> { + job.save_changes::(self.conn) .map_err(with_err_ctx(format!("Can't update job {job:?}")))?; Ok(()) } diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index c1f037d..be2a260 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -2,19 +2,25 @@ use std::sync::Arc; use crate::db::{PgRepo, UDB}; use crate::error::Error; -use crate::ValidJobMeta; +use serde::Deserialize; +use u_lib::jobs::join_payload; use u_lib::{ - jobs::{fat_meta_to_thin, thin_meta_to_fat}, + jobs::split_payload, messaging::{AsMsg, Reportable}, misc::OneOrVec, models::*, - types::Id + types::Id, }; use warp::reject::not_found; use warp::Rejection; type EndpResult = Result; +#[derive(Deserialize)] +pub struct GetJobQuery { + force_payload: bool, +} + pub struct Endpoints; impl Endpoints { @@ -35,25 +41,32 @@ impl Endpoints { .map_err(From::from) } - pub async fn get_job(repo: Arc, id: Id) -> EndpResult { + pub async fn get_job(repo: Arc, id: Id, params: GetJobQuery) -> EndpResult { let Some(job) = repo.interact(move |mut db| db.get_job(id)).await? else { return Err(not_found()) }; - let fat_meta = thin_meta_to_fat(job).map_err(Error::from)?; - Ok(fat_meta) + if let Some(meta) = &job.payload_meta { + let max_readable_payload_size = 1024 * 8; + if !params.force_payload && meta.size > max_readable_payload_size { + return Ok(FatJob { + job: job.job, + payload_meta: job.payload_meta, + payload_data: None, + }); + } + } + + Ok(join_payload(job).map_err(Error::from)?) } - pub async fn get_jobs(repo: Arc) -> EndpResult> { + pub async fn get_jobs(repo: Arc) -> EndpResult> { repo.interact(move |mut db| db.get_jobs()) .await .map_err(From::from) } - pub async fn get_agent_jobs( - repo: Arc, - id: Option, - ) -> EndpResult> { + pub async fn get_agent_jobs(repo: Arc, id: Option) -> EndpResult> { repo.interact(move |mut db| db.get_exact_jobs(id, false)) .await .map_err(From::from) @@ -65,18 +78,18 @@ impl Endpoints { match agent { Some(mut agent) => { agent.touch(); - db.update_agent(&agent)?; + db.upsert_agent(&agent)?; } None => { let new_agent = Agent::with_id(id); - db.insert_agent(&new_agent)?; + db.upsert_agent(&new_agent)?; let job = db .find_job_by_alias("agent_hello")? .expect("agent_hello job not found"); - db.set_jobs_for_agent(id, &[job.id])?; + db.set_jobs_for_agent(id, &[job.job.id])?; } } @@ -92,30 +105,23 @@ impl Endpoints { .map_err(From::from) } - pub async fn upload_jobs( - repo: Arc, - msg: Vec, - ) -> EndpResult> { + pub async fn upload_jobs(repo: Arc, msg: Vec) -> EndpResult<()> { let jobs = msg .into_iter() - .map(|meta| Ok(fat_meta_to_thin(meta)?)) - .collect::, Error>>()?; + .map(|meta| Ok(split_payload(meta)?)) + .collect::, Error>>()?; repo.interact(move |mut db| db.insert_jobs(&jobs)) .await .map_err(From::from) } - pub async fn del(repo: Arc, id: Id) -> EndpResult { + pub async fn del(repo: Arc, id: Id) -> EndpResult<()> { repo.transaction(move |mut db| { - let del_fns = &[UDB::del_agents, UDB::del_jobs, UDB::del_results]; - for del_fn in del_fns { - let affected = del_fn(&mut db, &[id])?; - if affected > 0 { - return Ok(affected); - } - } - Ok(0) + [UDB::del_agents, UDB::del_jobs, UDB::del_results] + .iter() + .map(|f| f(&mut db, &[id])) + .collect::>() }) .await .map_err(From::from) @@ -125,7 +131,7 @@ impl Endpoints { repo: Arc, agent_id: Id, job_idents: Vec, - ) -> EndpResult> { + ) -> EndpResult<()> { repo.transaction(move |mut db| { job_idents .into_iter() @@ -134,7 +140,7 @@ impl Endpoints { let job_from_db = db.find_job_by_alias(&ident); match job_from_db { Ok(job) => match job { - Some(j) => Ok(j.id), + Some(j) => Ok(j.job.id), None => { Err(Error::ProcessingError(format!("unknown ident {ident}"))) } @@ -153,7 +159,7 @@ impl Endpoints { pub async fn report + AsMsg + Send + Sync + 'static>( repo: Arc, msg: Data, - agent_id: Id + agent_id: Id, ) -> EndpResult<()> { repo.transaction(move |mut db| { for entry in msg.into_vec() { @@ -165,13 +171,13 @@ impl Endpoints { continue; } result.touch(); - + info!("agent {agent_id} updated job {}", result.id); match result.exec_type { JobType::Init => { result.state = JobState::Finished; - + match &result.result { Some(rbytes) => { let mut agent: Agent = match serde_json::from_slice(&rbytes) { @@ -182,7 +188,7 @@ impl Endpoints { } }; agent.state = AgentState::Active; - db.insert_agent(&agent)?; + db.upsert_agent(&agent)?; } None => error!("Empty agent data"), }}, @@ -206,29 +212,17 @@ impl Endpoints { .map_err(From::from) } - pub async fn update_agent( - repo: Arc, - agent: Agent, - ) -> EndpResult<()> { - repo.interact(move |mut db| db.update_agent(&agent)) - .await?; + pub async fn update_agent(repo: Arc, agent: Agent) -> EndpResult<()> { + repo.interact(move |mut db| db.upsert_agent(&agent)).await?; Ok(()) } - pub async fn update_job( - repo: Arc, - job: ValidJobMeta, - ) -> EndpResult<()> { - let thin_meta = fat_meta_to_thin(job).map_err(Error::from)?; - repo.interact(move |mut db| db.update_job(&thin_meta)) - .await?; + pub async fn update_job(repo: Arc, job: JobModel) -> EndpResult<()> { + repo.interact(move |mut db| db.update_job(&job)).await?; Ok(()) } - pub async fn update_assigned_job( - repo: Arc, - assigned: AssignedJob, - ) -> EndpResult<()> { + pub async fn update_assigned_job(repo: Arc, assigned: AssignedJob) -> EndpResult<()> { repo.interact(move |mut db| db.update_result(&assigned)) .await?; Ok(()) diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 34b37d4..40b2656 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -15,7 +15,6 @@ use std::{convert::Infallible, sync::Arc}; use u_lib::{ config, db::async_pool, - jobs::fat_meta_to_thin, messaging::{AsMsg, Reportable}, models::*, types::Id, @@ -27,9 +26,7 @@ use warp::{ Filter, Rejection, Reply, }; -use crate::handlers::Endpoints; - -type ValidJobMeta = FatJobMeta; +use crate::handlers::{Endpoints, GetJobQuery}; fn into_message(msg: M) -> Json { json(&msg) @@ -55,13 +52,14 @@ pub fn init_endpoints( let upload_jobs = path("upload_jobs") .and(with_db.clone()) - .and(body::json::>()) + .and(body::json::>()) .and_then(Endpoints::upload_jobs) .map(into_message); let get_job = path("get_job") .and(with_db.clone()) .and(warp::path::param::()) + .and(warp::query::()) .and_then(Endpoints::get_job) .map(into_message); @@ -110,7 +108,7 @@ pub fn init_endpoints( let update_job = path("update_job") .and(with_db.clone()) - .and(body::json::()) + .and(body::json::()) .and_then(Endpoints::update_job) .map(ok); @@ -156,13 +154,11 @@ pub async fn preload_jobs(repo: &PgRepo) -> Result<(), ServerError> { let job_alias = "agent_hello"; let if_job_exists = db.find_job_by_alias(job_alias)?; if if_job_exists.is_none() { - let agent_hello = fat_meta_to_thin( - FatJobMeta::builder() - .with_type(JobType::Init) - .with_alias(job_alias) - .build() - .unwrap(), - )?; + let agent_hello = RawJob::builder() + .with_type(JobType::Init) + .with_alias(job_alias) + .build() + .unwrap(); db.insert_jobs(&[agent_hello])?; } Ok(()) diff --git a/integration/Cargo.toml b/integration/Cargo.toml index 706d97d..16ea59a 100644 --- a/integration/Cargo.toml +++ b/integration/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +futures = { version = "0.3", features = ["executor"] } once_cell = "1.10.0" reqwest = { workspace = true } rstest = "0.12" diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs index af023f3..7e3f3df 100644 --- a/integration/tests/fixtures/agent.rs +++ b/integration/tests/fixtures/agent.rs @@ -1,36 +1,35 @@ -use crate::helpers::ENV; +use super::connections::*; +use super::run_async; use u_lib::{ - api::ClientHandler, config::get_self_id, jobs::fat_meta_to_thin, messaging::Reportable, - models::*, types::Id, + api::ClientHandler, config::get_self_id, jobs::split_payload, messaging::Reportable, models::*, + types::Id, }; pub struct RegisteredAgent { pub id: Id, } -impl RegisteredAgent { - pub async fn unregister(self) { - let cli = ClientHandler::new(&ENV.u_server, None).await.unwrap(); - cli.del(self.id).await.unwrap(); - } -} - #[fixture] -pub async fn register_agent() -> RegisteredAgent { - let cli = ClientHandler::new(&ENV.u_server, None).await.unwrap(); - let agent_id = get_self_id(); - println!("registering agent {agent_id}"); - let resp = cli - .get_personal_jobs(agent_id) - .await - .unwrap() - .pop() - .unwrap(); - let job_id = resp.job_id; - let job = cli.get_job(job_id).await.unwrap(); - assert_eq!(job.alias, Some("agent_hello".to_string())); - let mut agent_data = AssignedJob::from((&fat_meta_to_thin(job).unwrap(), resp)); - agent_data.set_result(&Agent::with_id(agent_id)); - cli.report(Reportable::Assigned(agent_data)).await.unwrap(); - RegisteredAgent { id: agent_id } +#[once] +pub fn registered_agent(client: &ClientHandler) -> RegisteredAgent { + run_async(async { + let agent_id = get_self_id(); + println!("registering agent {agent_id}"); + let resp = client + .get_personal_jobs(agent_id) + .await + .unwrap() + .pop() + .unwrap(); + let job_id = resp.job_id; + let job = client.get_job(job_id).await.unwrap(); + assert_eq!(job.job.alias, Some("agent_hello".to_string())); + let mut agent_data = AssignedJob::from((&split_payload(job).unwrap().job, resp)); + agent_data.set_result(&Agent::with_id(agent_id)); + client + .report(Reportable::Assigned(agent_data)) + .await + .unwrap(); + RegisteredAgent { id: agent_id } + }) } diff --git a/integration/tests/fixtures/connections.rs b/integration/tests/fixtures/connections.rs new file mode 100644 index 0000000..d4c6986 --- /dev/null +++ b/integration/tests/fixtures/connections.rs @@ -0,0 +1,28 @@ +use super::env::*; +use super::run_async; + +pub use u_lib::api::ClientHandler; +use u_lib::db::unpooled; +pub use u_lib::db::PgConnection; + +#[fixture] +#[once] +pub fn client(env_default: EndpointsEnv) -> ClientHandler { + run_async(ClientHandler::new(&env_default.u_server, None)).unwrap() +} + +#[fixture] +#[once] +pub fn client_panel(env_access: AccessEnv) -> ClientHandler { + run_async(ClientHandler::new( + &env_access.u_server, + Some(env_access.admin_auth_token), + )) + .unwrap() +} + +#[fixture] +#[once] +pub fn db(env_db: DBEnv) -> PgConnection { + unpooled(&env_db) +} diff --git a/integration/tests/fixtures/env.rs b/integration/tests/fixtures/env.rs new file mode 100644 index 0000000..4738400 --- /dev/null +++ b/integration/tests/fixtures/env.rs @@ -0,0 +1,16 @@ +pub use u_lib::config::{AccessEnv, DBEnv, EndpointsEnv}; + +#[fixture] +pub fn env_default() -> EndpointsEnv { + EndpointsEnv::load() +} + +#[fixture] +pub fn env_access() -> AccessEnv { + AccessEnv::load().unwrap() +} + +#[fixture] +pub fn env_db() -> DBEnv { + DBEnv::load().unwrap() +} diff --git a/integration/tests/fixtures/mod.rs b/integration/tests/fixtures/mod.rs index f17bc55..c2cdc85 100644 --- a/integration/tests/fixtures/mod.rs +++ b/integration/tests/fixtures/mod.rs @@ -1 +1,9 @@ pub mod agent; +pub mod connections; +pub mod env; + +use std::future::Future; + +fn run_async(fut: impl Future) -> R { + futures::executor::block_on(fut) +} diff --git a/integration/tests/helpers/mod.rs b/integration/tests/helpers/mod.rs index 3ac8a05..19ca72b 100644 --- a/integration/tests/helpers/mod.rs +++ b/integration/tests/helpers/mod.rs @@ -2,8 +2,3 @@ pub mod jobs; pub mod panel; pub use panel::Panel; - -use once_cell::sync::Lazy; -use u_lib::config::EndpointsEnv; - -pub static ENV: Lazy = Lazy::new(|| EndpointsEnv::load()); diff --git a/integration/tests/integration/api.rs b/integration/tests/integration/api.rs new file mode 100644 index 0000000..12af506 --- /dev/null +++ b/integration/tests/integration/api.rs @@ -0,0 +1 @@ +//async fn diff --git a/integration/tests/integration/behaviour.rs b/integration/tests/integration/behaviour.rs index 8faa9b3..a5f4620 100644 --- a/integration/tests/integration/behaviour.rs +++ b/integration/tests/integration/behaviour.rs @@ -9,12 +9,11 @@ use uuid::Uuid; #[rstest] #[tokio::test] -async fn registration(#[future] register_agent: RegisteredAgent) { - let agent = register_agent.await; +async fn registration(registered_agent: &RegisteredAgent) { let agents: Vec = Panel::check_output("agents read"); - let found = agents.iter().find(|v| v.id == agent.id); + let found = agents.iter().find(|v| v.id == registered_agent.id); assert!(found.is_some()); - Panel::check_status(format!("agents delete {}", agent.id)); + Panel::check_status(format!("agents delete {}", registered_agent.id)); } #[tokio::test] @@ -52,14 +51,14 @@ async fn large_payload() { let job_alias = "large_payload"; - let job = FatJobMeta::builder() + let job = RawJob::builder() .with_alias(job_alias) .with_payload_path("./tests/bin/echoer") .with_shell("{} type echo") .build() .unwrap(); - Panel::check_status(["jobs", "create", &to_string(&job).unwrap()]); + Panel::check_status(["jobs", "create", &to_string(&RawJob::from(job)).unwrap()]); let cmd = format!("map create {agent_id} {job_alias}"); let assigned_ids: Vec = Panel::check_output(cmd); diff --git a/integration/tests/integration/connection.rs b/integration/tests/integration/connection.rs index 156a6a1..2822f86 100644 --- a/integration/tests/integration/connection.rs +++ b/integration/tests/integration/connection.rs @@ -1,14 +1,15 @@ -use crate::helpers::ENV; +use crate::fixtures::env::*; use u_lib::config::MASTER_PORT; +#[rstest] #[tokio::test] -async fn non_auth_connection_dropped() { +async fn non_auth_connection_dropped(env_default: EndpointsEnv) { let client = reqwest::ClientBuilder::new() .danger_accept_invalid_certs(true) .build() .unwrap(); match client - .get(format!("https://{}:{}", &ENV.u_server, MASTER_PORT)) + .get(format!("https://{}:{}", &env_default.u_server, MASTER_PORT)) .send() .await { diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 9d306a9..c1d18d4 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -75,11 +75,11 @@ impl ClientHandler { }) } - async fn req(&self, url: impl AsRef) -> Result { + async fn req(&self, url: impl AsRef) -> Result { self.req_with_payload(url, ()).await } - async fn req_with_payload( + async fn req_with_payload( &self, url: impl AsRef, payload: P, @@ -93,7 +93,6 @@ impl ClientHandler { .send() .await .context("error while sending request")?; - let content_len = response.content_length(); let is_success = match response.error_for_status_ref() { Ok(_) => Ok(()), Err(e) => Err(UError::from(e)), @@ -101,10 +100,7 @@ impl ClientHandler { let resp = response.text().await.context("resp")?; let result = match is_success { - Ok(_) => from_str::(&resp).or_else(|e| match content_len { - Some(0) => Ok(Default::default()), - _ => Err(UError::NetError(e.to_string(), resp)), - }), + Ok(_) => from_str::(&resp).map_err(|e| UError::NetError(e.to_string(), resp)), Err(UError::NetError(err, _)) => Err(UError::NetError(err, resp)), _ => unreachable!(), } @@ -131,12 +127,12 @@ impl ClientHandler { } /// get exact job - pub async fn get_job(&self, job: Id) -> Result> { + pub async fn get_job(&self, job: Id) -> Result { self.req(format!("get_job/{job}")).await } /// get all available jobs - pub async fn get_jobs(&self) -> Result> { + pub async fn get_jobs(&self) -> Result> { self.req("get_jobs").await } } @@ -156,7 +152,7 @@ impl ClientHandler { } /// update job - pub async fn update_job(&self, job: FatJobMeta) -> Result<()> { + pub async fn update_job(&self, job: FatJob) -> Result<()> { self.req_with_payload("update_job", job).await } @@ -166,7 +162,7 @@ impl ClientHandler { } /// create and upload job - pub async fn upload_jobs(&self, payload: impl OneOrVec>) -> Result> { + pub async fn upload_jobs(&self, payload: impl OneOrVec) -> Result> { self.req_with_payload("upload_jobs", payload.into_vec()) .await } diff --git a/lib/u_lib/src/cache.rs b/lib/u_lib/src/cache.rs index 17ccfaa..c025db4 100644 --- a/lib/u_lib/src/cache.rs +++ b/lib/u_lib/src/cache.rs @@ -1,10 +1,11 @@ -use crate::models::ThinJobMeta; +use crate::models::ThinJob; use crate::types::Id; use lazy_static::lazy_static; use parking_lot::{RwLock, RwLockReadGuard}; use std::{collections::HashMap, ops::Deref}; -type Cache = HashMap; +type Val = ThinJob; +type Cache = HashMap; lazy_static! { static ref JOB_CACHE: RwLock = RwLock::new(HashMap::new()); @@ -13,8 +14,8 @@ lazy_static! { pub struct JobCache; impl JobCache { - pub fn insert(job_meta: ThinJobMeta) { - JOB_CACHE.write().insert(job_meta.id, job_meta); + pub fn insert(job: Val) { + JOB_CACHE.write().insert(job.job.id, job); } pub fn contains(id: Id) -> bool { @@ -37,7 +38,7 @@ impl JobCache { pub struct JobCacheHolder<'jh>(pub RwLockReadGuard<'jh, Cache>, pub Id); impl<'jh> Deref for JobCacheHolder<'jh> { - type Target = ThinJobMeta; + type Target = Val; fn deref(&self) -> &Self::Target { self.0.get(&self.1).unwrap() diff --git a/lib/u_lib/src/db.rs b/lib/u_lib/src/db.rs index 3e874d2..0c5d8ef 100644 --- a/lib/u_lib/src/db.rs +++ b/lib/u_lib/src/db.rs @@ -1,5 +1,6 @@ use deadpool_diesel::{Manager as DManager, Pool as DPool, Runtime}; -use diesel::pg::PgConnection; +pub use diesel::pg::PgConnection; +use diesel::Connection; use std::time::Duration; use crate::config::DBEnv; @@ -24,3 +25,9 @@ pub fn async_pool(config: &DBEnv) -> PgAsyncPool { .build() .unwrap() } + +pub fn unpooled(config: &DBEnv) -> PgConnection { + let db_url = generate_postgres_url(config); + + PgConnection::establish(&db_url).unwrap() +} diff --git a/lib/u_lib/src/jobs.rs b/lib/u_lib/src/jobs.rs index ea0efd3..c81bd3d 100644 --- a/lib/u_lib/src/jobs.rs +++ b/lib/u_lib/src/jobs.rs @@ -2,7 +2,7 @@ use crate::{ combined_result::CombinedResult, executor::{ExecResult, Waiter}, misc::OneOrVec, - models::{Agent, AssignedJob, AssignedJobById, FatJobMeta, JobType, Payload, ThinJobMeta}, + models::{Agent, AssignedJob, AssignedJobById, FatJob, JobType, RawJob, ThinJob}, proc_output::ProcOutput, ufs, }; @@ -16,11 +16,10 @@ pub struct AnonymousJobBatch { } impl AnonymousJobBatch { - pub fn from_meta_with_id(jobs: impl OneOrVec<(ThinJobMeta, AssignedJobById)>) -> Self { - let jobs = jobs.into_vec(); + pub fn from_meta_with_id(jobs: impl OneOrVec<(ThinJob, AssignedJobById)>) -> Self { let mut waiter = Waiter::new(); - for (meta, job) in jobs { - waiter.push(run_assigned_job(meta, job)); + for (job, ids) in jobs.into_vec() { + waiter.push(run_assigned_job(job, ids)); } Self { waiter, @@ -28,14 +27,14 @@ impl AnonymousJobBatch { } } - pub fn from_meta(metas: impl OneOrVec) -> Self { - let jobs: Vec<_> = metas + pub fn from_meta(jobs: impl OneOrVec) -> Self { + let jobs_ids: Vec<_> = jobs .into_vec() .into_iter() - .map(|meta| { - let job_id = meta.id; + .map(|job| { + let job_id = job.job.id; ( - meta, + job, AssignedJobById { job_id, ..Default::default() @@ -43,7 +42,7 @@ impl AnonymousJobBatch { ) }) .collect(); - AnonymousJobBatch::from_meta_with_id(jobs) + AnonymousJobBatch::from_meta_with_id(jobs_ids) } /// Spawn jobs @@ -85,18 +84,8 @@ impl NamedJobBatch { .into_vec() .into_iter() .filter_map(|(alias, cmd)| { - match FatJobMeta::builder() - .with_shell(cmd) - .with_alias(alias) - .build() - { - Ok(fat_meta) => match fat_meta_to_thin(fat_meta) { - Ok(thin_meta) => Some(thin_meta), - Err(e) => { - result.err(e); - None - } - }, + match RawJob::builder().with_shell(cmd).with_alias(alias).build() { + Ok(jpm) => Some(jpm), Err(e) => { result.err(e); None @@ -108,14 +97,14 @@ impl NamedJobBatch { result } - pub fn from_meta(named_jobs: impl OneOrVec) -> Self { - let (job_names, job_metas): (Vec<_>, Vec<_>) = named_jobs + pub fn from_meta(named_jobs: impl OneOrVec) -> Self { + let (job_names, jobs): (Vec<_>, Vec<_>) = named_jobs .into_vec() .into_iter() - .map(|meta| (meta.alias.clone().unwrap(), meta)) + .map(|job| (job.job.alias.clone().unwrap(), job)) .unzip(); Self { - runner: Some(AnonymousJobBatch::from_meta(job_metas)), + runner: Some(AnonymousJobBatch::from_meta(jobs)), job_names, results: HashMap::new(), } @@ -145,17 +134,18 @@ impl NamedJobBatch { } } -pub async fn run_assigned_job(meta: ThinJobMeta, ids: AssignedJobById) -> ExecResult { - let mut job = AssignedJob::from((&meta, ids)); - match meta.exec_type { +pub async fn run_assigned_job(job: ThinJob, ids: AssignedJobById) -> ExecResult { + let ThinJob { job, payload_meta } = job; + let mut result = AssignedJob::from((&job, ids)); + match job.exec_type { JobType::Shell => { let (argv, _prepared_payload) = { - if let Some(ref payload) = meta.payload { - let (prep_exec, prep_exec_path) = ufs::prepare_executable(payload)?; - let argv_with_exec = meta.argv.replace("{}", &prep_exec_path); + if let Some(meta) = payload_meta { + let (prep_exec, prep_exec_path) = ufs::prepare_executable(meta.name)?; + let argv_with_exec = job.argv.replace("{}", &prep_exec_path); (argv_with_exec, Some(prep_exec)) } else { - (meta.argv, None) + (job.argv, None) } }; @@ -175,69 +165,55 @@ pub async fn run_assigned_job(meta: ThinJobMeta, ids: AssignedJobById) -> ExecRe None, ), }; - job.result = Some(data); - job.retcode = retcode; + result.result = Some(data); + result.retcode = retcode; } JobType::Init => { - job.set_result(&Agent::run().await); - job.retcode = Some(0); + result.set_result(&Agent::run().await); + result.retcode = Some(0); } JobType::Service => todo!(), JobType::Update => todo!(), JobType::Terminate => exit(0), }; - Ok(job) + Ok(result) } -pub fn fat_meta_to_thin(meta: FatJobMeta) -> Result { - let payload_ident = if let Some(mut payload) = meta.payload { - let job_name = match &meta.alias { - Some(a) => a.to_string(), - None => meta.id.simple().to_string(), - }; - payload.write_self_into(&job_name)?; - Some(job_name) - } else { - None - }; +pub fn split_payload(job: FatJob) -> Result { + let FatJob { + job, + payload_meta, + payload_data, + } = job; - Ok(ThinJobMeta { - alias: meta.alias, - argv: meta.argv, - id: meta.id, - exec_type: meta.exec_type, - platform: meta.platform, - payload: payload_ident, - schedule: meta.schedule, - }) -} + if let Some(meta) = &payload_meta { + if !ufs::in_index(&meta.name) { + ufs::put(&meta.name, payload_data.unwrap())?; + } + } -pub fn thin_meta_to_fat(meta: ThinJobMeta) -> Result, ufs::Error> { - let payload = if let Some(payload) = meta.payload { - let mut fat_payload = Payload::Ident(payload); - fat_payload.read_into_self()?; - Some(fat_payload) - } else { - None - }; + Ok(ThinJob { job, payload_meta }) +} - Ok(FatJobMeta { - alias: meta.alias, - argv: meta.argv, - id: meta.id, - exec_type: meta.exec_type, - platform: meta.platform, - payload, - schedule: meta.schedule, +pub fn join_payload(job: ThinJob) -> Result { + let ThinJob { job, payload_meta } = job; + let payload_data = payload_meta + .as_ref() + .map(|p| ufs::read(&p.name)) + .transpose()?; + + Ok(FatJob { + job, + payload_meta, + payload_data, }) } #[cfg(test)] mod tests { - use super::*; use crate::{ jobs::{AnonymousJobBatch, NamedJobBatch}, - models::{misc::JobType, FatJobMeta}, + models::{misc::JobType, RawJob}, unwrap_enum, UError, }; use std::time::SystemTime; @@ -247,10 +223,8 @@ mod tests { #[tokio::test] async fn test_is_really_async() { const SLEEP_SECS: u64 = 1; - let job = FatJobMeta::from_shell(format!("sleep {SLEEP_SECS}")).unwrap(); - let sleep_jobs = (0..50) - .map(|_| fat_meta_to_thin(job.clone()).unwrap()) - .collect::>(); + let job = RawJob::from_shell(format!("sleep {SLEEP_SECS}")).unwrap(); + let sleep_jobs = (0..50).map(|_| job.clone()).collect::>(); let now = SystemTime::now(); AnonymousJobBatch::from_meta(sleep_jobs).wait().await; @@ -287,11 +261,11 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] expected_result: &str, ) -> TestResult { - let mut job = FatJobMeta::builder().with_shell(cmd); + let mut job = RawJob::builder().with_shell(cmd); if let Some(p) = payload { job = job.with_payload(p); } - let job = fat_meta_to_thin(job.build().unwrap()).unwrap(); + let job = job.build().unwrap(); let result = AnonymousJobBatch::from_meta(job).wait_one().await.unwrap(); let result = result.to_str_result(); assert_eq!(result.trim(), expected_result); @@ -302,23 +276,19 @@ mod tests { async fn test_complex_load() -> TestResult { const SLEEP_SECS: u64 = 1; let now = SystemTime::now(); - let longest_job = FatJobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); - let longest_job = AnonymousJobBatch::from_meta(fat_meta_to_thin(longest_job).unwrap()) - .spawn() - .await; - let ls = AnonymousJobBatch::from_meta( - fat_meta_to_thin(FatJobMeta::from_shell("ls").unwrap()).unwrap(), - ) - .wait_one() - .await - .unwrap(); + let longest_job = RawJob::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); + let longest_job = AnonymousJobBatch::from_meta(longest_job).spawn().await; + let ls = AnonymousJobBatch::from_meta(RawJob::from_shell("ls").unwrap()) + .wait_one() + .await + .unwrap(); assert_eq!(ls.retcode.unwrap(), 0); let folders = ls.to_str_result(); let subfolders_jobs = folders .lines() - .map(|f| fat_meta_to_thin(FatJobMeta::from_shell(format!("ls {f}")).unwrap()).unwrap()) + .map(|f| RawJob::from_shell(format!("ls {f}")).unwrap()) .collect::>(); let ls_subfolders = AnonymousJobBatch::from_meta(subfolders_jobs).wait().await; @@ -351,7 +321,7 @@ mod tests { */ #[tokio::test] async fn test_failing_shell_job() -> TestResult { - let job = fat_meta_to_thin(FatJobMeta::from_shell("lol_kek_puk").unwrap()).unwrap(); + let job = RawJob::from_shell("lol_kek_puk").unwrap(); let job_result = AnonymousJobBatch::from_meta(job).wait_one().await.unwrap(); let output = job_result.to_str_result(); assert!(output.contains("No such file")); @@ -368,7 +338,7 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] err_str: &str, ) -> TestResult { - let mut job = FatJobMeta::builder().with_shell(cmd); + let mut job = RawJob::builder().with_shell(cmd); if let Some(p) = payload { job = job.with_payload(p); } @@ -380,23 +350,18 @@ mod tests { #[tokio::test] async fn test_different_job_types() -> TestResult { - let mut jobs = NamedJobBatch::from_meta( - [ - FatJobMeta::builder() - .with_shell("sleep 3") - .with_alias("sleeper") - .build() - .unwrap(), - FatJobMeta::builder() - .with_type(JobType::Init) - .with_alias("gatherer") - .build() - .unwrap(), - ] - .into_iter() - .map(|meta| fat_meta_to_thin(meta).unwrap()) - .collect::>(), - ) + let mut jobs = NamedJobBatch::from_meta(vec![ + RawJob::builder() + .with_shell("sleep 3") + .with_alias("sleeper") + .build() + .unwrap(), + RawJob::builder() + .with_type(JobType::Init) + .with_alias("gatherer") + .build() + .unwrap(), + ]) .wait() .await; let gathered = jobs.pop("gatherer").unwrap(); diff --git a/lib/u_lib/src/messaging/mod.rs b/lib/u_lib/src/messaging.rs similarity index 80% rename from lib/u_lib/src/messaging/mod.rs rename to lib/u_lib/src/messaging.rs index 1dc699f..88089e2 100644 --- a/lib/u_lib/src/messaging/mod.rs +++ b/lib/u_lib/src/messaging.rs @@ -1,22 +1,20 @@ -mod files; - use crate::models::*; use crate::types::Id; use crate::UError; -pub use files::*; use serde::{Deserialize, Serialize}; use std::fmt::Debug; +/// Represents types that could be used in client-server interaction pub trait AsMsg: Clone + Serialize + Debug {} impl AsMsg for Agent {} impl AsMsg for AssignedJob {} impl AsMsg for AssignedJobById {} -impl AsMsg for DownloadInfo {} -impl AsMsg for FatJobMeta {} +impl AsMsg for JobModel {} +impl AsMsg for FatJob {} impl AsMsg for Reportable {} impl AsMsg for String {} -impl AsMsg for ThinJobMeta {} +impl AsMsg for ThinJob {} impl AsMsg for Id {} impl AsMsg for i32 {} impl AsMsg for u8 {} diff --git a/lib/u_lib/src/messaging/files.rs b/lib/u_lib/src/messaging/files.rs deleted file mode 100644 index 4cdb804..0000000 --- a/lib/u_lib/src/messaging/files.rs +++ /dev/null @@ -1,8 +0,0 @@ -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -#[derive(Serialize, Deserialize, Clone, Debug, Default)] -pub struct DownloadInfo { - hashsum: String, - dl_fid: Uuid, -} diff --git a/lib/u_lib/src/misc.rs b/lib/u_lib/src/misc.rs index bbc6484..7d185f1 100644 --- a/lib/u_lib/src/misc.rs +++ b/lib/u_lib/src/misc.rs @@ -20,7 +20,7 @@ macro_rules! unwrap_enum { if let $t(result) = $src { result } else { - panic!("wrong type {}", stringify!($t)) + panic!("wrong type '{}'", stringify!($t)) } }; } diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index 0e302ef..041451c 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -1,4 +1,4 @@ -use super::{JobState, JobType, ThinJobMeta}; +use super::{JobModel, JobState, JobType}; #[cfg(feature = "server")] use crate::models::schema::*; use crate::{ @@ -60,8 +60,8 @@ pub struct AssignedJobById { pub job_id: Id, } -impl From<(&ThinJobMeta, AssignedJobById)> for AssignedJob { - fn from((meta, ids): (&ThinJobMeta, AssignedJobById)) -> Self { +impl From<(&JobModel, AssignedJobById)> for AssignedJob { + fn from((job, ids): (&JobModel, AssignedJobById)) -> Self { let AssignedJobById { agent_id, id, @@ -72,8 +72,8 @@ impl From<(&ThinJobMeta, AssignedJobById)> for AssignedJob { id, agent_id, job_id, - alias: meta.alias.clone(), - exec_type: meta.exec_type, + alias: job.alias.clone(), + exec_type: job.exec_type, ..Default::default() } } diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 2feca3d..5f525f0 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -1,23 +1,26 @@ use std::fmt; use super::JobType; -use crate::models::payload::Payload; +use crate::conv::bytes_to_string; #[cfg(feature = "server")] use crate::models::schema::*; +use crate::models::PayloadMeta; use crate::platform::Platform; use crate::types::Id; -use crate::{UError, UResult}; +use crate::{ufs, UError, UResult}; #[cfg(feature = "server")] use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; +use std::fs::metadata; +use std::process::Command; -#[derive(Serialize, Deserialize, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr( feature = "server", derive(Queryable, Identifiable, Insertable, AsChangeset), diesel(table_name = jobs) )] -pub struct ThinJobMeta { +pub struct JobModel { pub alias: Option, /// string like `bash -c {} -a 1 --arg2`, /// where {} is replaced by executable's tmp path @@ -26,27 +29,40 @@ pub struct ThinJobMeta { pub exec_type: JobType, /// target triple pub platform: String, - pub payload: Option, + pub payload: Option, /// cron-like string pub schedule: Option, } -impl fmt::Debug for ThinJobMeta { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ThinJobMeta") - .field("alias", &self.alias) - .field("argv", &self.argv) - .field("id", &self.id.to_string()) - .field("exec_type", &self.exec_type) - .field("platform", &self.platform) - .field("payload", &self.payload) - .field("schedule", &self.schedule) - .finish() - } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FatJob { + pub job: JobModel, + pub payload_meta: Option, + pub payload_data: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ThinJob { + pub job: JobModel, + pub payload_meta: Option, } -#[derive(Serialize, Deserialize, Clone)] -pub struct FatJobMeta { +// impl fmt::Debug for ThinJobMeta { +// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +// f.debug_struct("ThinJobMeta") +// .field("alias", &self.alias) +// .field("argv", &self.argv) +// .field("id", &self.id.to_string()) +// .field("exec_type", &self.exec_type) +// .field("platform", &self.platform) +// .field("payload", &self.payload) +// .field("schedule", &self.schedule) +// .finish() +// } +// } + +#[derive(Serialize, Deserialize, Clone, Default)] +pub struct RawJob { #[serde(default)] pub alias: Option, @@ -66,16 +82,16 @@ pub struct FatJobMeta { pub platform: String, #[serde(default)] - pub payload: Option, + pub payload: Option, /// cron-like string #[serde(default)] pub schedule: Option, } -impl fmt::Debug for FatJobMeta { +impl fmt::Debug for RawJob { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("FatJobMeta") + f.debug_struct("RawJob") .field("alias", &self.alias) .field("argv", &self.argv) .field("id", &self.id.to_string()) @@ -87,53 +103,75 @@ impl fmt::Debug for FatJobMeta { } } -impl FatJobMeta { - pub fn validated(self) -> UResult> { - JobMetaBuilder { inner: self }.build() +impl From for RawJob { + fn from(job: ThinJob) -> Self { + let ThinJob { job, payload_meta } = job; + RawJob { + alias: job.alias, + argv: job.argv, + id: job.id, + exec_type: job.exec_type, + platform: job.platform, + payload: payload_meta.map(|m| m.name), + schedule: job.schedule, + } } +} - pub fn from_shell(cmd: impl Into) -> UResult> { - Self::builder().with_shell(cmd).build() +impl RawJob { + pub fn validated(self) -> UResult { + JobBuilder { + inner: self, + raw_payload: None, + } + .build() } - pub fn builder() -> JobMetaBuilder { - JobMetaBuilder::default() + pub fn from_shell(cmd: impl Into) -> UResult { + Self::builder().with_shell(cmd).build() } -} -impl Default for FatJobMeta { - fn default() -> Self { - Self { - id: Id::new_v4(), - alias: None, - argv: String::new(), - exec_type: JobType::Shell, - platform: Platform::current_as_string(), - payload: None, - schedule: None, - } + pub fn builder<'p>() -> JobBuilder<'p> { + JobBuilder::default() } } +// impl Default for RawJob { +// fn default() -> Self { +// Self { +// id: Id::new_v4(), +// alias: None, +// argv: String::new(), +// exec_type: JobType::Shell, +// platform: Platform::current_as_string(), +// payload: None, +// schedule: None, +// } +// } +//} + #[derive(Default)] -pub struct JobMetaBuilder { - inner: FatJobMeta, +pub struct JobBuilder<'p> { + inner: RawJob, + raw_payload: Option<&'p [u8]>, } -impl JobMetaBuilder { +impl<'p> JobBuilder<'p> { pub fn with_shell(mut self, shell_cmd: impl Into) -> Self { self.inner.argv = shell_cmd.into(); self.inner.exec_type = JobType::Shell; self } - pub fn with_payload(mut self, payload: impl Into>) -> Self { - self.inner.payload = Some(Payload::from_payload(payload)); + pub fn with_payload(mut self, raw_payload: &'p [u8]) -> Self { + self.raw_payload = Some(raw_payload); + self.inner.payload = None; self } pub fn with_payload_path(mut self, path: impl Into) -> Self { - self.inner.payload = Some(Payload::Ident(path.into())); + self.inner.payload = Some(path.into()); + self.raw_payload = None; self } @@ -147,16 +185,42 @@ impl JobMetaBuilder { self } - pub fn build(self) -> UResult> { + pub fn build(self) -> UResult { let mut inner = self.inner; - let validated = |jmeta: FatJobMeta| FatJobMeta:: { - alias: jmeta.alias, - argv: jmeta.argv, - id: jmeta.id, - exec_type: jmeta.exec_type, - platform: jmeta.platform, - payload: jmeta.payload, - schedule: jmeta.schedule, + + let raw_into_job = |raw: RawJob| -> UResult { + let payload_id = raw.payload.as_ref().map(|_| Id::new_v4()); + + Ok(ThinJob { + job: JobModel { + alias: raw.alias, + argv: raw.argv, + id: raw.id, + exec_type: raw.exec_type, + platform: raw.platform, + payload: payload_id, + schedule: raw.schedule, + }, + payload_meta: raw + .payload + .map(|payload_path| { + Ok::<_, UError>(PayloadMeta { + id: payload_id.unwrap(), + mime_type: bytes_to_string( + &Command::new("file") + .arg("-b") + .arg("--mime-type") + .arg(&payload_path) + .output() + .map_err(|e| UError::JobBuildError(e.to_string()))? + .stdout, + ), + name: payload_path.clone(), + size: metadata(payload_path).unwrap().len() as i64, + }) + }) + .transpose()?, + }) }; match inner.exec_type { @@ -180,19 +244,28 @@ impl JobMetaBuilder { return Err(empty_err.into()); } - if let Some(payload) = &mut inner.payload { - payload.add_to_index()?; + if let Some(payload_path) = &inner.payload { + ufs::put_external(payload_path)?; + } + + if let Some(raw) = self.raw_payload { + match inner.payload { + Some(_) => { + return Err(UError::JobBuildError( + "Can't use both raw payload with payload path".to_string(), + )) + } + None => inner.payload = Some(ufs::create(raw)?), + } } match inner.payload.as_ref() { - Some(p) => { - if let Payload::Data(d) = p { - if !d.is_empty() && !inner.argv.contains("{}") { - return Err(UError::JobBuildError( - "Argv contains no executable placeholder".into(), - ) - .into()); - } + Some(_) => { + if !inner.argv.contains("{}") { + return Err(UError::JobBuildError( + "Argv contains no executable placeholder".into(), + ) + .into()); } } None => { @@ -213,9 +286,9 @@ impl JobMetaBuilder { ))); } - Ok(validated(inner)) + raw_into_job(inner) } - _ => Ok(validated(inner)), + _ => raw_into_job(inner), } } } diff --git a/lib/u_lib/src/models/payload.rs b/lib/u_lib/src/models/payload.rs index 4543ec3..46fcec8 100644 --- a/lib/u_lib/src/models/payload.rs +++ b/lib/u_lib/src/models/payload.rs @@ -1,76 +1,20 @@ -use crate::{conv::bytes_to_string_truncated, ufs}; +use crate::types::Id; use serde::{Deserialize, Serialize}; -use std::{fmt, path::PathBuf}; -#[derive(Clone, Deserialize, Serialize)] -#[serde(untagged)] -pub enum Payload { - /// Raw payload data - Data(Vec), - /// Payload identifier in ufs - Ident(String), -} - -impl Payload { - pub fn read_into_self(&mut self) -> Result<(), ufs::Error> { - match self { - Payload::Data(_) => Ok(()), - Payload::Ident(ident) => { - let data = ufs::read(ident)?; - - *self = Payload::Data(data); - Ok(()) - } - } - } - - pub fn write_self_into(&mut self, name: impl AsRef) -> Result<(), ufs::Error> { - match self { - Payload::Ident(_) => Ok(()), - Payload::Data(data) => { - ufs::put(&name, data)?; - - *self = Payload::Ident(name.as_ref().to_string()); - Ok(()) - } - } - } - - pub fn from_payload(data: impl Into>) -> Self { - Payload::Data(data.into()) - } - - pub fn from_path(path: impl Into) -> Result { - let path: PathBuf = path.into(); - - if !path.exists() || path.is_dir() { - return Err(ufs::Error::not_found(path)); - } - - ufs::put_existing(&path)?; - - Ok(Payload::Ident(path.to_string_lossy().to_string())) - } - - pub fn add_to_index(&self) -> Result<(), ufs::Error> { - match self { - Payload::Ident(ident) => Payload::from_path(ident).map(|_| ()), - _ => Ok(()), - } - } -} - -impl fmt::Debug for Payload { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Data(data) => { - let mut dbg = &mut f.debug_tuple("Data"); - let data = bytes_to_string_truncated(data, 256); - - dbg = dbg.field(&data); - dbg.finish() - } - Self::Ident(ident) => f.debug_tuple("Ident").field(ident).finish(), - } - } +#[cfg(feature = "server")] +use crate::models::schema::*; +#[cfg(feature = "server")] +use diesel::Identifiable; + +#[cfg_attr( + feature = "server", + derive(Insertable, Queryable, Identifiable), + diesel(table_name = payloads) +)] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PayloadMeta { + pub id: Id, + pub mime_type: String, + pub name: String, + pub size: i64, } diff --git a/lib/u_lib/src/models/schema.rs b/lib/u_lib/src/models/schema.rs index 7e48a3b..6e5a3ef 100644 --- a/lib/u_lib/src/models/schema.rs +++ b/lib/u_lib/src/models/schema.rs @@ -36,16 +36,6 @@ diesel::table! { } } -diesel::table! { - use crate::schema_exports::*; - - certificates (id) { - agent_id -> Uuid, - id -> Uuid, - is_revoked -> Bool, - } -} - diesel::table! { use crate::schema_exports::*; use super::sql_types::Jobtype; @@ -56,11 +46,22 @@ diesel::table! { id -> Uuid, exec_type -> Jobtype, platform -> Text, - payload -> Nullable, + payload -> Nullable, schedule -> Nullable, } } +diesel::table! { + use crate::schema_exports::*; + + payloads (id) { + id -> Uuid, + mime_type -> Text, + name -> Text, + size -> Int8, + } +} + diesel::table! { use crate::schema_exports::*; use super::sql_types::Jobstate; @@ -80,8 +81,8 @@ diesel::table! { } } -diesel::joinable!(certificates -> agents (agent_id)); +diesel::joinable!(jobs -> payloads (payload)); diesel::joinable!(results -> agents (agent_id)); diesel::joinable!(results -> jobs (job_id)); -diesel::allow_tables_to_appear_in_same_query!(agents, certificates, jobs, results,); +diesel::allow_tables_to_appear_in_same_query!(agents, jobs, payloads, results,); diff --git a/lib/u_lib/src/ufs/mod.rs b/lib/u_lib/src/ufs/mod.rs index d8ec748..be35999 100644 --- a/lib/u_lib/src/ufs/mod.rs +++ b/lib/u_lib/src/ufs/mod.rs @@ -1,5 +1,5 @@ // This module is aiming to store obfuscated payloads, get them by name, -// delete or prepare to execute via memfd_create (unix) +// rename, update, delete or prepare to execute via memfd_create (unix) use once_cell::sync::Lazy; use parking_lot::RwLock; @@ -20,6 +20,7 @@ struct FileMeta { path: PathBuf, obfuscated: bool, extension: Option, + external: bool, } /// Remove deleted files from index @@ -58,8 +59,18 @@ pub fn read(name: impl AsRef) -> Result, Error> { fs::read(&meta.path).map_err(|e| Error::new(e, name)) } +pub fn create(data: impl AsRef<[u8]>) -> Result { + let name = Uuid::new_v4().simple().to_string(); + + put(&name, data)?; + + Ok(name) +} + /// Create new file and add to index pub fn put(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<(), Error> { + sync_index(); + let name = name.as_ref(); let obfuscate = !cfg!(feature = "server") && !cfg!(feature = "panel"); @@ -83,21 +94,83 @@ pub fn put(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<(), Error> { fs::write(&path, data).map_err(|e| Error::new(e, name))?; - let mut index = INDEX.write(); - index.insert( + INDEX.write().insert( name.to_string(), FileMeta { path, obfuscated: obfuscate, extension, + external: false, }, ); Ok(()) } -/// Add existing file to index -pub fn put_existing(path: impl AsRef) -> Result<(), Error> { +pub fn remove(name: impl AsRef) -> Result<(), Error> { + sync_index(); + + let name = name.as_ref(); + + match INDEX.write().remove(name) { + Some(value) => fs::remove_file(value.path).map_err(|e| Error::new(e, name)), + None => Ok(()), + } +} + +pub fn rename(old_name: impl AsRef, new_name: impl AsRef) -> Result<(), Error> { + sync_index(); + + let old_name = old_name.as_ref(); + let new_name = new_name.as_ref(); + + if old_name == new_name { + return Ok(()); + } + + if !in_index(old_name) { + return Err(Error::not_found(old_name)); + } + + if in_index(new_name) { + return Err(Error::already_exists(new_name)); + } + + let mut value = INDEX.write().remove(old_name).unwrap(); + + if !value.obfuscated { + let old_path = value.path.clone(); + + value.path.pop(); + value.path.push(new_name); + + fs::rename(old_path, &value.path).map_err(|e| Error::new(e, &value.path))?; + } + + INDEX.write().insert(new_name.to_string(), value); + + Ok(()) +} + +pub fn update_payload_data(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<(), Error> { + sync_index(); + + let name = name.as_ref(); + let external = INDEX.read().get(name).map(|v| v.external).unwrap_or(false); + + if external { + INDEX.write().remove(name); + } else if in_index(&name) { + remove(&name)?; + } + + put(name, data) +} + +/// Add an existing file to index +pub fn put_external(path: impl AsRef) -> Result<(), Error> { + sync_index(); + let path = path.as_ref(); let path_str = path.as_os_str().to_string_lossy().to_string(); @@ -109,19 +182,20 @@ pub fn put_existing(path: impl AsRef) -> Result<(), Error> { return Err(Error::already_exists(&path)); } - let mut index = INDEX.write(); - index.insert( + INDEX.write().insert( path_str, FileMeta { path: path.to_owned(), obfuscated: false, extension: path.file_stem().map(ToOwned::to_owned), + external: true, }, ); Ok(()) } +/// Prepare executable file: unpack, decipher if needed and send under memfd #[cfg(unix)] pub fn prepare_executable(name: impl AsRef) -> Result<(File, String), Error> { use libc::getpid; diff --git a/migrations/2020-10-24-111622_create_all/down.sql b/migrations/2020-10-24-111622_create_all/down.sql index 3ded775..ec60637 100644 --- a/migrations/2020-10-24-111622_create_all/down.sql +++ b/migrations/2020-10-24-111622_create_all/down.sql @@ -1,8 +1,7 @@ -DROP TABLE ip_addrs; -DROP TABLE results; -DROP TABLE certificates; -DROP TABLE jobs; -DROP TABLE agents; +DROP TABLE IF EXISTS results; +DROP TABLE IF EXISTS jobs; +DROP TABLE IF EXISTS payloads; +DROP TABLE IF EXISTS agents; DROP TYPE IF EXISTS JobState; DROP TYPE IF EXISTS JobType; diff --git a/migrations/2020-10-24-111622_create_all/up.sql b/migrations/2020-10-24-111622_create_all/up.sql index 41bf5e7..8e0af5a 100644 --- a/migrations/2020-10-24-111622_create_all/up.sql +++ b/migrations/2020-10-24-111622_create_all/up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS agents ( alias TEXT, hostname TEXT NOT NULL, host_info TEXT NOT NULL, - id UUID NOT NULL DEFAULT uuid_generate_v4(), + id UUID NOT NULL, ip_gray TEXT, ip_white TEXT, is_root BOOLEAN NOT NULL DEFAULT false, @@ -22,15 +22,25 @@ CREATE TABLE IF NOT EXISTS agents ( PRIMARY KEY(id) ); +CREATE TABLE IF NOT EXISTS payloads ( + id UUID NOT NULL, + mime_type TEXT NOT NULL, + name TEXT NOT NULL UNIQUE, + size BIGINT NOT NULL, + + PRIMARY KEY(id) +); + CREATE TABLE IF NOT EXISTS jobs ( alias TEXT, argv TEXT NOT NULL, - id UUID NOT NULL DEFAULT uuid_generate_v4(), + id UUID NOT NULL, exec_type JobType NOT NULL DEFAULT 'shell', platform TEXT NOT NULL, - payload TEXT, + payload UUID, schedule TEXT, + FOREIGN KEY(payload) REFERENCES payloads(id) ON DELETE SET NULL, PRIMARY KEY(id) ); @@ -38,7 +48,7 @@ CREATE TABLE IF NOT EXISTS results ( agent_id UUID NOT NULL, alias TEXT, created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - id UUID NOT NULL DEFAULT uuid_generate_v4(), + id UUID NOT NULL, job_id UUID NOT NULL, result BYTEA, state JobState NOT NULL DEFAULT 'queued', @@ -49,13 +59,4 @@ CREATE TABLE IF NOT EXISTS results ( FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE, FOREIGN KEY(job_id) REFERENCES jobs(id) ON DELETE CASCADE, PRIMARY KEY(id) -); - -CREATE TABLE IF NOT EXISTS certificates ( - agent_id UUID NOT NULL, - id UUID NOT NULL DEFAULT uuid_generate_v4(), - is_revoked BOOLEAN NOT NULL DEFAULT FALSE, - - PRIMARY KEY(id), - FOREIGN KEY(agent_id) REFERENCES agents(id) ); \ No newline at end of file -- 2.36.2 From d0d7d0aca5c7b75a84a8a717e4f8ae9983c11847 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Wed, 5 Apr 2023 18:44:45 +0300 Subject: [PATCH 02/10] improve api, add integration tests --- Cargo.lock | 699 +++++++++++++--------- bin/u_agent/src/lib.rs | 17 +- bin/u_panel/src/argparse.rs | 16 +- bin/u_panel/src/gui/mod.rs | 6 +- bin/u_panel/src/main.rs | 4 +- bin/u_server/src/db.rs | 12 +- integration/Cargo.toml | 4 +- integration/tests/fixtures/agent.rs | 8 +- integration/tests/fixtures/connections.rs | 10 +- integration/tests/integration/api.rs | 46 +- integration/tests/integration/mod.rs | 1 + lib/u_lib/src/api.rs | 54 +- lib/u_lib/src/messaging.rs | 2 + lib/u_lib/src/misc.rs | 16 + lib/u_lib/src/models/jobs/meta.rs | 6 +- lib/u_lib/src/models/jobs/misc.rs | 2 +- lib/u_lib/src/models/payload.rs | 2 +- 17 files changed, 550 insertions(+), 355 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e3b502..90198d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,15 +36,15 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.3.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0070905b2c4a98d184c4e81025253cb192aa8a73827553f38e9410801ceb35bb" +checksum = "c2079246596c18b4a33e274ae10c0e50613f4d32a4198e09c7b93771013fed74" dependencies = [ "actix-codec", "actix-rt", "actix-service", "actix-utils", - "ahash", + "ahash 0.8.3", "base64 0.21.0", "bitflags", "brotli", @@ -80,7 +80,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" dependencies = [ "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -147,9 +147,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.3.0" +version = "4.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464e0fddc668ede5f26ec1f9557a8d44eda948732f40c6b0ad79126930eb775f" +checksum = "cd3cb42f9566ab176e1ef0b8b3a896529062b4efc6be0123046095914c4c1c96" dependencies = [ "actix-codec", "actix-http", @@ -160,7 +160,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web-codegen", - "ahash", + "ahash 0.7.6", "bytes", "bytestring", "cfg-if 1.0.0", @@ -182,20 +182,20 @@ dependencies = [ "serde_urlencoded", "smallvec", "socket2", - "time 0.3.19", + "time 0.3.20", "url", ] [[package]] name = "actix-web-codegen" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa9362663c8643d67b2d5eafba49e4cb2c8a053a29ed00a0bea121f17c76b13" +checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9" dependencies = [ "actix-router", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -221,6 +221,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if 1.0.0", + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.20" @@ -271,13 +283,13 @@ checksum = "a26fa4d7e3f2eebadf743988fc8aec9fa9a9e82611acafd77c1462ed6262440a" [[package]] name = "async-trait" -version = "0.1.64" +version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.13", ] [[package]] @@ -317,9 +329,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "block-buffer" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] @@ -351,16 +363,6 @@ dependencies = [ "alloc-stdlib", ] -[[package]] -name = "buf_redux" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f" -dependencies = [ - "memchr", - "safemem", -] - [[package]] name = "bumpalo" version = "3.12.0" @@ -381,9 +383,9 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "bytestring" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7f83e57d9154148e355404702e2694463241880b939570d7c97c014da7a69a1" +checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae" dependencies = [ "bytes", ] @@ -411,9 +413,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.23" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ "iana-time-zone", "js-sys", @@ -462,7 +464,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding", - "time 0.3.19", + "time 0.3.20", "version_check", ] @@ -478,15 +480,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" dependencies = [ "libc", ] @@ -502,9 +504,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -512,9 +514,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", @@ -523,9 +525,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg", "cfg-if 1.0.0", @@ -536,9 +538,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if 1.0.0", ] @@ -555,9 +557,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.91" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" +checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" dependencies = [ "cc", "cxxbridge-flags", @@ -567,9 +569,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.91" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" +checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" dependencies = [ "cc", "codespan-reporting", @@ -577,24 +579,24 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn", + "syn 2.0.13", ] [[package]] name = "cxxbridge-flags" -version = "1.0.91" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" +checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" [[package]] name = "cxxbridge-macro" -version = "1.0.91" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" +checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.13", ] [[package]] @@ -659,7 +661,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 1.0.109", ] [[package]] @@ -686,19 +688,19 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "diesel_derives" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b758c91dbc3fe1fdcb0dba5bd13276c6a66422f2ef5795b58488248a310aa" +checksum = "0ad74fdcf086be3d4fdd142f67937678fe60ed431c3b2f08599e7687269410c4" dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -769,6 +771,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "errno" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.45.0", +] + [[package]] name = "errno-dragonfly" version = "0.1.2" @@ -830,9 +843,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", @@ -845,9 +858,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", "futures-sink", @@ -855,15 +868,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" [[package]] name = "futures-executor" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" dependencies = [ "futures-core", "futures-task", @@ -872,38 +885,44 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-macro" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.13", ] [[package]] name = "futures-sink" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" [[package]] name = "futures-task" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-channel", "futures-core", @@ -919,9 +938,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -944,7 +963,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35a8ce923c7490629d84e12fa2f75e1733f1ec692a47c264f9b7fd632855afc" dependencies = [ - "errno", + "errno 0.2.8", "libc", "log", "winapi", @@ -952,9 +971,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -1033,11 +1052,17 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "http" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", @@ -1069,9 +1094,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.24" +version = "0.14.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" dependencies = [ "bytes", "futures-channel", @@ -1106,16 +1131,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows", ] [[package]] @@ -1159,7 +1184,7 @@ dependencies = [ "proc-macro-hack", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1174,9 +1199,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown", @@ -1198,7 +1223,7 @@ dependencies = [ "futures", "once_cell", "reqwest", - "rstest", + "rstest 0.17.0", "serde", "serde_json", "shlex", @@ -1208,23 +1233,34 @@ dependencies = [ "uuid", ] +[[package]] +name = "io-lifetimes" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.45.0", +] + [[package]] name = "ipnet" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" +checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "jobserver" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" dependencies = [ "libc", ] @@ -1252,9 +1288,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.141" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" [[package]] name = "libflate" @@ -1285,6 +1321,12 @@ dependencies = [ "cc", ] +[[package]] +name = "linux-raw-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" + [[package]] name = "local-channel" version = "0.1.3" @@ -1339,9 +1381,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ "autocfg", ] @@ -1380,9 +1422,9 @@ dependencies = [ [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" @@ -1416,21 +1458,17 @@ dependencies = [ ] [[package]] -name = "multipart" -version = "0.18.0" +name = "multiparty" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00dec633863867f29cb39df64a397cdf4a6354708ddd7759f70c7fb51c5f9182" +checksum = "ed1ec6589a6d4a1e0b33b4c0a3f6ee96dfba88ebdb3da51403fd7cf0a24a4b04" dependencies = [ - "buf_redux", + "bytes", + "futures-core", "httparse", - "log", - "mime", - "mime_guess", - "quick-error", - "rand", - "safemem", - "tempfile", - "twoway", + "memchr", + "pin-project-lite", + "try-lock", ] [[package]] @@ -1520,9 +1558,9 @@ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "openssl" -version = "0.10.45" +version = "0.10.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +checksum = "4d2f106ab837a24e03672c59b1239669a0596406ff657c3c0835b6b7f0f35a33" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -1535,13 +1573,13 @@ dependencies = [ [[package]] name = "openssl-macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.13", ] [[package]] @@ -1552,11 +1590,10 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.80" +version = "0.9.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +checksum = "3a20eace9dc2d82904039cb76dcf50fb1a0bba071cfd1629720b5d6f1ddba0fa" dependencies = [ - "autocfg", "cc", "libc", "pkg-config", @@ -1587,16 +1624,16 @@ checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", "windows-sys 0.45.0", ] [[package]] name = "paste" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" [[package]] name = "percent-encoding" @@ -1621,7 +1658,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1672,7 +1709,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "version_check", ] @@ -1695,24 +1732,18 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" dependencies = [ "unicode-ident", ] -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - [[package]] name = "quote" -version = "1.0.23" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -1749,9 +1780,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -1759,9 +1790,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -1778,11 +1809,20 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" -version = "1.7.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" dependencies = [ "aho-corasick", "memchr", @@ -1800,24 +1840,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" - -[[package]] -name = "remove_dir_all" -version = "0.5.3" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "reqwest" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" +checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" dependencies = [ "base64 0.21.0", "bytes", @@ -1887,14 +1918,40 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 1.0.109", +] + +[[package]] +name = "rstest" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de1bb486a691878cd320c2f0d319ba91eeaa2e894066d8b5f8f117c000e9d962" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290ca1a1c8ca7edb7c3283bd44dc35dd54fdec6253a3912e201ba1072018fca8" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", + "unicode-ident", ] [[package]] name = "rust-embed" -version = "6.4.2" +version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "283ffe2f866869428c92e0d61c2f35dfb4355293cdfdc48f49e895c15f1333d1" +checksum = "1b68543d5527e158213414a92832d2aab11a84d2571a5eb021ebe22c43aab066" dependencies = [ "include-flate", "rust-embed-impl", @@ -1904,22 +1961,22 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "6.3.1" +version = "6.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31ab23d42d71fb9be1b643fe6765d292c5e14d46912d13f3ae2815ca048ea04d" +checksum = "4d4e0f0ced47ded9a68374ac145edd65a6c1fa13a96447b873660b2a568a0fd7" dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn", + "syn 1.0.109", "walkdir", ] [[package]] name = "rust-embed-utils" -version = "7.3.0" +version = "7.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1669d81dfabd1b5f8e2856b8bbe146c6192b0ba22162edc738ac0a5de18f054" +checksum = "512b0ab6853f7e14e3c8754acb43d6f748bb9ced66aa5915a6553ac8213f7731" dependencies = [ "sha2", "walkdir", @@ -1934,6 +1991,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.37.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aae838e49b3d63e9274e1c01833cc8139d3fec468c3b84688c628f44b1ae11d" +dependencies = [ + "bitflags", + "errno 0.3.0", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.45.0", +] + [[package]] name = "rustls" version = "0.20.8" @@ -1948,24 +2019,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "0.2.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64 0.13.1", + "base64 0.21.0", ] [[package]] name = "ryu" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" - -[[package]] -name = "safemem" -version = "0.3.3" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "same-file" @@ -1999,9 +2064,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "sct" @@ -2038,35 +2103,35 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.152" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.13", ] [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" dependencies = [ "itoa", "ryu", @@ -2085,17 +2150,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha-1" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" -dependencies = [ - "cfg-if 1.0.0", - "cpufeatures", - "digest", -] - [[package]] name = "sha1" version = "0.10.5" @@ -2144,9 +2198,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -2159,9 +2213,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -2200,7 +2254,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2230,7 +2284,7 @@ dependencies = [ "heck 0.3.3", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2242,14 +2296,25 @@ dependencies = [ "heck 0.3.3", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" dependencies = [ "proc-macro2", "quote", @@ -2273,16 +2338,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.3.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if 1.0.0", "fastrand", - "libc", - "redox_syscall", - "remove_dir_all", - "winapi", + "redox_syscall 0.3.5", + "rustix", + "windows-sys 0.45.0", ] [[package]] @@ -2320,7 +2384,7 @@ checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2346,9 +2410,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53250a3b3fed8ff8fd988587d8925d26a83ac3845d9e03b220b37f34c2b8d6c2" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ "itoa", "serde", @@ -2364,9 +2428,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a460aeb8de6dcb0f381e1ee05f1cd56fcf5a5f6eb8187ff3d8f0b11078d38b7c" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" dependencies = [ "time-core", ] @@ -2388,14 +2452,13 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.25.0" +version = "1.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" dependencies = [ "autocfg", "bytes", "libc", - "memchr", "mio", "num_cpus", "parking_lot", @@ -2403,18 +2466,18 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.13", ] [[package]] @@ -2440,9 +2503,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" dependencies = [ "futures-core", "pin-project-lite", @@ -2451,9 +2514,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.17.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" +checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" dependencies = [ "futures-util", "log", @@ -2510,7 +2573,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" dependencies = [ "crossbeam-channel", - "time 0.3.19", + "time 0.3.20", "tracing-subscriber", ] @@ -2522,7 +2585,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2572,9 +2635,9 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "tungstenite" -version = "0.17.3" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" +checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" dependencies = [ "base64 0.13.1", "byteorder", @@ -2583,21 +2646,12 @@ dependencies = [ "httparse", "log", "rand", - "sha-1", + "sha1", "thiserror", "url", "utf-8", ] -[[package]] -name = "twoway" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" -dependencies = [ - "memchr", -] - [[package]] name = "typenum" version = "1.16.0" @@ -2637,7 +2691,7 @@ dependencies = [ "parking_lot", "platforms", "reqwest", - "rstest", + "rstest 0.12.0", "serde", "serde_json", "shlex", @@ -2694,7 +2748,7 @@ dependencies = [ "mime_guess", "once_cell", "openssl", - "rstest", + "rstest 0.12.0", "serde", "serde_json", "thiserror", @@ -2716,15 +2770,15 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.10" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -2812,12 +2866,11 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "walkdir" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" dependencies = [ "same-file", - "winapi", "winapi-util", ] @@ -2833,9 +2886,9 @@ dependencies = [ [[package]] name = "warp" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed7b8be92646fc3d18b06147664ebc5f48d222686cb11a8755e561a735aacc6d" +checksum = "27e1a710288f0f91a98dd8a74f05b76a10768db245ce183edf64dc1afdc3016c" dependencies = [ "bytes", "futures-channel", @@ -2846,7 +2899,7 @@ dependencies = [ "log", "mime", "mime_guess", - "multipart", + "multiparty", "percent-encoding", "pin-project", "rustls-pemfile", @@ -2896,7 +2949,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] @@ -2930,7 +2983,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2992,19 +3045,28 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.0", +] + [[package]] name = "windows-sys" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -3013,65 +3075,122 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets", + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] name = "windows-targets" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" [[package]] name = "windows_i686_gnu" -version = "0.42.1" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" [[package]] name = "windows_x86_64_gnu" -version = "0.42.1" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.1" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winreg" diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 7f08635..a6633bf 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -5,7 +5,7 @@ use std::process::exit; use tokio::runtime::Builder; use tokio::time::{sleep, Duration}; use u_lib::{ - api::ClientHandler, + api::HttpClient, cache::JobCache, config::{get_self_id, EndpointsEnv, AGENT_ITERATION_INTERVAL}, error::ErrChan, @@ -16,13 +16,14 @@ use u_lib::{ models::AssignedJobById, }; -pub async fn process_request(jobs: Vec, client: &ClientHandler) { +pub async fn process_request(jobs: Vec, client: &HttpClient) { if !jobs.is_empty() { for jr in &jobs { if !JobCache::contains(jr.job_id) { info!("Fetching job: {}", &jr.job_id); let fetched_job = loop { - match client.get_job(jr.job_id).await { + //todo: use payload cache + match client.get_job(jr.job_id, true).await { Ok(result) => break result, Err(err) => { debug!("{:?} \nretrying...", err); @@ -58,12 +59,12 @@ pub async fn process_request(jobs: Vec, client: &ClientHandler) } } -async fn error_reporting(client: ClientHandler) -> ! { +async fn error_reporting(client: HttpClient) -> ! { loop { match ErrChan::recv().await { Some(err) => { 'retry: for _ in 0..3 { - match client.report(Reportable::Error(err.clone())).await { + match client.report(&Reportable::Error(err.clone())).await { Ok(_) => break 'retry, Err(e) => { debug!("Reporting error: {:?}", e); @@ -77,7 +78,7 @@ async fn error_reporting(client: ClientHandler) -> ! { } } -async fn agent_loop(client: ClientHandler) -> ! { +async fn agent_loop(client: HttpClient) -> ! { let self_id = get_self_id(); loop { match client.get_personal_jobs(self_id).await { @@ -97,7 +98,7 @@ async fn agent_loop(client: ClientHandler) -> ! { .collect(); if !result.is_empty() { - if let Err(err) = client.report(result).await { + if let Err(err) = client.report(&result).await { ErrChan::send(err, "report").await; } } @@ -131,7 +132,7 @@ pub fn run_forever() -> ! { .build() .unwrap() .block_on(async { - match ClientHandler::new(&env.u_server, None).await { + match HttpClient::new(&env.u_server, None).await { Ok(client) => { tokio::spawn(error_reporting(client.clone())); agent_loop(client).await diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index eeaf168..e9da109 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -1,7 +1,7 @@ use serde_json::{from_str, to_value, Value}; use structopt::StructOpt; use u_lib::{ - api::ClientHandler, + api::HttpClient, jobs::join_payload, messaging::AsMsg, models::{Agent, AssignedJob, RawJob}, @@ -75,14 +75,14 @@ pub fn into_value(data: M) -> Value { to_value(data).unwrap() } -pub async fn process_cmd(client: ClientHandler, args: Args) -> PanelResult { +pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { let catcher: UResult = (|| async { Ok(match args.cmd { Cmd::Agents(action) => match action { RUD::Read { id } => into_value(client.get_agents(id).await?), RUD::Update { item } => { let agent = from_str::(&item)?; - into_value(client.update_agent(agent).await?) + into_value(client.update_agent(&agent).await?) } RUD::Delete { id } => into_value(client.del(id).await?), }, @@ -92,17 +92,17 @@ pub async fn process_cmd(client: ClientHandler, args: Args) -> PanelResult match id { //todo: use vec not to break frontend api, possibly refactor later - Some(id) => into_value(vec![client.get_job(id).await?]), + Some(id) => into_value(vec![client.get_job(id, false).await?]), None => into_value(client.get_jobs().await?), }, JobCRUD::RUD(RUD::Update { item }) => { let raw_job = from_str::(&item)?; let job = raw_job.validated()?; - into_value(client.update_job(join_payload(job)?).await?) + into_value(client.update_job(&join_payload(job)?).await?) } JobCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), }, @@ -110,11 +110,11 @@ pub async fn process_cmd(client: ClientHandler, args: Args) -> PanelResult into_value(client.set_jobs(agent_id, job_idents).await?), + } => into_value(client.set_jobs(agent_id, &job_idents).await?), JobMapCRUD::RUD(RUD::Read { id }) => into_value(client.get_agent_jobs(id).await?), JobMapCRUD::RUD(RUD::Update { item }) => { let assigned = from_str::(&item)?; - into_value(client.update_result(assigned).await?) + into_value(client.update_result(&assigned).await?) } JobMapCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), }, diff --git a/bin/u_panel/src/gui/mod.rs b/bin/u_panel/src/gui/mod.rs index 5be4487..729a7ad 100644 --- a/bin/u_panel/src/gui/mod.rs +++ b/bin/u_panel/src/gui/mod.rs @@ -8,7 +8,7 @@ use futures_util::StreamExt; use rust_embed::RustEmbed; use std::borrow::Cow; use structopt::StructOpt; -use u_lib::{api::ClientHandler, unwrap_enum}; +use u_lib::{api::HttpClient, unwrap_enum}; #[derive(RustEmbed)] #[folder = "./src/gui/fe/dist/fe/"] @@ -42,7 +42,7 @@ async fn resources_adapter(path: web::Path<(String,)>) -> impl Responder { #[post("/cmd/")] async fn send_cmd( mut body: web::Payload, - client: web::Data, + client: web::Data, ) -> Result { let mut bytes = web::BytesMut::new(); @@ -74,7 +74,7 @@ async fn send_cmd( Ok(response) } -pub async fn serve(client: ClientHandler) -> anyhow::Result<()> { +pub async fn serve(client: HttpClient) -> anyhow::Result<()> { info!("Connecting to u_server..."); client.ping().await?; diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index f707aef..226c592 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -7,14 +7,14 @@ extern crate tracing; use anyhow::Result as AnyResult; use argparse::{process_cmd, Args}; use structopt::StructOpt; -use u_lib::api::ClientHandler; +use u_lib::api::HttpClient; use u_lib::config::AccessEnv; use u_lib::logging::init_logger; #[actix_web::main] async fn main() -> AnyResult<()> { let env = AccessEnv::load()?; - let client = ClientHandler::new(&env.u_server, Some(env.admin_auth_token)).await?; + let client = HttpClient::new(&env.u_server, Some(env.admin_auth_token)).await?; let args = Args::from_args(); init_logger(None::<&str>); diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index 6d43f96..7963774 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -250,9 +250,15 @@ impl UDB<'_> { } pub fn upsert_agent(&mut self, agent: &Agent) -> Result<()> { - agent - .save_changes::(self.conn) - .map_err(with_err_ctx(format!("Can't upsert agent {agent:?}")))?; + use schema::agents; + + diesel::insert_into(agents::table) + .values(agent) + .on_conflict(agents::id) + .do_update() + .set(agent) + .execute(self.conn) + .map_err(with_err_ctx(format!("Can't insert agent {agent:?}")))?; Ok(()) } diff --git a/integration/Cargo.toml b/integration/Cargo.toml index 16ea59a..e5ec3fc 100644 --- a/integration/Cargo.toml +++ b/integration/Cargo.toml @@ -10,14 +10,14 @@ edition = "2021" futures = { version = "0.3", features = ["executor"] } once_cell = "1.10.0" reqwest = { workspace = true } -rstest = "0.12" +rstest = "0.17" serde = { workspace = true } serde_json = { workspace = true } shlex = "1.0.0" tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "time"] } tracing = { workspace = true } uuid = { workspace = true, features = ["serde", "v4"] } -u_lib = { path = "../lib/u_lib", features = ["panel"] } +u_lib = { path = "../lib/u_lib", features = ["panel", "server"] } [[test]] diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs index 7e3f3df..c7cf217 100644 --- a/integration/tests/fixtures/agent.rs +++ b/integration/tests/fixtures/agent.rs @@ -1,7 +1,7 @@ use super::connections::*; use super::run_async; use u_lib::{ - api::ClientHandler, config::get_self_id, jobs::split_payload, messaging::Reportable, models::*, + api::HttpClient, config::get_self_id, jobs::split_payload, messaging::Reportable, models::*, types::Id, }; @@ -11,7 +11,7 @@ pub struct RegisteredAgent { #[fixture] #[once] -pub fn registered_agent(client: &ClientHandler) -> RegisteredAgent { +pub fn registered_agent(client: &HttpClient) -> RegisteredAgent { run_async(async { let agent_id = get_self_id(); println!("registering agent {agent_id}"); @@ -22,12 +22,12 @@ pub fn registered_agent(client: &ClientHandler) -> RegisteredAgent { .pop() .unwrap(); let job_id = resp.job_id; - let job = client.get_job(job_id).await.unwrap(); + let job = client.get_job(job_id, true).await.unwrap(); assert_eq!(job.job.alias, Some("agent_hello".to_string())); let mut agent_data = AssignedJob::from((&split_payload(job).unwrap().job, resp)); agent_data.set_result(&Agent::with_id(agent_id)); client - .report(Reportable::Assigned(agent_data)) + .report(&Reportable::Assigned(agent_data)) .await .unwrap(); RegisteredAgent { id: agent_id } diff --git a/integration/tests/fixtures/connections.rs b/integration/tests/fixtures/connections.rs index d4c6986..87ae099 100644 --- a/integration/tests/fixtures/connections.rs +++ b/integration/tests/fixtures/connections.rs @@ -1,20 +1,20 @@ use super::env::*; use super::run_async; -pub use u_lib::api::ClientHandler; +pub use u_lib::api::HttpClient; use u_lib::db::unpooled; pub use u_lib::db::PgConnection; #[fixture] #[once] -pub fn client(env_default: EndpointsEnv) -> ClientHandler { - run_async(ClientHandler::new(&env_default.u_server, None)).unwrap() +pub fn client(env_default: EndpointsEnv) -> HttpClient { + run_async(HttpClient::new(&env_default.u_server, None)).unwrap() } #[fixture] #[once] -pub fn client_panel(env_access: AccessEnv) -> ClientHandler { - run_async(ClientHandler::new( +pub fn client_panel(env_access: AccessEnv) -> HttpClient { + run_async(HttpClient::new( &env_access.u_server, Some(env_access.admin_auth_token), )) diff --git a/integration/tests/integration/api.rs b/integration/tests/integration/api.rs index 12af506..a0c5c25 100644 --- a/integration/tests/integration/api.rs +++ b/integration/tests/integration/api.rs @@ -1 +1,45 @@ -//async fn +// get_personal_jobs(&self, url_param: Id) +// report(&self, payload: impl OneOrVec) +// dl(&self, file: String) +// get_job(&self, job: Id) +// get_jobs(&self) +// get_agents(&self, agent: Option) +// update_agent(&self, agent: Agent) +// update_job(&self, job: FatJob) +// update_result(&self, result: AssignedJob) +// upload_jobs(&self, payload: impl OneOrVec) +// del(&self, item: Id) +// set_jobs(&self, agent: Id, job_idents: impl OneOrVec) +// get_agent_jobs(&self, agent: Option) +// ping(&self) + +use crate::fixtures::connections::*; +use u_lib::jobs::join_payload; +use u_lib::models::RawJob; + +#[rstest] +#[tokio::test] +async fn test_jobs_endpoints(client_panel: &HttpClient) { + let job_alias = "henlo"; + let job = RawJob::builder() + .with_shell("echo henlo") + .with_alias(job_alias) + .build() + .unwrap(); + let job_id = job.job.id; + let mut fat_job = join_payload(job).unwrap(); + + client_panel.upload_jobs(&fat_job).await.unwrap(); + + let fetched_job = client_panel.get_job(job_id, false).await.unwrap(); + assert_eq!(fat_job, fetched_job); + + fat_job.job.alias = Some("henlo2".to_string()); + client_panel.update_job(&fat_job).await.unwrap(); + + let fetched_job = client_panel.get_job(job_id, false).await.unwrap(); + assert_eq!(fat_job, fetched_job); + + client_panel.del(job_id).await.unwrap(); + client_panel.get_job(job_id, false).await.unwrap(); // should fail with 404 +} diff --git a/integration/tests/integration/mod.rs b/integration/tests/integration/mod.rs index 0b76512..117235a 100644 --- a/integration/tests/integration/mod.rs +++ b/integration/tests/integration/mod.rs @@ -1,2 +1,3 @@ +mod api; mod behaviour; mod connection; diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index c1d18d4..df1f8a2 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -1,6 +1,6 @@ -use std::collections::HashMap; use std::fmt::Debug; use std::net::SocketAddr; +use std::{collections::HashMap, time::Duration}; use anyhow::{Context, Result}; use reqwest::{header, header::HeaderMap, Certificate, Client, Identity, Method, Url}; @@ -11,7 +11,7 @@ use crate::{ config::{get_self_id, MASTER_PORT}, conv::opt_to_string, messaging::{self, AsMsg}, - misc::OneOrVec, + misc::OneOrVecRef, models::*, types::Id, UError, UResult, @@ -21,12 +21,12 @@ const AGENT_IDENTITY: &[u8] = include_bytes!("../../../certs/alice.p12"); const ROOT_CA_CERT: &[u8] = include_bytes!("../../../certs/ca.crt"); #[derive(Clone, Debug)] -pub struct ClientHandler { +pub struct HttpClient { base_url: Url, client: Client, } -impl ClientHandler { +impl HttpClient { pub async fn new(server: &str, password: Option) -> UResult { let identity = Identity::from_pkcs12_der(AGENT_IDENTITY, "").unwrap(); let mut default_headers = @@ -41,7 +41,8 @@ impl ClientHandler { let client = Client::builder() .identity(identity) .default_headers(HeaderMap::try_from(&default_headers).unwrap()) - .add_root_certificate(Certificate::from_pem(ROOT_CA_CERT).unwrap()); + .add_root_certificate(Certificate::from_pem(ROOT_CA_CERT).unwrap()) + .timeout(Duration::from_secs(20)); let dns_response = Client::new() .request( @@ -76,18 +77,18 @@ impl ClientHandler { } async fn req(&self, url: impl AsRef) -> Result { - self.req_with_payload(url, ()).await + self.req_with_payload(url, &()).await } async fn req_with_payload( &self, url: impl AsRef, - payload: P, + payload: &P, ) -> Result { let request = self .client .post(self.base_url.join(url.as_ref()).unwrap()) - .json(&payload); + .json(payload); let response = request .send() @@ -112,23 +113,24 @@ impl ClientHandler { } // get jobs for client - pub async fn get_personal_jobs(&self, url_param: Id) -> Result> { - self.req(format!("get_personal_jobs/{}", url_param)).await + pub async fn get_personal_jobs(&self, agent_id: Id) -> Result> { + self.req(format!("get_personal_jobs/{}", agent_id)).await } // send something to server - pub async fn report(&self, payload: impl OneOrVec) -> Result<()> { - self.req_with_payload("report", payload.into_vec()).await + pub async fn report(&self, payload: impl OneOrVecRef) -> Result<()> { + self.req_with_payload("report", &payload.as_vec()).await } - // download file - pub async fn dl(&self, file: String) -> Result> { + // download payload + pub async fn dl(&self, file: &str) -> Result> { self.req(format!("dl/{file}")).await } /// get exact job - pub async fn get_job(&self, job: Id) -> Result { - self.req(format!("get_job/{job}")).await + pub async fn get_job(&self, job: Id, force_payload: bool) -> Result { + self.req(format!("get_job/{job}?force_payload={force_payload}")) + .await } /// get all available jobs @@ -139,7 +141,7 @@ impl ClientHandler { //##########// Admin area //##########// #[cfg(feature = "panel")] -impl ClientHandler { +impl HttpClient { /// agent listing pub async fn get_agents(&self, agent: Option) -> Result> { self.req(format!("get_agents/{}", opt_to_string(agent))) @@ -147,23 +149,23 @@ impl ClientHandler { } /// update agent - pub async fn update_agent(&self, agent: Agent) -> Result<()> { + pub async fn update_agent(&self, agent: &Agent) -> Result<()> { self.req_with_payload("update_agent", agent).await } /// update job - pub async fn update_job(&self, job: FatJob) -> Result<()> { + pub async fn update_job(&self, job: &FatJob) -> Result<()> { self.req_with_payload("update_job", job).await } /// update result - pub async fn update_result(&self, result: AssignedJob) -> Result<()> { + pub async fn update_result(&self, result: &AssignedJob) -> Result<()> { self.req_with_payload("update_result", result).await } /// create and upload job - pub async fn upload_jobs(&self, payload: impl OneOrVec) -> Result> { - self.req_with_payload("upload_jobs", payload.into_vec()) + pub async fn upload_jobs(&self, payload: impl OneOrVecRef) -> Result> { + self.req_with_payload("upload_jobs", &payload.as_vec()) .await } @@ -173,8 +175,12 @@ impl ClientHandler { } /// set jobs for any agent - pub async fn set_jobs(&self, agent: Id, job_idents: impl OneOrVec) -> Result> { - self.req_with_payload(format!("set_jobs/{agent}"), job_idents.into_vec()) + pub async fn set_jobs( + &self, + agent: Id, + job_idents: impl OneOrVecRef, + ) -> Result> { + self.req_with_payload(format!("set_jobs/{agent}"), &job_idents.as_vec()) .await } diff --git a/lib/u_lib/src/messaging.rs b/lib/u_lib/src/messaging.rs index 88089e2..816e12f 100644 --- a/lib/u_lib/src/messaging.rs +++ b/lib/u_lib/src/messaging.rs @@ -23,6 +23,8 @@ impl AsMsg for () {} impl AsMsg for Vec {} impl<'msg, M: AsMsg> AsMsg for &'msg [M] {} +impl AsMsg for &M {} + #[derive(Serialize, Deserialize, Clone, Debug)] pub enum Reportable { Assigned(AssignedJob), diff --git a/lib/u_lib/src/misc.rs b/lib/u_lib/src/misc.rs index 7d185f1..29723a5 100644 --- a/lib/u_lib/src/misc.rs +++ b/lib/u_lib/src/misc.rs @@ -14,6 +14,22 @@ impl OneOrVec for Vec { } } +pub trait OneOrVecRef { + fn as_vec(&self) -> Vec<&T>; +} + +impl OneOrVecRef for &T { + fn as_vec(&self) -> Vec<&T> { + vec![self] + } +} + +impl OneOrVecRef for &Vec { + fn as_vec(&self) -> Vec<&T> { + self.iter().collect() + } +} + #[macro_export] macro_rules! unwrap_enum { ($src:expr, $t:path) => { diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 5f525f0..cefd511 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; use std::fs::metadata; use std::process::Command; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr( feature = "server", derive(Queryable, Identifiable, Insertable, AsChangeset), @@ -34,14 +34,14 @@ pub struct JobModel { pub schedule: Option, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct FatJob { pub job: JobModel, pub payload_meta: Option, pub payload_data: Option>, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct ThinJob { pub job: JobModel, pub payload_meta: Option, diff --git a/lib/u_lib/src/models/jobs/misc.rs b/lib/u_lib/src/models/jobs/misc.rs index 6a99f7c..94de0b1 100644 --- a/lib/u_lib/src/models/jobs/misc.rs +++ b/lib/u_lib/src/models/jobs/misc.rs @@ -26,7 +26,7 @@ pub enum JobState { Finished, } -#[derive(Default, Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Display)] +#[derive(Default, Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Display)] #[cfg_attr( feature = "server", derive(DbEnum), diff --git a/lib/u_lib/src/models/payload.rs b/lib/u_lib/src/models/payload.rs index 46fcec8..6cac041 100644 --- a/lib/u_lib/src/models/payload.rs +++ b/lib/u_lib/src/models/payload.rs @@ -11,7 +11,7 @@ use diesel::Identifiable; derive(Insertable, Queryable, Identifiable), diesel(table_name = payloads) )] -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct PayloadMeta { pub id: Id, pub mime_type: String, -- 2.36.2 From 1b0cdae404e55bfa309a6fe6c2efc1cfa584ae96 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sat, 15 Apr 2023 00:43:54 +0300 Subject: [PATCH 03/10] improve ufs index, fix unit and most integration tests, remove futures --- Cargo.lock | 127 ++++++---- Makefile.toml | 10 +- bin/u_agent/Cargo.toml | 2 +- bin/u_panel/src/argparse.rs | 12 +- bin/u_server/src/db.rs | 2 +- bin/u_server/src/handlers.rs | 12 +- bin/u_server/src/u_server.rs | 2 +- integration/Cargo.toml | 1 - integration/integration_tests.sh | 2 + integration/tests/fixtures/agent.rs | 10 +- integration/tests/fixtures/mod.rs | 12 +- integration/tests/helpers/panel.rs | 4 +- integration/tests/integration/api.rs | 1 + integration/tests/integration/behaviour.rs | 22 +- lib/u_lib/Cargo.toml | 4 +- lib/u_lib/src/api.rs | 17 +- lib/u_lib/src/error/mod.rs | 35 ++- lib/u_lib/src/executor.rs | 11 +- lib/u_lib/src/jobs.rs | 14 +- lib/u_lib/src/models/agent.rs | 55 ++--- lib/u_lib/src/models/jobs/meta.rs | 146 ++++++------ lib/u_lib/src/models/schema.rs | 2 +- lib/u_lib/src/platform.rs | 26 +- lib/u_lib/src/ufs/error.rs | 15 ++ lib/u_lib/src/ufs/index.rs | 101 ++++++++ lib/u_lib/src/ufs/mod.rs | 225 +++++++++--------- .../2020-10-24-111622_create_all/up.sql | 2 +- 27 files changed, 536 insertions(+), 336 deletions(-) create mode 100644 lib/u_lib/src/ufs/index.rs diff --git a/Cargo.lock b/Cargo.lock index 90198d3..1cd2c46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -289,7 +289,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -321,6 +321,15 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -504,9 +513,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -579,7 +588,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -596,7 +605,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -773,13 +782,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -897,7 +906,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -948,9 +957,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if 1.0.0", "libc", @@ -971,9 +980,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" +checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f" dependencies = [ "bytes", "fnv", @@ -1094,9 +1103,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.25" +version = "0.14.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" dependencies = [ "bytes", "futures-channel", @@ -1220,7 +1229,6 @@ dependencies = [ name = "integration" version = "0.1.0" dependencies = [ - "futures", "once_cell", "reqwest", "rstest 0.17.0", @@ -1235,13 +1243,13 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" dependencies = [ "hermit-abi 0.3.1", "libc", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -1274,6 +1282,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keccak" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3afef3b6eff9ce9d8ff9b3601125eec7f0c8cbac7abd14f355d053fa56c98768" +dependencies = [ + "cpufeatures", +] + [[package]] name = "language-tags" version = "0.3.2" @@ -1558,9 +1575,9 @@ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "openssl" -version = "0.10.49" +version = "0.10.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d2f106ab837a24e03672c59b1239669a0596406ff657c3c0835b6b7f0f35a33" +checksum = "7e30d8bc91859781f0a943411186324d580f2bbeb71b452fe91ae344806af3f1" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -1579,7 +1596,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -1590,9 +1607,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.84" +version = "0.9.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a20eace9dc2d82904039cb76dcf50fb1a0bba071cfd1629720b5d6f1ddba0fa" +checksum = "0d3d193fb1488ad46ffe3aaabc912cc931d02ee8518fe2959aea8ef52718b0c0" dependencies = [ "cc", "libc", @@ -1993,16 +2010,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.7" +version = "0.37.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aae838e49b3d63e9274e1c01833cc8139d3fec468c3b84688c628f44b1ae11d" +checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" dependencies = [ "bitflags", - "errno 0.3.0", + "errno 0.3.1", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -2109,29 +2126,29 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" +checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] name = "serde_json" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", @@ -2172,6 +2189,16 @@ dependencies = [ "digest", ] +[[package]] +name = "sha3" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54c2bb1a323307527314a36bfb73f24febb08ce2b8a554bf4ffd6f51ad15198c" +dependencies = [ + "digest", + "keccak", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -2312,9 +2339,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.13" +version = "2.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" dependencies = [ "proc-macro2", "quote", @@ -2477,7 +2504,7 @@ checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -2675,6 +2702,7 @@ name = "u_lib" version = "0.1.0" dependencies = [ "anyhow", + "bincode", "chrono", "daemonize", "deadpool-diesel", @@ -2682,7 +2710,6 @@ dependencies = [ "diesel-derive-enum", "dotenv", "envy", - "futures", "guess_host_triple", "lazy_static", "libc", @@ -2694,6 +2721,7 @@ dependencies = [ "rstest 0.12.0", "serde", "serde_json", + "sha3", "shlex", "strum 0.20.0", "thiserror", @@ -2826,9 +2854,9 @@ checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "uuid" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" +checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb" dependencies = [ "getrandom", "serde", @@ -3078,6 +3106,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -3212,9 +3249,9 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "6.0.4+zstd.1.5.4" +version = "6.0.5+zstd.1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7afb4b54b8910cf5447638cb54bf4e8a65cbedd783af98b98c62ffe91f185543" +checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" dependencies = [ "libc", "zstd-sys", @@ -3222,9 +3259,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.7+zstd.1.5.4" +version = "2.0.8+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" dependencies = [ "cc", "libc", diff --git a/Makefile.toml b/Makefile.toml index 08e5742..ad3c9cd 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -69,11 +69,14 @@ clear = true [tasks.run] disabled = true -[tasks.unit] +[tasks.unit-tests] command = "${CARGO}" args = ["test", "--target", "${TARGET}", "--lib", "--", "${@}"] -[tasks.integration] +[tasks.ut] +alias = "unit-tests" + +[tasks.integration-tests] dependencies = ["cargo_update"] script = ''' [[ ! -d "./target/${TARGET}/${PROFILE_OVERRIDE}" ]] && echo 'No target folder. Build project first' && exit 1 @@ -81,6 +84,9 @@ cd ./integration bash integration_tests.sh ${@} ''' +[tasks.it] +alias = "integration-tests" + [tasks.test] dependencies = ["unit", "integration"] diff --git a/bin/u_agent/Cargo.toml b/bin/u_agent/Cargo.toml index 3b16a92..eced273 100644 --- a/bin/u_agent/Cargo.toml +++ b/bin/u_agent/Cargo.toml @@ -12,5 +12,5 @@ reqwest = { workspace = true } sysinfo = "0.10.5" tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "time"] } uuid = { workspace = true } -u_lib = { path = "../../lib/u_lib" } +u_lib = { path = "../../lib/u_lib", features = ["agent"] } diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index e9da109..75750bb 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -81,14 +81,16 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { Cmd::Agents(action) => match action { RUD::Read { id } => into_value(client.get_agents(id).await?), RUD::Update { item } => { - let agent = from_str::(&item)?; + let agent = from_str::(&item) + .map_err(|e| UError::DeserializeError(e.to_string(), item))?; into_value(client.update_agent(&agent).await?) } RUD::Delete { id } => into_value(client.del(id).await?), }, Cmd::Jobs(action) => match action { JobCRUD::Create { job } => { - let raw_job = from_str::(&job)?; + let raw_job = from_str::(&job) + .map_err(|e| UError::DeserializeError(e.to_string(), job))?; let job = raw_job.validated()?; let fat_job = join_payload(job)?; @@ -100,7 +102,8 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { None => into_value(client.get_jobs().await?), }, JobCRUD::RUD(RUD::Update { item }) => { - let raw_job = from_str::(&item)?; + let raw_job = from_str::(&item) + .map_err(|e| UError::DeserializeError(e.to_string(), item))?; let job = raw_job.validated()?; into_value(client.update_job(&join_payload(job)?).await?) } @@ -113,7 +116,8 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { } => into_value(client.set_jobs(agent_id, &job_idents).await?), JobMapCRUD::RUD(RUD::Read { id }) => into_value(client.get_agent_jobs(id).await?), JobMapCRUD::RUD(RUD::Update { item }) => { - let assigned = from_str::(&item)?; + let assigned = from_str::(&item) + .map_err(|e| UError::DeserializeError(e.to_string(), item))?; into_value(client.update_result(&assigned).await?) } JobMapCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index 7963774..3de24e6 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -186,7 +186,7 @@ impl UDB<'_> { }; let jobs_meta = jobs::table - .select((jobs::id, jobs::alias, jobs::platform)) + .select((jobs::id, jobs::alias, jobs::target_platforms)) .filter(jobs::id.eq_any(job_ids)) .load::<(Id, Option, String)>(self.conn) .map_err(with_err_ctx(format!("Can't find jobs {job_ids:?}")))?; diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index be2a260..bc82823 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -81,7 +81,8 @@ impl Endpoints { db.upsert_agent(&agent)?; } None => { - let new_agent = Agent::with_id(id); + let mut new_agent = Agent::empty(); + new_agent.id = id; db.upsert_agent(&new_agent)?; @@ -131,9 +132,9 @@ impl Endpoints { repo: Arc, agent_id: Id, job_idents: Vec, - ) -> EndpResult<()> { + ) -> EndpResult> { repo.transaction(move |mut db| { - job_idents + let assigned_job_idents = job_idents .into_iter() .map(|ident| { Id::parse_str(&ident).or_else(|_| { @@ -149,8 +150,9 @@ impl Endpoints { } }) }) - .collect::, Error>>() - .and_then(|j| db.set_jobs_for_agent(agent_id, &j)) + .collect::, Error>>()?; + db.set_jobs_for_agent(agent_id, &assigned_job_idents)?; + Ok(assigned_job_idents) }) .await .map_err(From::from) diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 40b2656..ee3f440 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -221,7 +221,7 @@ fn logger(info: Info<'_>) { } fn ok(_: T) -> impl Reply { - reply() + "null" } /* diff --git a/integration/Cargo.toml b/integration/Cargo.toml index e5ec3fc..5639c21 100644 --- a/integration/Cargo.toml +++ b/integration/Cargo.toml @@ -7,7 +7,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -futures = { version = "0.3", features = ["executor"] } once_cell = "1.10.0" reqwest = { workspace = true } rstest = "0.17" diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index b8efc7e..0b3c416 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -2,5 +2,7 @@ set -e export DOCKER_UID=$(id -u) export DOCKER_GID=$(id -g) + +rm ../logs/u_agent* [[ "$@" =~ "--release" ]] && export PROFILE=release || export PROFILE=debug python integration_tests.py $@ diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs index c7cf217..09a8e30 100644 --- a/integration/tests/fixtures/agent.rs +++ b/integration/tests/fixtures/agent.rs @@ -1,9 +1,6 @@ use super::connections::*; use super::run_async; -use u_lib::{ - api::HttpClient, config::get_self_id, jobs::split_payload, messaging::Reportable, models::*, - types::Id, -}; +use u_lib::{api::HttpClient, jobs::split_payload, messaging::Reportable, models::*, types::Id}; pub struct RegisteredAgent { pub id: Id, @@ -13,7 +10,8 @@ pub struct RegisteredAgent { #[once] pub fn registered_agent(client: &HttpClient) -> RegisteredAgent { run_async(async { - let agent_id = get_self_id(); + let agent = Agent::with_current_platform(); + let agent_id = agent.id; println!("registering agent {agent_id}"); let resp = client .get_personal_jobs(agent_id) @@ -25,7 +23,7 @@ pub fn registered_agent(client: &HttpClient) -> RegisteredAgent { let job = client.get_job(job_id, true).await.unwrap(); assert_eq!(job.job.alias, Some("agent_hello".to_string())); let mut agent_data = AssignedJob::from((&split_payload(job).unwrap().job, resp)); - agent_data.set_result(&Agent::with_id(agent_id)); + agent_data.set_result(&agent); client .report(&Reportable::Assigned(agent_data)) .await diff --git a/integration/tests/fixtures/mod.rs b/integration/tests/fixtures/mod.rs index c2cdc85..5372b09 100644 --- a/integration/tests/fixtures/mod.rs +++ b/integration/tests/fixtures/mod.rs @@ -3,7 +3,15 @@ pub mod connections; pub mod env; use std::future::Future; +use std::thread; +use tokio::runtime::Runtime; -fn run_async(fut: impl Future) -> R { - futures::executor::block_on(fut) +// tokio runtime cannot be created inside another runtime, +// so i create a separate non-'static thread not to interfere +fn run_async(fut: impl Future + Send) -> R { + thread::scope(|s| { + s.spawn(|| Runtime::new().unwrap().block_on(fut)) + .join() + .expect("async task failed") + }) } diff --git a/integration/tests/helpers/panel.rs b/integration/tests/helpers/panel.rs index 865e16e..5d3a1b7 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration/tests/helpers/panel.rs @@ -40,8 +40,8 @@ impl Panel { .as_ref(), ); match &result { - PanelResult::Ok(r) => eprintln!("+<< {r:?}"), - PanelResult::Err(e) => eprintln!("!<< {e:?}"), + PanelResult::Ok(r) => eprintln!("+<< {r:#?}"), + PanelResult::Err(e) => eprintln!("!<< {e:#?}"), } result } diff --git a/integration/tests/integration/api.rs b/integration/tests/integration/api.rs index a0c5c25..f438fcf 100644 --- a/integration/tests/integration/api.rs +++ b/integration/tests/integration/api.rs @@ -26,6 +26,7 @@ async fn test_jobs_endpoints(client_panel: &HttpClient) { .with_alias(job_alias) .build() .unwrap(); + let job_id = job.job.id; let mut fat_job = join_payload(job).unwrap(); diff --git a/integration/tests/integration/behaviour.rs b/integration/tests/integration/behaviour.rs index a5f4620..51ea208 100644 --- a/integration/tests/integration/behaviour.rs +++ b/integration/tests/integration/behaviour.rs @@ -2,7 +2,7 @@ use crate::fixtures::agent::*; use crate::helpers::{jobs::retry_with_interval, Panel}; use rstest::rstest; -use serde_json::{json, to_string}; +use serde_json::to_string; use u_lib::config::AGENT_ITERATION_INTERVAL; use u_lib::models::*; use uuid::Uuid; @@ -21,13 +21,17 @@ async fn setup_tasks() { let agents: Vec = Panel::check_output("agents read"); let agent_id = agents[0].id; let job_alias = "passwd_contents"; - let job = json!( - {"alias": job_alias, "payload": b"cat /etc/passwd", "argv": "/bin/bash {}" } - ); + let job = RawJob::builder() + .with_alias(job_alias) + .with_raw_payload(b"cat /etc/passwd".as_slice()) + .with_shell("/bin/bash {}") + .with_target_platforms("*linux*") + .build() + .unwrap(); - Panel::check_status(["jobs", "create", &to_string(&job).unwrap()]); + Panel::check_status(["jobs", "create", &to_string(&RawJob::from(job)).unwrap()]); - let cmd = format!("map create {} {}", agent_id, job_alias); + let cmd = format!("map create {agent_id} {job_alias}"); let assigned_ids: Vec = Panel::check_output(cmd); retry_with_interval(5, AGENT_ITERATION_INTERVAL, || { @@ -47,14 +51,14 @@ async fn setup_tasks() { #[tokio::test] async fn large_payload() { - let agent_id = Panel::check_output::>("agents read")[0].id; - + let agent = &Panel::check_output::>("agents read")[0]; + let agent_id = agent.id; let job_alias = "large_payload"; - let job = RawJob::builder() .with_alias(job_alias) .with_payload_path("./tests/bin/echoer") .with_shell("{} type echo") + .with_target_platforms(&agent.platform) .build() .unwrap(); diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index f702142..f5fc62f 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -14,7 +14,6 @@ diesel-derive-enum = { version = "2.0.0", features = ["postgres"], optional = tr deadpool-diesel = { workspace = true, optional = true } dotenv = "0.15.0" envy = "0.4.2" -futures = "0.3.5" guess_host_triple = "0.1.2" libc = "^0.2" lazy_static = "1.4.0" @@ -32,12 +31,15 @@ tracing-appender = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } uuid = { workspace = true, features = ["serde", "v4"] } parking_lot = "0.12.1" +bincode = "1.3.3" +sha3 = "0.10.7" [target.'cfg(unix)'.dependencies] daemonize = "0.4.1" nix = "0.17" [features] +agent = [] panel = [] server = ["dep:diesel", "dep:diesel-derive-enum", "dep:deadpool-diesel"] diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index df1f8a2..2fbadf7 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -85,29 +85,34 @@ impl HttpClient { url: impl AsRef, payload: &P, ) -> Result { + let url = url.as_ref(); let request = self .client - .post(self.base_url.join(url.as_ref()).unwrap()) + .post(self.base_url.join(url).unwrap()) .json(payload); let response = request .send() .await .context("error while sending request")?; + let is_success = match response.error_for_status_ref() { Ok(_) => Ok(()), Err(e) => Err(UError::from(e)), }; + let resp = response.text().await.context("resp")?; let result = match is_success { - Ok(_) => from_str::(&resp).map_err(|e| UError::NetError(e.to_string(), resp)), - Err(UError::NetError(err, _)) => Err(UError::NetError(err, resp)), + Ok(_) => { + from_str::(&resp).map_err(|e| UError::DeserializeError(e.to_string(), resp)) + } + Err(UError::NetError(err, url, _)) => Err(UError::NetError(err, url, resp)), _ => unreachable!(), } .map_err(From::from); - debug!("url = {}, resp = {:?}", url.as_ref(), result); + debug!("url = {}, resp = {:?}", url, result); result } @@ -164,13 +169,13 @@ impl HttpClient { } /// create and upload job - pub async fn upload_jobs(&self, payload: impl OneOrVecRef) -> Result> { + pub async fn upload_jobs(&self, payload: impl OneOrVecRef) -> Result<()> { self.req_with_payload("upload_jobs", &payload.as_vec()) .await } /// delete something - pub async fn del(&self, item: Id) -> Result { + pub async fn del(&self, item: Id) -> Result<()> { self.req(format!("del/{item}")).await } diff --git a/lib/u_lib/src/error/mod.rs b/lib/u_lib/src/error/mod.rs index 0a63f1d..82a5700 100644 --- a/lib/u_lib/src/error/mod.rs +++ b/lib/u_lib/src/error/mod.rs @@ -15,8 +15,8 @@ pub enum UError { #[error("Runtime error: {0}")] Runtime(String), - #[error("Connection error: {0}. Body: {1}")] - NetError(String, String), + #[error("Connection error: {0}; url: {1}; body: '''{2}'''")] + NetError(String, String, String), #[error("Parse error")] ParseError, @@ -42,27 +42,44 @@ pub enum UError { #[error("Panel error: {0}")] PanelError(String), - #[error("Deserialize from json error: {0}")] - DeserializeError(String), + #[error("Deserialize from json error: {0}, body: {1}")] + DeserializeError(String, String), + + #[error("{0}\n{1}")] + Contexted(Box, String), } impl From for UError { fn from(e: ReqError) -> Self { - UError::NetError(e.to_string(), String::new()) + UError::NetError( + e.to_string(), + e.url().map(|u| u.to_string()).unwrap_or_default(), + String::new(), + ) } } - +/* impl From for UError { fn from(e: serde_json::Error) -> Self { UError::DeserializeError(e.to_string()) } } - +*/ impl From for UError { fn from(e: anyhow::Error) -> Self { + let ctx = e + .chain() + .skip(1) + .map(|cause| format!("ctx: {}", cause)) + .collect::>() + .join("\n"); + match e.downcast::() { - Ok(err) => err, - Err(err) => UError::Runtime(err.to_string()), + Ok(err) => UError::Contexted(Box::new(err), ctx), + Err(err) => match err.downcast::() { + Ok(err) => UError::Contexted(Box::new(UError::FSError(err)), ctx), + Err(err) => UError::Runtime(err.to_string()), + }, } } } diff --git a/lib/u_lib/src/executor.rs b/lib/u_lib/src/executor.rs index 169002f..929011b 100644 --- a/lib/u_lib/src/executor.rs +++ b/lib/u_lib/src/executor.rs @@ -1,16 +1,18 @@ use crate::{models::AssignedJob, UResult}; -use futures::{future::BoxFuture, lock::Mutex}; use lazy_static::lazy_static; use std::collections::HashMap; use std::future::Future; +use std::pin::Pin; use tokio::{ runtime::Handle, sync::mpsc::{channel, Receiver, Sender}, + sync::Mutex, task::{spawn, spawn_blocking, JoinHandle}, }; use uuid::Uuid; pub type ExecResult = UResult; +type BoxFuture<'a, T> = Pin + Send + 'a>>; lazy_static! { static ref FUT_RESULTS: Mutex> = Mutex::new(HashMap::new()); @@ -94,10 +96,9 @@ impl Waiter { async fn init_receiver() { while let Some(fid) = FUT_CHANNEL.1.lock().await.recv().await { - if let Some(mut lock) = FUT_RESULTS.try_lock() { - if let Some(j) = lock.get_mut(&fid) { - j.completed = true; - } + let mut lock = FUT_RESULTS.lock().await; + if let Some(j) = lock.get_mut(&fid) { + j.completed = true; } } } diff --git a/lib/u_lib/src/jobs.rs b/lib/u_lib/src/jobs.rs index c81bd3d..67e3494 100644 --- a/lib/u_lib/src/jobs.rs +++ b/lib/u_lib/src/jobs.rs @@ -169,7 +169,9 @@ pub async fn run_assigned_job(job: ThinJob, ids: AssignedJobById) -> ExecResult result.retcode = retcode; } JobType::Init => { - result.set_result(&Agent::run().await); + let agent = Agent::gather().await; + debug!("gathered info from agent: {agent:?}"); + result.set_result(&agent); result.retcode = Some(0); } JobType::Service => todo!(), @@ -234,12 +236,12 @@ mod tests { #[rstest] #[case::sh_payload( "/bin/sh {}", - Some(b"echo test01 > /tmp/asd; cat /tmp/asd".as_slice()), + Some(b"echo test01 > /tmp/asd; cat /tmp/asd".as_slice()), "test01" )] #[case::python_cmd(r#"/usr/bin/python3 -c 'print("test02")'"#, None, "test02")] #[case::sh_multiline_payload( - "/{}", + "/{}", Some( br#"#!/bin/sh TMPPATH=/tmp/lol @@ -252,7 +254,7 @@ mod tests { )] #[case::standalone_binary_with_args( "/{} 'some msg as arg'", - Some(include_bytes!("../tests/fixtures/echoer").as_slice()), + Some(include_bytes!("../tests/fixtures/echoer").as_slice()), "some msg as arg" )] #[tokio::test] @@ -263,7 +265,7 @@ mod tests { ) -> TestResult { let mut job = RawJob::builder().with_shell(cmd); if let Some(p) = payload { - job = job.with_payload(p); + job = job.with_raw_payload(p); } let job = job.build().unwrap(); let result = AnonymousJobBatch::from_meta(job).wait_one().await.unwrap(); @@ -340,7 +342,7 @@ mod tests { ) -> TestResult { let mut job = RawJob::builder().with_shell(cmd); if let Some(p) = payload { - job = job.with_payload(p); + job = job.with_raw_payload(p); } let err = job.build().unwrap_err(); let err_msg = unwrap_enum!(err, UError::JobBuildError); diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index 5a81866..a05e74d 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -14,7 +14,7 @@ use self::server::*; use crate::{ config::get_self_id, conv::systime_to_string, executor::ExecResult, jobs::NamedJobBatch, - platform::Platform, types::Id, + platform, types::Id, }; #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Display)] @@ -54,10 +54,29 @@ pub struct Agent { } impl Agent { - pub fn with_id(id: Id) -> Self { + pub fn empty() -> Self { Self { - id, - ..Default::default() + alias: None, + id: get_self_id(), + hostname: String::new(), + host_info: String::new(), + is_root: false, + is_root_allowed: false, + last_active: SystemTime::now(), + platform: String::new(), + regtime: SystemTime::now(), + state: AgentState::New, + token: None, + username: String::new(), + ip_gray: None, + ip_white: None, + } + } + + pub fn with_current_platform() -> Self { + Self { + platform: platform::current_as_string(), + ..Self::empty() } } @@ -91,33 +110,7 @@ impl Agent { host_info: decoder(builder.pop("host_info")), is_root: &decoder(builder.pop("is_root")) == "0", username: decoder(builder.pop("username")), - platform: Platform::current().into_string(), - ..Default::default() - } - } - - pub async fn run() -> Agent { - Agent::gather().await - } -} - -impl Default for Agent { - fn default() -> Self { - Self { - alias: None, - id: get_self_id(), - hostname: String::new(), - host_info: String::new(), - is_root: false, - is_root_allowed: false, - last_active: SystemTime::now(), - platform: String::new(), - regtime: SystemTime::now(), - state: AgentState::New, - token: None, - username: String::new(), - ip_gray: None, - ip_white: None, + ..Self::with_current_platform() } } } diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index cefd511..bad92b4 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -5,13 +5,13 @@ use crate::conv::bytes_to_string; #[cfg(feature = "server")] use crate::models::schema::*; use crate::models::PayloadMeta; -use crate::platform::Platform; +use crate::platform; use crate::types::Id; use crate::{ufs, UError, UResult}; #[cfg(feature = "server")] use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; -use std::fs::metadata; +use std::borrow::Cow; use std::process::Command; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -28,7 +28,7 @@ pub struct JobModel { pub id: Id, pub exec_type: JobType, /// target triple - pub platform: String, + pub target_platforms: String, pub payload: Option, /// cron-like string pub schedule: Option, @@ -47,22 +47,8 @@ pub struct ThinJob { pub payload_meta: Option, } -// impl fmt::Debug for ThinJobMeta { -// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { -// f.debug_struct("ThinJobMeta") -// .field("alias", &self.alias) -// .field("argv", &self.argv) -// .field("id", &self.id.to_string()) -// .field("exec_type", &self.exec_type) -// .field("platform", &self.platform) -// .field("payload", &self.payload) -// .field("schedule", &self.schedule) -// .finish() -// } -// } - -#[derive(Serialize, Deserialize, Clone, Default)] -pub struct RawJob { +#[derive(Serialize, Deserialize, Clone)] +pub struct RawJob<'p> { #[serde(default)] pub alias: Option, @@ -78,32 +64,51 @@ pub struct RawJob { pub exec_type: JobType, /// target triple - #[serde(default = "Platform::current_as_string")] - pub platform: String, + #[serde(default = "platform::current_as_string")] + pub target_platforms: String, #[serde(default)] - pub payload: Option, + pub payload_path: Option, + + #[serde(default)] + pub raw_payload: Option>, /// cron-like string #[serde(default)] pub schedule: Option, } -impl fmt::Debug for RawJob { +impl Default for RawJob<'_> { + fn default() -> Self { + Self { + alias: None, + argv: String::new(), + id: Id::new_v4(), + exec_type: JobType::default(), + target_platforms: String::new(), + payload_path: None, + raw_payload: None, + schedule: None, + } + } +} + +impl fmt::Debug for RawJob<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RawJob") .field("alias", &self.alias) .field("argv", &self.argv) .field("id", &self.id.to_string()) .field("exec_type", &self.exec_type) - .field("platform", &self.platform) - .field("payload", &self.payload) + .field("platform", &self.target_platforms) + .field("payload_path", &self.payload_path) + .field("raw_payload", &self.raw_payload) .field("schedule", &self.schedule) .finish() } } -impl From for RawJob { +impl From for RawJob<'_> { fn from(job: ThinJob) -> Self { let ThinJob { job, payload_meta } = job; RawJob { @@ -111,49 +116,31 @@ impl From for RawJob { argv: job.argv, id: job.id, exec_type: job.exec_type, - platform: job.platform, - payload: payload_meta.map(|m| m.name), + target_platforms: job.target_platforms, + payload_path: payload_meta.map(|m| m.name), + raw_payload: None, schedule: job.schedule, } } } -impl RawJob { +impl<'p> RawJob<'p> { pub fn validated(self) -> UResult { - JobBuilder { - inner: self, - raw_payload: None, - } - .build() + JobBuilder { inner: self }.build() } pub fn from_shell(cmd: impl Into) -> UResult { Self::builder().with_shell(cmd).build() } - pub fn builder<'p>() -> JobBuilder<'p> { + pub fn builder() -> JobBuilder<'p> { JobBuilder::default() } } -// impl Default for RawJob { -// fn default() -> Self { -// Self { -// id: Id::new_v4(), -// alias: None, -// argv: String::new(), -// exec_type: JobType::Shell, -// platform: Platform::current_as_string(), -// payload: None, -// schedule: None, -// } -// } -//} - #[derive(Default)] pub struct JobBuilder<'p> { - inner: RawJob, - raw_payload: Option<&'p [u8]>, + inner: RawJob<'p>, } impl<'p> JobBuilder<'p> { @@ -163,15 +150,15 @@ impl<'p> JobBuilder<'p> { self } - pub fn with_payload(mut self, raw_payload: &'p [u8]) -> Self { - self.raw_payload = Some(raw_payload); - self.inner.payload = None; + pub fn with_raw_payload(mut self, raw_payload: impl Into>) -> Self { + self.inner.raw_payload = Some(raw_payload.into()); + self.inner.payload_path = None; self } pub fn with_payload_path(mut self, path: impl Into) -> Self { - self.inner.payload = Some(path.into()); - self.raw_payload = None; + self.inner.payload_path = Some(path.into()); + self.inner.raw_payload = None; self } @@ -185,11 +172,16 @@ impl<'p> JobBuilder<'p> { self } + pub fn with_target_platforms(mut self, platform: impl Into) -> Self { + self.inner.target_platforms = platform.into(); + self + } + pub fn build(self) -> UResult { let mut inner = self.inner; let raw_into_job = |raw: RawJob| -> UResult { - let payload_id = raw.payload.as_ref().map(|_| Id::new_v4()); + let payload_id = raw.payload_path.as_ref().map(|_| Id::new_v4()); Ok(ThinJob { job: JobModel { @@ -197,27 +189,29 @@ impl<'p> JobBuilder<'p> { argv: raw.argv, id: raw.id, exec_type: raw.exec_type, - platform: raw.platform, + target_platforms: raw.target_platforms, payload: payload_id, schedule: raw.schedule, }, payload_meta: raw - .payload - .map(|payload_path| { - Ok::<_, UError>(PayloadMeta { + .payload_path + .map(|payload_ident| { + let ufs_meta = ufs::read_meta(&payload_ident)?; + let payload_meta = PayloadMeta { id: payload_id.unwrap(), mime_type: bytes_to_string( &Command::new("file") .arg("-b") .arg("--mime-type") - .arg(&payload_path) + .arg(&ufs_meta.path) .output() .map_err(|e| UError::JobBuildError(e.to_string()))? .stdout, ), - name: payload_path.clone(), - size: metadata(payload_path).unwrap().len() as i64, - }) + name: payload_ident.clone(), + size: ufs_meta.size as i64, + }; + Ok::<_, UError>(payload_meta) }) .transpose()?, }) @@ -244,22 +238,22 @@ impl<'p> JobBuilder<'p> { return Err(empty_err.into()); } - if let Some(payload_path) = &inner.payload { - ufs::put_external(payload_path)?; + if let Some(path) = &inner.payload_path { + ufs::put_external(path)?; } - if let Some(raw) = self.raw_payload { - match inner.payload { + if let Some(raw_payload) = &inner.raw_payload { + match inner.payload_path { Some(_) => { return Err(UError::JobBuildError( "Can't use both raw payload with payload path".to_string(), )) } - None => inner.payload = Some(ufs::create(raw)?), + None => inner.payload_path = Some(ufs::create_anonymous(raw_payload)?), } } - match inner.payload.as_ref() { + match inner.payload_path.as_ref() { Some(_) => { if !inner.argv.contains("{}") { return Err(UError::JobBuildError( @@ -279,10 +273,14 @@ impl<'p> JobBuilder<'p> { } }; - if Platform::new(&inner.platform).find_valid().is_none() { + if inner.target_platforms.is_empty() { + inner.target_platforms = "*".to_string(); + } + + if !platform::is_valid_glob(&inner.target_platforms) { return Err(UError::JobBuildError(format!( - "Unknown platform {}", - inner.platform + "Unknown platform '{}'", + inner.target_platforms ))); } diff --git a/lib/u_lib/src/models/schema.rs b/lib/u_lib/src/models/schema.rs index 6e5a3ef..5fac813 100644 --- a/lib/u_lib/src/models/schema.rs +++ b/lib/u_lib/src/models/schema.rs @@ -45,7 +45,7 @@ diesel::table! { argv -> Text, id -> Uuid, exec_type -> Jobtype, - platform -> Text, + target_platforms -> Text, payload -> Nullable, schedule -> Nullable, } diff --git a/lib/u_lib/src/platform.rs b/lib/u_lib/src/platform.rs index 529f41a..cf6e7e9 100644 --- a/lib/u_lib/src/platform.rs +++ b/lib/u_lib/src/platform.rs @@ -11,18 +11,10 @@ impl Platform { Platform(p.into()) } - pub fn current() -> Platform { - Self(guess_host_triple().unwrap().to_string()) - } - - pub fn current_as_string() -> String { - Self::current().into_string() - } - pub fn matches(&self, pf: impl AsRef) -> bool { // this finder needs a full triple, so when the .0 is empty, return true // this is fucked up tbh - let Some(platform_to_match_against) = self.find_valid() else { + let Some(platform_to_match_against) = LibPlatform::find(&self.0) else { return self.0.is_empty() }; @@ -32,11 +24,19 @@ impl Platform { } } - pub fn find_valid(&self) -> Option<&'static LibPlatform> { - LibPlatform::find(&self.0) - } - pub fn into_string(self) -> String { self.0 } } + +pub fn current() -> Platform { + Platform(guess_host_triple().unwrap().to_string()) +} + +pub fn current_as_string() -> String { + current().into_string() +} + +pub fn is_valid_glob(platform_req: impl AsRef) -> bool { + PlatformReq::from_str(platform_req.as_ref()).is_ok() +} diff --git a/lib/u_lib/src/ufs/error.rs b/lib/u_lib/src/ufs/error.rs index 28f985a..90baac9 100644 --- a/lib/u_lib/src/ufs/error.rs +++ b/lib/u_lib/src/ufs/error.rs @@ -35,3 +35,18 @@ impl From for Error { } } } + +impl From for Error { + fn from(e: anyhow::Error) -> Self { + let err = e + .chain() + .map(|cause| format!("ctx: {}", cause)) + .collect::>() + .join("\n"); + + Error { + err, + path: String::new(), + } + } +} diff --git a/lib/u_lib/src/ufs/index.rs b/lib/u_lib/src/ufs/index.rs new file mode 100644 index 0000000..9a58ef6 --- /dev/null +++ b/lib/u_lib/src/ufs/index.rs @@ -0,0 +1,101 @@ +use super::{Error, FileMeta}; +use once_cell::sync::Lazy; +use parking_lot::Mutex; +use std::collections::HashMap; +use std::env::temp_dir; +use std::fs; +use std::path::PathBuf; + +// index format: given_name -> payload_meta +type Index = HashMap; + +static IDX_FILE_NAME: Lazy = Lazy::new(|| temp_dir().join(".i")); +static INDEX: Lazy> = Lazy::new(|| { + let idx_name = &*IDX_FILE_NAME; + let deserialized_idx = fs::read(idx_name) + .map_err(|e| Error::new(e, idx_name)) + .and_then(|raw_idx| { + bincode::deserialize::(&raw_idx).map_err(|e| Error::new(e, idx_name)) + }); + + let idx = match deserialized_idx { + Ok(idx) => idx, + Err(e) => { + error!("index loading failed: {e}"); + HashMap::new() + } + }; + Mutex::new(idx) +}); + +mod sync { + use super::{Index, IDX_FILE_NAME}; + use std::fs; + + pub(super) fn deleted(index: &mut Index) { + let files_to_delete: Vec = index + .iter() + .filter_map(|(name, meta)| { + if meta.path.exists() { + None + } else { + Some(name.to_string()) + } + }) + .collect(); + + files_to_delete.into_iter().for_each(|f| { + index.remove(&f); + }); + } + + pub(super) fn index2fs(index: &Index) { + let serialized_idx = bincode::serialize(index).expect("broken index"); + if let Err(e) = fs::write(&*IDX_FILE_NAME, serialized_idx) { + error!("index dumping failed: {e}"); + } + } +} + +pub fn get(name: impl AsRef) -> Option { + let mut index = INDEX.lock(); + + sync::deleted(&mut index); + + index.get(name.as_ref()).cloned() +} + +pub fn get_by_hash(hash: impl AsRef<[u8]>) -> Option<(String, FileMeta)> { + let mut index = INDEX.lock(); + + sync::deleted(&mut index); + + index + .iter() + .find(|(_name, meta)| meta.hash == hash.as_ref()) + .map(|(n, m)| (n.to_owned(), m.clone())) +} + +pub fn insert(name: impl Into, meta: FileMeta) { + let mut index = INDEX.lock(); + + sync::deleted(&mut index); + + index.insert(name.into(), meta); + + #[cfg(any(feature = "panel", feature = "server"))] + sync::index2fs(&mut index); +} + +pub fn remove(name: impl AsRef) -> Option { + let mut index = INDEX.lock(); + + sync::deleted(&mut index); + + let result = index.remove(name.as_ref()); + + #[cfg(any(feature = "panel", feature = "server"))] + sync::index2fs(&mut index); + + result +} diff --git a/lib/u_lib/src/ufs/mod.rs b/lib/u_lib/src/ufs/mod.rs index be35999..4ab8c40 100644 --- a/lib/u_lib/src/ufs/mod.rs +++ b/lib/u_lib/src/ufs/mod.rs @@ -1,9 +1,8 @@ // This module is aiming to store obfuscated payloads, get them by name, // rename, update, delete or prepare to execute via memfd_create (unix) -use once_cell::sync::Lazy; -use parking_lot::RwLock; -use std::collections::HashMap; +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; use std::env::temp_dir; use std::ffi::{CString, OsString}; use std::fs::{self, File}; @@ -11,116 +10,120 @@ use std::path::{Path, PathBuf}; use uuid::Uuid; mod error; +mod index; pub use error::Error; -// INDEX format: given_name -> payload_meta -static INDEX: Lazy>> = Lazy::new(|| RwLock::new(HashMap::new())); +const OBFUSCATE: bool = cfg!(feature = "agent"); -struct FileMeta { - path: PathBuf, - obfuscated: bool, +#[derive(Clone, Deserialize, Serialize)] +pub struct FileMeta { extension: Option, external: bool, + hash: Vec, + pub path: PathBuf, + pub size: u64, } -/// Remove deleted files from index -pub fn sync_index() { - let mut index = INDEX.write(); - - let files_to_delete: Vec = index - .iter() - .filter_map(|(name, meta)| { - if meta.path.exists() { - None - } else { - Some(name.to_string()) - } +impl FileMeta { + pub fn new( + full_path: impl Into, + hash: Vec, + external: bool, + ) -> Result { + let full_path: PathBuf = full_path.into(); + let extension = full_path.extension().map(ToOwned::to_owned); + let size = fs::metadata(&full_path)?.len(); + + Ok(FileMeta { + path: full_path, + extension, + external, + hash, + size, }) - .collect(); - - files_to_delete.into_iter().for_each(|f| { - index.remove(&f); - }); + } } pub fn in_index(name: impl AsRef) -> bool { - sync_index(); - - INDEX.read().get(name.as_ref()).is_some() + read_meta(name).is_ok() } -pub fn read(name: impl AsRef) -> Result, Error> { - sync_index(); - - let name = name.as_ref(); - let index = INDEX.read(); - let meta = index.get(name).ok_or_else(|| Error::not_found(name))?; +#[inline] +pub fn read_meta(name: impl AsRef) -> Result { + index::get(&name) + .ok_or_else(|| Error::not_found(name.as_ref())) + .context("meta") +} - fs::read(&meta.path).map_err(|e| Error::new(e, name)) +pub fn read(name: impl AsRef) -> Result> { + let meta = read_meta(&name).context("read_meta")?; + fs::read(&meta.path) + .map_err(|e| Error::new(e, name.as_ref())) + .context("read") } -pub fn create(data: impl AsRef<[u8]>) -> Result { +pub fn create_anonymous(data: impl AsRef<[u8]>) -> Result { + if let Some((name, _)) = index::get_by_hash(hash_data(&data)) { + return Ok(name); + } + let name = Uuid::new_v4().simple().to_string(); - put(&name, data)?; + put(&name, data).context("cr_anon")?; Ok(name) } /// Create new file and add to index -pub fn put(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<(), Error> { - sync_index(); - +pub fn put(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<()> { let name = name.as_ref(); - let obfuscate = !cfg!(feature = "server") && !cfg!(feature = "panel"); + let data_hash = hash_data(&data); if in_index(&name) { - return Err(Error::already_exists(&name)); + return Err(Error::already_exists(&name)).context("put_exists"); } - let path = { - let exec_name = if obfuscate { - PathBuf::from(Uuid::new_v4().simple().to_string()) - } else { - PathBuf::from(name) - }; - - let mut path = temp_dir(); - path.push(exec_name); - path + let path = match index::get_by_hash(&data_hash) { + Some((_, meta)) => meta.path, + None => { + let path = { + let exec_name = if OBFUSCATE { + PathBuf::from(Uuid::new_v4().simple().to_string()) + } else { + PathBuf::from(name) + }; + + let mut path = temp_dir(); + path.push(exec_name); + path + }; + + fs::write(&path, &data) + .map_err(|e| Error::new(e, name)) + .context("put_write")?; + path + } }; - - let extension = path.file_stem().map(ToOwned::to_owned); - - fs::write(&path, data).map_err(|e| Error::new(e, name))?; - - INDEX.write().insert( - name.to_string(), - FileMeta { - path, - obfuscated: obfuscate, - extension, - external: false, - }, + index::insert( + name, + FileMeta::new(path, data_hash, false).context("put_insert")?, ); Ok(()) } -pub fn remove(name: impl AsRef) -> Result<(), Error> { - sync_index(); - +pub fn remove(name: impl AsRef) -> Result<()> { let name = name.as_ref(); - - match INDEX.write().remove(name) { - Some(value) => fs::remove_file(value.path).map_err(|e| Error::new(e, name)), - None => Ok(()), + match index::remove(name) { + Some(value) if !value.external => fs::remove_file(value.path) + .map_err(|e| Error::new(e, name)) + .context("remove"), + _ => Ok(()), } } -pub fn rename(old_name: impl AsRef, new_name: impl AsRef) -> Result<(), Error> { - sync_index(); - +// todo: don't rename external files +pub fn rename(old_name: impl AsRef, new_name: impl AsRef) -> Result<()> { let old_name = old_name.as_ref(); let new_name = new_name.as_ref(); @@ -129,67 +132,63 @@ pub fn rename(old_name: impl AsRef, new_name: impl AsRef) -> Result<() } if !in_index(old_name) { - return Err(Error::not_found(old_name)); + return Err(Error::not_found(old_name)).context("rename"); } if in_index(new_name) { - return Err(Error::already_exists(new_name)); + return Err(Error::already_exists(new_name)).context("rename"); } - let mut value = INDEX.write().remove(old_name).unwrap(); + let mut value = index::remove(old_name).unwrap(); - if !value.obfuscated { + if !OBFUSCATE { let old_path = value.path.clone(); value.path.pop(); value.path.push(new_name); - fs::rename(old_path, &value.path).map_err(|e| Error::new(e, &value.path))?; + fs::rename(old_path, &value.path) + .map_err(|e| Error::new(e, &value.path)) + .context("rename")?; } - INDEX.write().insert(new_name.to_string(), value); + index::insert(new_name, value); Ok(()) } -pub fn update_payload_data(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<(), Error> { - sync_index(); - +pub fn update_payload_data(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<()> { let name = name.as_ref(); - let external = INDEX.read().get(name).map(|v| v.external).unwrap_or(false); + let external = index::get(name).map(|v| v.external).unwrap_or(false); if external { - INDEX.write().remove(name); + index::remove(name); } else if in_index(&name) { - remove(&name)?; + remove(&name).context("upd")?; } - put(name, data) + put(name, data).context("upd") } /// Add an existing file to index -pub fn put_external(path: impl AsRef) -> Result<(), Error> { - sync_index(); - +pub fn put_external(path: impl AsRef) -> Result<()> { let path = path.as_ref(); let path_str = path.as_os_str().to_string_lossy().to_string(); - if !path.exists() || path.is_dir() { - return Err(Error::not_found(path)); + if in_index(&path_str) { + return Ok(()); } - if in_index(&path_str) { - return Err(Error::already_exists(&path)); + if !path.exists() || path.is_dir() { + return Err(Error::not_found(path)).context("ext1"); } - INDEX.write().insert( + let file_data = fs::read(&path_str).unwrap(); + let data_hash = hash_data(&file_data); + + index::insert( path_str, - FileMeta { - path: path.to_owned(), - obfuscated: false, - extension: path.file_stem().map(ToOwned::to_owned), - external: true, - }, + FileMeta::new(path, data_hash, true).context("ext2")?, ); Ok(()) @@ -197,7 +196,7 @@ pub fn put_external(path: impl AsRef) -> Result<(), Error> { /// Prepare executable file: unpack, decipher if needed and send under memfd #[cfg(unix)] -pub fn prepare_executable(name: impl AsRef) -> Result<(File, String), Error> { +pub fn prepare_executable(name: impl AsRef) -> Result<(File, String)> { use libc::getpid; use nix::sys::memfd::*; use std::io::{Read, Write}; @@ -206,12 +205,8 @@ pub fn prepare_executable(name: impl AsRef) -> Result<(File, String), Error const FAKE_EXEC_NAME: &str = "/usr/sbin/lvmetad"; const BUFFER_LEN: usize = 4096; - sync_index(); - let mut buffer: [u8; BUFFER_LEN] = [0; BUFFER_LEN]; - let name = name.as_ref(); - let index = INDEX.read(); - let payload_meta = index.get(name).ok_or_else(|| Error::not_found(name))?; + let payload_meta = read_meta(name).context("prep")?; let fd = memfd_create( CString::new(FAKE_EXEC_NAME).unwrap().as_c_str(), @@ -235,15 +230,16 @@ pub fn prepare_executable(name: impl AsRef) -> Result<(File, String), Error let payload_path = format!("/proc/{}/fd/{}", unsafe { getpid() }, fd); Ok((payload_dest, payload_path)) } - Err(e) => Err(Error::new(e, FAKE_EXEC_NAME)), + Err(e) => Err(Error::new(e, FAKE_EXEC_NAME)).context("prep"), } } #[cfg(windows)] -pub fn prepare_executable(name: impl AsRef) -> Result<(File, String), Error> { +pub fn prepare_executable(name: impl AsRef) -> Result<(File, String)> { todo!() } +/* pub fn cleanup() { let index = INDEX.read(); @@ -251,3 +247,12 @@ pub fn cleanup() { fs::remove_file(&f.path).ok(); }); } +*/ + +fn hash_data(data: impl AsRef<[u8]>) -> Vec { + use sha3::{Digest, Sha3_256}; + + let mut hasher = Sha3_256::new(); + hasher.update(data); + hasher.finalize().to_vec() +} diff --git a/migrations/2020-10-24-111622_create_all/up.sql b/migrations/2020-10-24-111622_create_all/up.sql index 8e0af5a..7ddd216 100644 --- a/migrations/2020-10-24-111622_create_all/up.sql +++ b/migrations/2020-10-24-111622_create_all/up.sql @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS jobs ( argv TEXT NOT NULL, id UUID NOT NULL, exec_type JobType NOT NULL DEFAULT 'shell', - platform TEXT NOT NULL, + target_platforms TEXT NOT NULL, payload UUID, schedule TEXT, -- 2.36.2 From f5d2190fc4e85920e722858b12cfcc6f35fa6dd2 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Thu, 20 Apr 2023 00:13:20 +0300 Subject: [PATCH 04/10] fix last integrations test --- Cargo.lock | 31 +++++-------- bin/u_server/src/db.rs | 20 +++++++-- bin/u_server/src/handlers.rs | 65 +++++++++++++++++++-------- bin/u_server/src/u_server.rs | 2 +- integration/tests/integration/api.rs | 4 +- lib/u_lib/Cargo.toml | 2 +- lib/u_lib/src/api.rs | 18 ++++++++ lib/u_lib/src/error/mod.rs | 9 +--- lib/u_lib/src/jobs.rs | 8 +++- lib/u_lib/src/models/jobs/assigned.rs | 17 +++++++ lib/u_lib/src/ufs/error.rs | 2 + lib/u_lib/src/ufs/mod.rs | 31 ++++++++++++- 12 files changed, 153 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1cd2c46..08a23ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -345,12 +345,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "boxfnonce" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5988cb1d626264ac94100be357308f29ff7cbdd3b36bda27f450a4ee3f713426" - [[package]] name = "brotli" version = "3.3.4" @@ -610,11 +604,10 @@ dependencies = [ [[package]] name = "daemonize" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70c24513e34f53b640819f0ac9f705b673fcf4006d7aab8778bee72ebfc89815" +checksum = "ab8bfdaacb3c887a54d41bdf48d3af8873b3f5566469f8ba21b92057509f116e" dependencies = [ - "boxfnonce", "libc", ] @@ -675,9 +668,9 @@ dependencies = [ [[package]] name = "diesel" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4391a22b19c916e50bec4d6140f29bdda3e3bb187223fe6e3ea0b6e4d1021c04" +checksum = "72eb77396836a4505da85bae0712fa324b74acfe1876d7c2f7e694ef3d0ee373" dependencies = [ "bitflags", "byteorder", @@ -980,9 +973,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f" +checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" dependencies = [ "bytes", "fnv", @@ -1340,9 +1333,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "9b085a4f2cde5781fc4b1717f2e86c62f5cda49de7ba99a7c2eae02b61c9064c" [[package]] name = "local-channel" @@ -1710,9 +1703,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "pq-sys" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b845d6d8ec554f972a2c5298aad68953fd64e7441e846075450b44656a016d1" +checksum = "31c0052426df997c0cbd30789eb44ca097e3541717a7b8fa36b1c464ee7edebd" dependencies = [ "vcpkg", ] @@ -2010,9 +2003,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.11" +version = "0.37.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +checksum = "722529a737f5a942fdbac3a46cee213053196737c5eaa3386d52e85b786f2659" dependencies = [ "bitflags", "errno 0.3.1", diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index 3de24e6..053bf26 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -1,10 +1,12 @@ use crate::error::Error; use diesel::{pg::PgConnection, prelude::*, result::Error as DslError, Connection}; use std::mem::drop; -use u_lib::db::PgAsyncPool; -use u_lib::models::{schema, Agent, AssignedJob, JobModel, JobState, PayloadMeta, ThinJob}; -use u_lib::platform::Platform; -use u_lib::types::Id; +use u_lib::{ + db::PgAsyncPool, + models::{schema, Agent, AssignedJob, JobModel, JobState, PayloadMeta, ThinJob}, + platform::Platform, + types::Id, +}; type Result = std::result::Result; @@ -249,6 +251,16 @@ impl UDB<'_> { .map_err(with_err_ctx("Can't delete agents")) } + pub fn del_payloads(&mut self, ids: &[Id]) -> Result<()> { + use schema::payloads; + + diesel::delete(payloads::table) + .filter(payloads::id.eq_any(ids)) + .execute(self.conn) + .map(drop) + .map_err(with_err_ctx("Can't delete payloads")) + } + pub fn upsert_agent(&mut self, agent: &Agent) -> Result<()> { use schema::agents; diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index bc82823..0fd739f 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -3,9 +3,9 @@ use std::sync::Arc; use crate::db::{PgRepo, UDB}; use crate::error::Error; use serde::Deserialize; -use u_lib::jobs::join_payload; use u_lib::{ - jobs::split_payload, + api::retypes, + jobs::{join_payload, split_payload}, messaging::{AsMsg, Reportable}, misc::OneOrVec, models::*, @@ -24,7 +24,7 @@ pub struct GetJobQuery { pub struct Endpoints; impl Endpoints { - pub async fn get_agents(repo: Arc, id: Option) -> EndpResult> { + pub async fn get_agents(repo: Arc, id: Option) -> EndpResult { repo.interact(move |mut db| { Ok(match id { Some(id) => { @@ -41,7 +41,11 @@ impl Endpoints { .map_err(From::from) } - pub async fn get_job(repo: Arc, id: Id, params: GetJobQuery) -> EndpResult { + pub async fn get_job( + repo: Arc, + id: Id, + params: GetJobQuery, + ) -> EndpResult { let Some(job) = repo.interact(move |mut db| db.get_job(id)).await? else { return Err(not_found()) }; @@ -60,19 +64,25 @@ impl Endpoints { Ok(join_payload(job).map_err(Error::from)?) } - pub async fn get_jobs(repo: Arc) -> EndpResult> { + pub async fn get_jobs(repo: Arc) -> EndpResult { repo.interact(move |mut db| db.get_jobs()) .await .map_err(From::from) } - pub async fn get_agent_jobs(repo: Arc, id: Option) -> EndpResult> { + pub async fn get_agent_jobs( + repo: Arc, + id: Option, + ) -> EndpResult { repo.interact(move |mut db| db.get_exact_jobs(id, false)) .await .map_err(From::from) } - pub async fn get_personal_jobs(repo: Arc, id: Id) -> EndpResult> { + pub async fn get_personal_jobs( + repo: Arc, + id: Id, + ) -> EndpResult { repo.transaction(move |mut db| { let agent = db.get_agent(id)?; match agent { @@ -100,13 +110,19 @@ impl Endpoints { db.update_job_status(job.id, JobState::Running)?; } - Ok(assigned_jobs) + Ok(assigned_jobs + .into_iter() + .map(|j| AssignedJobById::from(&j)) + .collect()) }) .await .map_err(From::from) } - pub async fn upload_jobs(repo: Arc, msg: Vec) -> EndpResult<()> { + pub async fn upload_jobs( + repo: Arc, + msg: Vec, + ) -> EndpResult { let jobs = msg .into_iter() .map(|meta| Ok(split_payload(meta)?)) @@ -119,10 +135,15 @@ impl Endpoints { pub async fn del(repo: Arc, id: Id) -> EndpResult<()> { repo.transaction(move |mut db| { - [UDB::del_agents, UDB::del_jobs, UDB::del_results] - .iter() - .map(|f| f(&mut db, &[id])) - .collect::>() + [ + UDB::del_agents, + UDB::del_jobs, + UDB::del_results, + UDB::del_payloads, + ] + .iter() + .map(|f| f(&mut db, &[id])) + .collect::>() }) .await .map_err(From::from) @@ -132,7 +153,7 @@ impl Endpoints { repo: Arc, agent_id: Id, job_idents: Vec, - ) -> EndpResult> { + ) -> EndpResult { repo.transaction(move |mut db| { let assigned_job_idents = job_idents .into_iter() @@ -162,7 +183,7 @@ impl Endpoints { repo: Arc, msg: Data, agent_id: Id, - ) -> EndpResult<()> { + ) -> EndpResult { repo.transaction(move |mut db| { for entry in msg.into_vec() { match entry { @@ -214,17 +235,23 @@ impl Endpoints { .map_err(From::from) } - pub async fn update_agent(repo: Arc, agent: Agent) -> EndpResult<()> { + pub async fn update_agent(repo: Arc, agent: Agent) -> EndpResult { repo.interact(move |mut db| db.upsert_agent(&agent)).await?; Ok(()) } - pub async fn update_job(repo: Arc, job: JobModel) -> EndpResult<()> { - repo.interact(move |mut db| db.update_job(&job)).await?; + pub async fn update_job(repo: Arc, job: FatJob) -> EndpResult { + let thin_job = split_payload(job).map_err(Error::from)?; + + repo.interact(move |mut db| db.update_job(&thin_job.job)) + .await?; Ok(()) } - pub async fn update_assigned_job(repo: Arc, assigned: AssignedJob) -> EndpResult<()> { + pub async fn update_assigned_job( + repo: Arc, + assigned: AssignedJob, + ) -> EndpResult { repo.interact(move |mut db| db.update_result(&assigned)) .await?; Ok(()) diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index ee3f440..3ad73a1 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -108,7 +108,7 @@ pub fn init_endpoints( let update_job = path("update_job") .and(with_db.clone()) - .and(body::json::()) + .and(body::json::()) .and_then(Endpoints::update_job) .map(ok); diff --git a/integration/tests/integration/api.rs b/integration/tests/integration/api.rs index f438fcf..17d42cd 100644 --- a/integration/tests/integration/api.rs +++ b/integration/tests/integration/api.rs @@ -42,5 +42,7 @@ async fn test_jobs_endpoints(client_panel: &HttpClient) { assert_eq!(fat_job, fetched_job); client_panel.del(job_id).await.unwrap(); - client_panel.get_job(job_id, false).await.unwrap(); // should fail with 404 + + let not_found_err = client_panel.get_job(job_id, false).await.unwrap_err(); + assert!(not_found_err.to_string().contains("404 Not Found")) } diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index f5fc62f..8cf77d7 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -35,7 +35,7 @@ bincode = "1.3.3" sha3 = "0.10.7" [target.'cfg(unix)'.dependencies] -daemonize = "0.4.1" +daemonize = "0.5" nix = "0.17" [features] diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 2fbadf7..d656832 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -20,6 +20,24 @@ use crate::{ const AGENT_IDENTITY: &[u8] = include_bytes!("../../../certs/alice.p12"); const ROOT_CA_CERT: &[u8] = include_bytes!("../../../certs/ca.crt"); +pub mod retypes { + use super::*; + + pub type GetPersonalJobs = Vec; + pub type Report = (); + pub type GetJob = FatJob; + pub type GetJobs = Vec; + pub type GetAgents = Vec; + pub type UpdateAgent = (); + pub type UpdateJob = (); + pub type UpdateResult = (); + pub type UploadJobs = (); + pub type Del = (); + pub type SetJobs = Vec; + pub type GetAgentJobs = Vec; + pub type Ping = (); +} + #[derive(Clone, Debug)] pub struct HttpClient { base_url: Url, diff --git a/lib/u_lib/src/error/mod.rs b/lib/u_lib/src/error/mod.rs index 82a5700..b9678a2 100644 --- a/lib/u_lib/src/error/mod.rs +++ b/lib/u_lib/src/error/mod.rs @@ -58,17 +58,12 @@ impl From for UError { ) } } -/* -impl From for UError { - fn from(e: serde_json::Error) -> Self { - UError::DeserializeError(e.to_string()) - } -} -*/ + impl From for UError { fn from(e: anyhow::Error) -> Self { let ctx = e .chain() + .rev() .skip(1) .map(|cause| format!("ctx: {}", cause)) .collect::>() diff --git a/lib/u_lib/src/jobs.rs b/lib/u_lib/src/jobs.rs index 67e3494..94c2c0f 100644 --- a/lib/u_lib/src/jobs.rs +++ b/lib/u_lib/src/jobs.rs @@ -189,8 +189,12 @@ pub fn split_payload(job: FatJob) -> Result { } = job; if let Some(meta) = &payload_meta { - if !ufs::in_index(&meta.name) { - ufs::put(&meta.name, payload_data.unwrap())?; + if let Some(data) = payload_data { + if ufs::in_index(&meta.name) { + ufs::edit(&meta.name, data)?; + } else { + ufs::put(&meta.name, data)?; + } } } diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index 041451c..946fc4b 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -79,6 +79,23 @@ impl From<(&JobModel, AssignedJobById)> for AssignedJob { } } +impl From<&AssignedJob> for AssignedJobById { + fn from(j: &AssignedJob) -> Self { + let &AssignedJob { + agent_id, + id, + job_id, + .. + } = j; + + AssignedJobById { + agent_id, + id, + job_id, + } + } +} + impl Default for AssignedJobById { fn default() -> Self { Self { diff --git a/lib/u_lib/src/ufs/error.rs b/lib/u_lib/src/ufs/error.rs index 90baac9..3a5cbd8 100644 --- a/lib/u_lib/src/ufs/error.rs +++ b/lib/u_lib/src/ufs/error.rs @@ -40,6 +40,8 @@ impl From for Error { fn from(e: anyhow::Error) -> Self { let err = e .chain() + .rev() + .skip(1) .map(|cause| format!("ctx: {}", cause)) .collect::>() .join("\n"); diff --git a/lib/u_lib/src/ufs/mod.rs b/lib/u_lib/src/ufs/mod.rs index 4ab8c40..51a8ff6 100644 --- a/lib/u_lib/src/ufs/mod.rs +++ b/lib/u_lib/src/ufs/mod.rs @@ -44,6 +44,8 @@ impl FileMeta { } } +/// Check if file exists in index. +/// File may present in fs but not in index, thus fn will return false. pub fn in_index(name: impl AsRef) -> bool { read_meta(name).is_ok() } @@ -55,6 +57,7 @@ pub fn read_meta(name: impl AsRef) -> Result { .context("meta") } +/// Read file by index name pub fn read(name: impl AsRef) -> Result> { let meta = read_meta(&name).context("read_meta")?; fs::read(&meta.path) @@ -62,6 +65,7 @@ pub fn read(name: impl AsRef) -> Result> { .context("read") } +/// Create file with generated name pub fn create_anonymous(data: impl AsRef<[u8]>) -> Result { if let Some((name, _)) = index::get_by_hash(hash_data(&data)) { return Ok(name); @@ -74,7 +78,10 @@ pub fn create_anonymous(data: impl AsRef<[u8]>) -> Result { Ok(name) } -/// Create new file and add to index +/// Create new file and add to index. +/// +/// If index already contains a file with the same contents, it doesn't duplicate it, +/// but adds an symlink. pub fn put(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<()> { let name = name.as_ref(); let data_hash = hash_data(&data); @@ -112,6 +119,26 @@ pub fn put(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<()> { Ok(()) } +pub fn edit(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<()> { + let meta = read_meta(&name).context("edit_meta")?; + let data_hash = hash_data(&data); + + if meta.hash == data_hash { + return Ok(()); + } + + fs::write(&meta.path, &data) + .map_err(|e| Error::new(e, &meta.path)) + .context("edit_write")?; + + let new_meta = FileMeta::new(meta.path, data_hash, meta.external).context("edit_nmeta")?; + + index::remove(&name); + index::insert(name.as_ref(), new_meta); + + Ok(()) +} + pub fn remove(name: impl AsRef) -> Result<()> { let name = name.as_ref(); match index::remove(name) { @@ -167,7 +194,7 @@ pub fn update_payload_data(name: impl AsRef, data: impl AsRef<[u8]>) -> Res remove(&name).context("upd")?; } - put(name, data).context("upd") + put(name, data).context("upd_put") } /// Add an existing file to index -- 2.36.2 From a38e3a1561a8fa44c95161e14043a8a299fb9a63 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Wed, 26 Apr 2023 13:48:59 +0300 Subject: [PATCH 05/10] some progress - add payloads page - refactor frontend to more nice arch - rename integration to integration-tests --- Cargo.lock | 84 +++++++---- Cargo.toml | 2 +- Makefile.toml | 8 +- bin/u_agent/src/lib.rs | 12 +- bin/u_panel/src/argparse.rs | 46 +++--- .../src/gui/fe/src/app/app-routing.module.ts | 7 +- .../src/gui/fe/src/app/app.component.html | 3 +- .../src/gui/fe/src/app/app.component.ts | 5 +- bin/u_panel/src/gui/fe/src/app/app.module.ts | 15 +- .../agent-info-dialog.component.html} | 0 .../agent-info-dialog.component.ts} | 6 +- .../assign-job-dialog.component.html} | 0 .../assign-job-dialog.component.ts | 28 ++++ .../dialogs/base-info-dialog.component.less} | 0 .../fe/src/app/components/dialogs/index.ts | 5 + .../job-info-dialog.component.html | 52 +++++++ .../job-info-dialog.component.ts | 46 ++++++ .../payload-info-dialog.component.html | 24 +++ .../payload-info-dialog.component.ts | 14 ++ .../result-info-dialog.component.html} | 0 .../result-info-dialog.component.ts} | 6 +- .../global-error/global-error.component.html | 0 .../global-error/global-error.component.less | 0 .../global-error/global-error.component.ts | 34 +++++ .../agent-table/agent-table.component.html} | 2 +- .../agent-table/agent-table.component.ts | 42 ++++++ .../base-table/base-table.component.less} | 0 .../tables/base-table/base-table.component.ts | 58 ++++++++ .../gui/fe/src/app/components/tables/index.ts | 5 + .../job-table/job-table.component.html} | 4 +- .../tables/job-table/job-table.component.ts | 50 +++++++ .../payload-table.component.html | 60 ++++++++ .../payload-table/payload-table.component.ts | 30 ++++ .../result-table/result-table.component.html} | 4 +- .../result-table/result-table.component.ts | 36 +++++ .../src/gui/fe/src/app/core/models/index.ts | 14 -- .../gui/fe/src/app/core/models/job.model.ts | 11 -- .../fe/src/app/core/services/api.service.ts | 53 ------- .../fe/src/app/core/tables/agent.component.ts | 52 ------- .../tables/dialogs/assign_job.component.ts | 33 ----- .../fe/src/app/core/tables/dialogs/index.ts | 4 - .../core/tables/dialogs/job-info-dialog.html | 44 ------ .../core/tables/dialogs/job_info.component.ts | 30 ---- .../src/gui/fe/src/app/core/tables/index.ts | 3 - .../fe/src/app/core/tables/job.component.ts | 59 -------- .../src/app/core/tables/result.component.ts | 41 ------ .../fe/src/app/core/tables/table.component.ts | 84 ----------- .../src/gui/fe/src/app/{core => }/index.ts | 0 .../src/app/{core => }/models/agent.model.ts | 4 +- .../src/gui/fe/src/app/models/index.ts | 24 +++ .../src/gui/fe/src/app/models/job.model.ts | 16 ++ .../gui/fe/src/app/models/payload.model.ts | 17 +++ .../src/app/{core => }/models/result.model.ts | 4 +- .../gui/fe/src/app/services/api.service.ts | 138 ++++++++++++++++++ .../gui/fe/src/app/services/error.service.ts | 17 +++ .../fe/src/app/{core => }/services/index.ts | 0 .../src/gui/fe/src/app/{core => }/utils.ts | 0 bin/u_panel/src/main.rs | 6 +- bin/u_server/Cargo.toml | 1 + bin/u_server/src/db.rs | 36 ++++- bin/u_server/src/error.rs | 2 +- bin/u_server/src/handlers.rs | 110 ++++++++++---- bin/u_server/src/u_server.rs | 69 +++++---- {integration => integration-tests}/Cargo.lock | 0 {integration => integration-tests}/Cargo.toml | 5 +- .../docker-compose.yml | 25 ++-- {integration => integration-tests}/docker.py | 2 +- .../integration_tests.py | 20 +-- .../integration_tests.sh | 2 +- .../src/main.rs | 0 .../tests/fixtures/agent.rs | 7 +- .../tests/fixtures/connections.rs | 0 .../tests/fixtures/env.rs | 0 .../tests/fixtures/mod.rs | 0 .../tests/helpers/jobs.rs | 0 .../tests/helpers/mod.rs | 0 .../tests/helpers/panel.rs | 14 +- .../tests/integration_tests}/api.rs | 28 ++-- .../tests/integration_tests}/behaviour.rs | 4 +- .../tests/integration_tests}/connection.rs | 0 .../tests/integration_tests}/mod.rs | 0 integration-tests/tests/lib.rs | 14 ++ {integration => integration-tests}/utils.py | 0 integration/tests/lib.rs | 6 - lib/u_lib/src/api.rs | 93 ++++++++---- lib/u_lib/src/cache.rs | 4 +- lib/u_lib/src/combined_result.rs | 14 +- lib/u_lib/src/error/mod.rs | 22 ++- lib/u_lib/src/jobs.rs | 108 +++++++------- lib/u_lib/src/logging.rs | 23 ++- lib/u_lib/src/messaging.rs | 12 +- lib/u_lib/src/misc.rs | 32 ---- lib/u_lib/src/models/jobs/meta.rs | 61 +++----- lib/u_lib/src/models/mod.rs | 21 +++ lib/u_lib/src/models/payload.rs | 33 ++++- lib/u_lib/src/ufs/mod.rs | 16 +- 96 files changed, 1307 insertions(+), 819 deletions(-) rename bin/u_panel/src/gui/fe/src/app/{core/tables/dialogs/agent-info-dialog.html => components/dialogs/agent-info-dialog/agent-info-dialog.component.html} (100%) rename bin/u_panel/src/gui/fe/src/app/{core/tables/dialogs/agent_info.component.ts => components/dialogs/agent-info-dialog/agent-info-dialog.component.ts} (73%) rename bin/u_panel/src/gui/fe/src/app/{core/tables/dialogs/assign-job-dialog.html => components/dialogs/assign-job-dialog/assign-job-dialog.component.html} (100%) create mode 100644 bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts rename bin/u_panel/src/gui/fe/src/app/{core/tables/dialogs/info-dialog.component.less => components/dialogs/base-info-dialog.component.less} (100%) create mode 100644 bin/u_panel/src/gui/fe/src/app/components/dialogs/index.ts create mode 100644 bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html create mode 100644 bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts create mode 100644 bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html create mode 100644 bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts rename bin/u_panel/src/gui/fe/src/app/{core/tables/dialogs/result-info-dialog.html => components/dialogs/result-info-dialog/result-info-dialog.component.html} (100%) rename bin/u_panel/src/gui/fe/src/app/{core/tables/dialogs/result_info.component.ts => components/dialogs/result-info-dialog/result-info-dialog.component.ts} (74%) create mode 100644 bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.html create mode 100644 bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.less create mode 100644 bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.ts rename bin/u_panel/src/gui/fe/src/app/{core/tables/agent.component.html => components/tables/agent-table/agent-table.component.html} (97%) create mode 100644 bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts rename bin/u_panel/src/gui/fe/src/app/{core/tables/table.component.less => components/tables/base-table/base-table.component.less} (100%) create mode 100644 bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts create mode 100644 bin/u_panel/src/gui/fe/src/app/components/tables/index.ts rename bin/u_panel/src/gui/fe/src/app/{core/tables/job.component.html => components/tables/job-table/job-table.component.html} (96%) create mode 100644 bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.ts create mode 100644 bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.html create mode 100644 bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts rename bin/u_panel/src/gui/fe/src/app/{core/tables/result.component.html => components/tables/result-table/result-table.component.html} (95%) create mode 100644 bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts delete mode 100644 bin/u_panel/src/gui/fe/src/app/core/models/index.ts delete mode 100644 bin/u_panel/src/gui/fe/src/app/core/models/job.model.ts delete mode 100644 bin/u_panel/src/gui/fe/src/app/core/services/api.service.ts delete mode 100644 bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.ts delete mode 100644 bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/assign_job.component.ts delete mode 100644 bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/index.ts delete mode 100644 bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job-info-dialog.html delete mode 100644 bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job_info.component.ts delete mode 100644 bin/u_panel/src/gui/fe/src/app/core/tables/index.ts delete mode 100644 bin/u_panel/src/gui/fe/src/app/core/tables/job.component.ts delete mode 100644 bin/u_panel/src/gui/fe/src/app/core/tables/result.component.ts delete mode 100644 bin/u_panel/src/gui/fe/src/app/core/tables/table.component.ts rename bin/u_panel/src/gui/fe/src/app/{core => }/index.ts (100%) rename bin/u_panel/src/gui/fe/src/app/{core => }/models/agent.model.ts (77%) create mode 100644 bin/u_panel/src/gui/fe/src/app/models/index.ts create mode 100644 bin/u_panel/src/gui/fe/src/app/models/job.model.ts create mode 100644 bin/u_panel/src/gui/fe/src/app/models/payload.model.ts rename bin/u_panel/src/gui/fe/src/app/{core => }/models/result.model.ts (71%) create mode 100644 bin/u_panel/src/gui/fe/src/app/services/api.service.ts create mode 100644 bin/u_panel/src/gui/fe/src/app/services/error.service.ts rename bin/u_panel/src/gui/fe/src/app/{core => }/services/index.ts (100%) rename bin/u_panel/src/gui/fe/src/app/{core => }/utils.ts (100%) rename {integration => integration-tests}/Cargo.lock (100%) rename {integration => integration-tests}/Cargo.toml (90%) rename {integration => integration-tests}/docker-compose.yml (81%) rename {integration => integration-tests}/docker.py (98%) rename {integration => integration-tests}/integration_tests.py (75%) rename {integration => integration-tests}/integration_tests.sh (86%) rename {integration => integration-tests}/src/main.rs (100%) rename {integration => integration-tests}/tests/fixtures/agent.rs (78%) rename {integration => integration-tests}/tests/fixtures/connections.rs (100%) rename {integration => integration-tests}/tests/fixtures/env.rs (100%) rename {integration => integration-tests}/tests/fixtures/mod.rs (100%) rename {integration => integration-tests}/tests/helpers/jobs.rs (100%) rename {integration => integration-tests}/tests/helpers/mod.rs (100%) rename {integration => integration-tests}/tests/helpers/panel.rs (87%) rename {integration/tests/integration => integration-tests/tests/integration_tests}/api.rs (59%) rename {integration/tests/integration => integration-tests/tests/integration_tests}/behaviour.rs (96%) rename {integration/tests/integration => integration-tests/tests/integration_tests}/connection.rs (100%) rename {integration/tests/integration => integration-tests/tests/integration_tests}/mod.rs (100%) create mode 100644 integration-tests/tests/lib.rs rename {integration => integration-tests}/utils.py (100%) delete mode 100644 integration/tests/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 08a23ef..3cdbe68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] @@ -368,9 +368,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" [[package]] name = "byteorder" @@ -489,9 +489,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" dependencies = [ "libc", ] @@ -558,6 +558,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctor" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4056f63fce3b82d852c3da92b08ea59959890813a7f4ce9c0ff85b10cf301b" +dependencies = [ + "quote", + "syn 2.0.15", +] + [[package]] name = "cxx" version = "1.0.94" @@ -1219,9 +1229,10 @@ dependencies = [ ] [[package]] -name = "integration" +name = "integration-tests" version = "0.1.0" dependencies = [ + "ctor", "once_cell", "reqwest", "rstest 0.17.0", @@ -1298,9 +1309,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.141" +version = "0.2.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" [[package]] name = "libflate" @@ -1333,9 +1344,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b085a4f2cde5781fc4b1717f2e86c62f5cda49de7ba99a7c2eae02b61c9064c" +checksum = "36eb31c1778188ae1e64398743890d0877fef36d11521ac60406b42016e8c2cf" [[package]] name = "local-channel" @@ -1568,9 +1579,9 @@ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "openssl" -version = "0.10.50" +version = "0.10.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e30d8bc91859781f0a943411186324d580f2bbeb71b452fe91ae344806af3f1" +checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -1600,9 +1611,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.85" +version = "0.9.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d3d193fb1488ad46ffe3aaabc912cc931d02ee8518fe2959aea8ef52718b0c0" +checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" dependencies = [ "cc", "libc", @@ -1830,13 +1841,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.1", ] [[package]] @@ -1845,7 +1856,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] @@ -1854,6 +1865,12 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" + [[package]] name = "reqwest" version = "0.11.16" @@ -2003,9 +2020,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.12" +version = "0.37.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "722529a737f5a942fdbac3a46cee213053196737c5eaa3386d52e85b786f2659" +checksum = "d9b864d3c18a5785a05953adeed93e2dca37ed30f18e69bba9f30079d51f363f" dependencies = [ "bitflags", "errno 0.3.1", @@ -2148,6 +2165,20 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_qs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0431a35568651e363364210c91983c1da5eb29404d9f0928b67d4ebcfa7d330c" +dependencies = [ + "futures", + "percent-encoding", + "serde", + "thiserror", + "tracing", + "warp", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2599,13 +2630,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] @@ -2631,9 +2662,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ "matchers", "nu-ansi-term", @@ -2772,6 +2803,7 @@ dependencies = [ "rstest 0.12.0", "serde", "serde_json", + "serde_qs", "thiserror", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 850aad0..ed5ae87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ "bin/u_run", "bin/u_server", "lib/u_lib", - "integration", + "integration-tests", ] [workspace.dependencies] diff --git a/Makefile.toml b/Makefile.toml index ad3c9cd..0a103a1 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -63,7 +63,7 @@ upx -9 $BINS ''' [tasks.build] -dependencies = ["cargo_update", "cargo_build", "release_tasks"] +dependencies = ["cargo_build", "release_tasks"] clear = true [tasks.run] @@ -80,7 +80,7 @@ alias = "unit-tests" dependencies = ["cargo_update"] script = ''' [[ ! -d "./target/${TARGET}/${PROFILE_OVERRIDE}" ]] && echo 'No target folder. Build project first' && exit 1 -cd ./integration +cd ./integration-tests bash integration_tests.sh ${@} ''' @@ -88,10 +88,10 @@ bash integration_tests.sh ${@} alias = "integration-tests" [tasks.test] -dependencies = ["unit", "integration"] +dependencies = ["unit", "integration-tests"] [tasks.gen_schema] script = './scripts/gen_schema.sh' [tasks.deploy] -script = './scripts/deploy.sh' \ No newline at end of file +script = './scripts/deploy.sh' diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index a6633bf..942b1b6 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -16,14 +16,14 @@ use u_lib::{ models::AssignedJobById, }; -pub async fn process_request(jobs: Vec, client: &HttpClient) { +async fn process_request(jobs: Vec, client: &HttpClient) { if !jobs.is_empty() { for jr in &jobs { if !JobCache::contains(jr.job_id) { info!("Fetching job: {}", &jr.job_id); let fetched_job = loop { //todo: use payload cache - match client.get_job(jr.job_id, true).await { + match client.get_full_job(jr.job_id).await { Ok(result) => break result, Err(err) => { debug!("{:?} \nretrying...", err); @@ -64,7 +64,7 @@ async fn error_reporting(client: HttpClient) -> ! { match ErrChan::recv().await { Some(err) => { 'retry: for _ in 0..3 { - match client.report(&Reportable::Error(err.clone())).await { + match client.report([Reportable::Error(err.clone())]).await { Ok(_) => break 'retry, Err(e) => { debug!("Reporting error: {:?}", e); @@ -98,7 +98,7 @@ async fn agent_loop(client: HttpClient) -> ! { .collect(); if !result.is_empty() { - if let Err(err) = client.report(&result).await { + if let Err(err) = client.report(result).await { ErrChan::send(err, "report").await; } } @@ -119,7 +119,7 @@ pub fn run_forever() -> ! { .next() .unwrap() ); - init_logger(Some(logfile_uid)); + init_logger(Some(&logfile_uid)); } else { #[cfg(unix)] u_lib::unix::daemonize() @@ -139,7 +139,7 @@ pub fn run_forever() -> ! { } Err(e) => { error!("client init failed: {}", e); - exit(7) + exit(7) // todo: wtf? } } }) diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 75750bb..66bf9d1 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -4,7 +4,7 @@ use u_lib::{ api::HttpClient, jobs::join_payload, messaging::AsMsg, - models::{Agent, AssignedJob, RawJob}, + models::{Agent, AssignedJob, BriefMode, BriefOrFullJob, RawJob}, types::Id, types::PanelResult, UError, UResult, @@ -14,13 +14,16 @@ use u_lib::{ pub struct Args { #[structopt(subcommand)] cmd: Cmd, + #[structopt(short, long, default_value)] + brief: BriefMode, } #[derive(StructOpt, Debug)] enum Cmd { Agents(RUD), Jobs(JobCRUD), - Map(JobMapCRUD), + Map(MapCRUD), + Payloads(RUD), Ping, Serve, } @@ -35,13 +38,7 @@ enum JobCRUD { } #[derive(StructOpt, Debug)] -enum JobCmd { - #[structopt(external_subcommand)] - Cmd(Vec), -} - -#[derive(StructOpt, Debug)] -enum JobMapCRUD { +enum MapCRUD { Create { #[structopt(parse(try_from_str = parse_uuid))] agent_id: Id, @@ -92,35 +89,48 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { let raw_job = from_str::(&job) .map_err(|e| UError::DeserializeError(e.to_string(), job))?; let job = raw_job.validated()?; - let fat_job = join_payload(job)?; + let full_job = join_payload(job)?; - into_value(client.upload_jobs(&fat_job).await?) + into_value( + client + .upload_jobs([&BriefOrFullJob::Full(full_job)]) + .await?, + ) } JobCRUD::RUD(RUD::Read { id }) => match id { - //todo: use vec not to break frontend api, possibly refactor later - Some(id) => into_value(vec![client.get_job(id, false).await?]), + Some(id) => into_value(vec![client.get_job(id, args.brief).await?]), None => into_value(client.get_jobs().await?), }, JobCRUD::RUD(RUD::Update { item }) => { let raw_job = from_str::(&item) .map_err(|e| UError::DeserializeError(e.to_string(), item))?; let job = raw_job.validated()?; - into_value(client.update_job(&join_payload(job)?).await?) + let full_job = join_payload(job)?; + + into_value(client.update_job(&BriefOrFullJob::Full(full_job)).await?) } JobCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), }, Cmd::Map(action) => match action { - JobMapCRUD::Create { + MapCRUD::Create { agent_id, job_idents, } => into_value(client.set_jobs(agent_id, &job_idents).await?), - JobMapCRUD::RUD(RUD::Read { id }) => into_value(client.get_agent_jobs(id).await?), - JobMapCRUD::RUD(RUD::Update { item }) => { + MapCRUD::RUD(RUD::Read { id }) => into_value(client.get_assigned_jobs(id).await?), + MapCRUD::RUD(RUD::Update { item }) => { let assigned = from_str::(&item) .map_err(|e| UError::DeserializeError(e.to_string(), item))?; into_value(client.update_result(&assigned).await?) } - JobMapCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), + MapCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), + }, + Cmd::Payloads(action) => match action { + RUD::Read { id } => match id { + None => into_value(client.get_payloads().await?), + Some(id) => into_value(client.get_payload(id, args.brief).await?), + }, + RUD::Update { item: _item } => todo!(), + RUD::Delete { id } => into_value(client.del(id).await?), }, Cmd::Ping => into_value(client.ping().await?), Cmd::Serve => { diff --git a/bin/u_panel/src/gui/fe/src/app/app-routing.module.ts b/bin/u_panel/src/gui/fe/src/app/app-routing.module.ts index 0ec7bf4..8178913 100644 --- a/bin/u_panel/src/gui/fe/src/app/app-routing.module.ts +++ b/bin/u_panel/src/gui/fe/src/app/app-routing.module.ts @@ -1,14 +1,13 @@ import { NgModule } from '@angular/core'; import { RouterModule, Routes } from '@angular/router'; -import { AgentComponent } from './core/tables/agent.component'; -import { JobComponent } from './core/tables/job.component'; -import { ResultComponent } from './core/tables/result.component'; -import { AgentInfoDialogComponent } from './core/tables/dialogs/agent_info.component'; +import { JobComponent, ResultComponent, AgentComponent, PayloadComponent } from './components/tables'; +//import { AgentInfoDialogComponent } from './core/tables/dialogs/agent-info-dialog.component'; const routes: Routes = [ { path: '', redirectTo: 'agents', pathMatch: 'full' }, { path: 'agents', component: AgentComponent }, { path: 'jobs', component: JobComponent }, + { path: 'payloads', component: PayloadComponent }, { path: 'results', component: ResultComponent }, ]; diff --git a/bin/u_panel/src/gui/fe/src/app/app.component.html b/bin/u_panel/src/gui/fe/src/app/app.component.html index b35dfd7..09c8847 100644 --- a/bin/u_panel/src/gui/fe/src/app/app.component.html +++ b/bin/u_panel/src/gui/fe/src/app/app.component.html @@ -2,4 +2,5 @@ {{tab.name}} - \ No newline at end of file + + \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/app.component.ts b/bin/u_panel/src/gui/fe/src/app/app.component.ts index 566f46b..a8925de 100644 --- a/bin/u_panel/src/gui/fe/src/app/app.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/app.component.ts @@ -1,4 +1,4 @@ -import { Component, ViewChild, AfterViewInit } from '@angular/core'; +import { Component } from '@angular/core'; @Component({ selector: 'app-root', @@ -9,6 +9,7 @@ export class AppComponent { tabs = [ { name: 'Agents', link: '/agents' }, { name: 'Jobs', link: '/jobs' }, - { name: 'Results', link: '/results' } + { name: 'Results', link: '/results' }, + { name: 'Payloads', link: '/payloads' } ]; } diff --git a/bin/u_panel/src/gui/fe/src/app/app.module.ts b/bin/u_panel/src/gui/fe/src/app/app.module.ts index c1572f4..71d356f 100644 --- a/bin/u_panel/src/gui/fe/src/app/app.module.ts +++ b/bin/u_panel/src/gui/fe/src/app/app.module.ts @@ -8,22 +8,25 @@ import { MatTableModule } from '@angular/material/table'; import { MatFormFieldModule } from '@angular/material/form-field'; import { MatButtonModule } from '@angular/material/button' import { MatInputModule } from '@angular/material/input'; +import { MatSelectModule } from '@angular/material/select'; import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; import { HttpClientModule } from '@angular/common/http'; import { MatDialogModule } from '@angular/material/dialog'; import { MatIconModule } from '@angular/material/icon'; import { FormsModule } from '@angular/forms'; -import { AgentComponent, JobComponent, ResultComponent } from './core/tables'; +import { AgentComponent, JobComponent, ResultComponent, PayloadComponent } from './components/tables'; import { AgentInfoDialogComponent, AssignJobDialogComponent, JobInfoDialogComponent, - ResultInfoDialogComponent -} from './core/tables/dialogs'; + ResultInfoDialogComponent, + PayloadInfoDialogComponent +} from './components/dialogs'; import { APP_BASE_HREF } from '@angular/common'; import { MatTooltipModule } from '@angular/material/tooltip'; import { MatSnackBarModule } from '@angular/material/snack-bar'; import { MatListModule } from '@angular/material/list'; +import { GlobalErrorComponent } from './components/global-error/global-error.component'; @NgModule({ declarations: [ @@ -34,7 +37,10 @@ import { MatListModule } from '@angular/material/list'; AgentInfoDialogComponent, JobInfoDialogComponent, ResultInfoDialogComponent, - AssignJobDialogComponent + AssignJobDialogComponent, + PayloadComponent, + PayloadInfoDialogComponent, + GlobalErrorComponent ], imports: [ BrowserModule, @@ -50,6 +56,7 @@ import { MatListModule } from '@angular/material/list'; MatIconModule, MatTooltipModule, MatSnackBarModule, + MatSelectModule, MatListModule, FormsModule, BrowserAnimationsModule diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/agent-info-dialog.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.html similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/agent-info-dialog.html rename to bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.html diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/agent_info.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.ts similarity index 73% rename from bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/agent_info.component.ts rename to bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.ts index d24d3a9..1feb025 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/agent_info.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.ts @@ -1,12 +1,12 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA } from '@angular/material/dialog'; -import { AgentModel } from '../../models/agent.model'; +import { AgentModel } from '../../../models/agent.model'; import { EventEmitter } from '@angular/core'; @Component({ selector: 'agent-info-dialog', - templateUrl: 'agent-info-dialog.html', - styleUrls: ['info-dialog.component.less'] + templateUrl: 'agent-info-dialog.component.html', + styleUrls: ['../base-info-dialog.component.less'] }) export class AgentInfoDialogComponent { is_preview = true; diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/assign-job-dialog.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.html similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/assign-job-dialog.html rename to bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.html diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts new file mode 100644 index 0000000..1963201 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts @@ -0,0 +1,28 @@ +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { ApiTableService } from '../../../services'; + +@Component({ + selector: 'assign-job-dialog', + templateUrl: 'assign-job-dialog.component.html', + styleUrls: [] +}) +export class AssignJobDialogComponent { + rows: string[] = []; + selected_rows: string[] = []; + + constructor( + @Inject(MAT_DIALOG_DATA) public agent_id: string, + private dataSource: ApiTableService, + ) { + dataSource.getJobs().subscribe(resp => { + this.rows = resp.map(j => `${j.id} ${j.alias}`) + }) + } + + assignSelectedJobs() { + const job_ids = this.selected_rows.map(row => row.split(' ', 1)[0]).join(' '); + const request = `${this.agent_id} ${job_ids}` + this.dataSource.createResult(request) + } +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/info-dialog.component.less b/bin/u_panel/src/gui/fe/src/app/components/dialogs/base-info-dialog.component.less similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/info-dialog.component.less rename to bin/u_panel/src/gui/fe/src/app/components/dialogs/base-info-dialog.component.less diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/index.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/index.ts new file mode 100644 index 0000000..1a58ff1 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/index.ts @@ -0,0 +1,5 @@ +export * from './agent-info-dialog/agent-info-dialog.component'; +export * from './result-info-dialog/result-info-dialog.component'; +export * from './job-info-dialog/job-info-dialog.component'; +export * from './assign-job-dialog/assign-job-dialog.component'; +export * from './payload-info-dialog/payload-info-dialog.component'; \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html new file mode 100644 index 0000000..dc79aa7 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html @@ -0,0 +1,52 @@ +

Job info

+

Editing job info

+ +
+ + ID + + + + Alias + + + + Args + + +
+
+ + Type + + + + Platform + + + + Schedule + + +
+
+ + Payload + + {{ pld[1] }} + + + + Payload data + + + +
+ + + + + + \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts new file mode 100644 index 0000000..02a197e --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts @@ -0,0 +1,46 @@ +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { EventEmitter } from '@angular/core'; +import { BriefOrFullJobModel } from '../../../models/job.model'; +import { ApiTableService } from 'src/app/services'; +import { BriefOrFullPayloadModel, isFullPayload } from 'src/app/models'; + +@Component({ + selector: 'job-info-dialog', + templateUrl: 'job-info-dialog.component.html', + styleUrls: ['../base-info-dialog.component.less'] +}) +export class JobInfoDialogComponent { + isPreview = true; + isTooBigPayload = false; + decodedPayload = ""; + //[id, name] + allPayloads: [string, string][] = []; + + onSave = new EventEmitter(); + + constructor(@Inject(MAT_DIALOG_DATA) public data: BriefOrFullJobModel, dataSource: ApiTableService) { + if (data.payload !== null) { + this.showPayload(data.payload) + } + + dataSource.getPayloads().subscribe(resp => { + this.allPayloads = resp.map(r => [r.id, r.name]) + }) + } + + showPayload(payload: BriefOrFullPayloadModel) { + if (isFullPayload(payload)) { + this.decodedPayload = new TextDecoder().decode(new Uint8Array(payload.data)) + } else { + this.isTooBigPayload = true + } + } + + updateJob() { + // if (this.decodedPayload.length > 0) { + // this.data.payload = Array.from(new TextEncoder().encode(this.decodedPayload)) + // } + // this.onSave.emit(this.data); + } +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html new file mode 100644 index 0000000..5780429 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html @@ -0,0 +1,24 @@ +

Result

+ +
+ + ID + + + + Name + + + + MIME-type + + + + Size + + +
+ + + + \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts new file mode 100644 index 0000000..ba879a4 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts @@ -0,0 +1,14 @@ +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { PayloadModel } from 'src/app/models/payload.model'; + +@Component({ + selector: 'payload-info-dialog', + templateUrl: 'payload-info-dialog.component.html', + styleUrls: [] +}) +export class PayloadInfoDialogComponent { + + constructor(@Inject(MAT_DIALOG_DATA) public data: PayloadModel) { } + +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/result-info-dialog.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.html similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/result-info-dialog.html rename to bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.html diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/result_info.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.ts similarity index 74% rename from bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/result_info.component.ts rename to bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.ts index b02fae5..4d9734f 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/result_info.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.ts @@ -1,11 +1,11 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA } from '@angular/material/dialog'; -import { ResultModel } from '../../models/result.model'; +import { ResultModel } from '../../../models/result.model'; @Component({ selector: 'result-info-dialog', - templateUrl: 'result-info-dialog.html', - styleUrls: ['info-dialog.component.less'] + templateUrl: 'result-info-dialog.component.html', + styleUrls: ['../base-info-dialog.component.less'] }) export class ResultInfoDialogComponent { decodedResult: string; diff --git a/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.html b/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.html new file mode 100644 index 0000000..e69de29 diff --git a/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.less b/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.less new file mode 100644 index 0000000..e69de29 diff --git a/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.ts b/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.ts new file mode 100644 index 0000000..24e4327 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.ts @@ -0,0 +1,34 @@ +import { Component, OnInit } from '@angular/core'; +import { MatSnackBar, MatSnackBarConfig } from '@angular/material/snack-bar'; +import { ErrorService } from 'src/app/services/error.service'; + +@Component({ + selector: 'global-error', + templateUrl: './global-error.component.html', + styleUrls: ['./global-error.component.less'] +}) +export class GlobalErrorComponent implements OnInit { + + constructor( + private snackBar: MatSnackBar, + private errorService: ErrorService + ) { } + + ngOnInit() { + this.errorService.error$.subscribe(err => { + const _config = (duration: number): MatSnackBarConfig => { + return { + horizontalPosition: 'right', + verticalPosition: 'bottom', + duration + } + } + const error = true; + const cfg = error ? _config(0) : _config(2000) + + if (err != '') { + this.snackBar.open(err, 'Ok', cfg) + } + }) + } +} diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.html b/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.html similarity index 97% rename from bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.html rename to bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.html index fbaaba9..78aa311 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.html +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.html @@ -6,7 +6,7 @@ Filter - + diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts new file mode 100644 index 0000000..2a98125 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts @@ -0,0 +1,42 @@ +import { Component, OnInit } from '@angular/core'; +import { TablesComponent } from '../base-table/base-table.component'; +import { AgentModel, Area } from '../../../models'; +import { AssignJobDialogComponent, AgentInfoDialogComponent } from '../../dialogs'; + +@Component({ + selector: 'agent-table', + templateUrl: './agent-table.component.html', + styleUrls: ['../base-table/base-table.component.less'], +}) +export class AgentComponent extends TablesComponent implements OnInit { + area = 'agents' as Area + displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions'] + + showItemDialog(id: string) { + this.dataSource.getAgent(id).subscribe(resp => { + const dialog = this.infoDialog.open(AgentInfoDialogComponent, { + data: resp, + width: '1000px', + }); + + const saveSub = dialog.componentInstance.onSave.subscribe(result => { + this.dataSource.updateAgent(result).subscribe(_ => { + alert('Saved') + this.loadTableData() + }) + }) + + dialog.afterClosed().subscribe(result => { + saveSub.unsubscribe() + this.router.navigate(['.'], { relativeTo: this.route }) + }) + }) + } + + assignJobs(id: string) { + const dialog = this.infoDialog.open(AssignJobDialogComponent, { + data: id, + width: '1000px', + }); + } +} diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/table.component.less b/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.less similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/tables/table.component.less rename to bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.less diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts new file mode 100644 index 0000000..aa2183a --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts @@ -0,0 +1,58 @@ +import { OnInit, Directive, Component } from '@angular/core'; +import { ApiTableService } from '../../..'; +import { MatTableDataSource } from '@angular/material/table'; +import { MatDialog } from '@angular/material/dialog'; +import { ApiModel, Area } from '../../../models'; +import { ActivatedRoute, Router } from '@angular/router'; + +@Directive() +export abstract class TablesComponent implements OnInit { + abstract area: Area; + table_data: MatTableDataSource = new MatTableDataSource; + isLoadingResults = true; + + constructor( + public dataSource: ApiTableService, + public infoDialog: MatDialog, + public route: ActivatedRoute, + public router: Router, + ) { } + + ngOnInit() { + this.loadTableData(); + this.route.queryParams.subscribe(params => { + const id = params['id'] + const new_agent = params['new'] + if (id) { + this.showItemDialog(id); + } + if (new_agent) { + this.showItemDialog(null); + } + }) + //interval(10000).subscribe(_ => this.loadTableData()); + } + + loadTableData() { + this.isLoadingResults = true; + + this.dataSource.getMany(this.area).subscribe(resp => { + this.isLoadingResults = false; + this.table_data.data = resp; + }) + } + + applyFilter(event: Event) { + const filterValue = (event.target as HTMLInputElement).value; + this.table_data.filter = filterValue.trim().toLowerCase(); + } + + deleteItem(id: string) { + if (confirm(`Delete ${id}?`)) { + this.dataSource.delete(id, this.area) + } + } + + abstract displayedColumns: string[]; + abstract showItemDialog(id: string | null): void; +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/index.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/index.ts new file mode 100644 index 0000000..cf8d91d --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/index.ts @@ -0,0 +1,5 @@ +export * from './agent-table/agent-table.component'; +export * from './base-table/base-table.component'; +export * from './job-table/job-table.component'; +export * from './payload-table/payload-table.component'; +export * from './result-table/result-table.component'; \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/job.component.html b/bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.html similarity index 96% rename from bin/u_panel/src/gui/fe/src/app/core/tables/job.component.html rename to bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.html index bfd99b2..fc53ad0 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/job.component.html +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.html @@ -6,7 +6,7 @@ Filter - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Name + {{row.name}} + MIME-type + {{row.mime_type}} + Size + {{row.size}} + + + | + +
No data
+ + + + \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts new file mode 100644 index 0000000..7cd3ae4 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts @@ -0,0 +1,30 @@ +import { Component } from '@angular/core'; +import { Area } from 'src/app/models'; +import { PayloadModel } from 'src/app/models/payload.model'; +import { PayloadInfoDialogComponent } from '../../dialogs'; +import { TablesComponent } from '../base-table/base-table.component'; + +@Component({ + selector: 'payload-table', + templateUrl: './payload-table.component.html', + styleUrls: ['../base-table/base-table.component.less'], + providers: [{ provide: 'area', useValue: 'payloads' }] +}) +export class PayloadComponent extends TablesComponent { + area = 'payloads' as Area + displayedColumns = ["name", "mime_type", "size"]; + + showItemDialog(id: string) { + this.dataSource.getPayload(id).subscribe(resp => { + const dialog = this.infoDialog.open(PayloadInfoDialogComponent, { + data: resp, + width: '1000px', + }); + + dialog.afterClosed().subscribe(_ => { + this.router.navigate(['.'], { relativeTo: this.route }) + }) + }) + } + +} diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/result.component.html b/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.html similarity index 95% rename from bin/u_panel/src/gui/fe/src/app/core/tables/result.component.html rename to bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.html index afda1cb..08296e6 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/result.component.html +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.html @@ -6,7 +6,7 @@ Filter - + @@ -49,7 +49,7 @@ - ID + Last updated {{row.updated.secs_since_epoch * 1000| date:'long'}} diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts new file mode 100644 index 0000000..5d1a4cf --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts @@ -0,0 +1,36 @@ +import { Component, OnInit } from '@angular/core'; +import { TablesComponent } from '../base-table/base-table.component'; +import { Area, ResultModel } from '../../../models'; +import { ResultInfoDialogComponent } from '../../dialogs'; + +@Component({ + selector: 'results-table', + templateUrl: './result-table.component.html', + styleUrls: ['../base-table/base-table.component.less'], + providers: [{ provide: 'area', useValue: 'map' }] +}) +export class ResultComponent extends TablesComponent { + area = 'map' as Area + displayedColumns = [ + 'id', + 'alias', + 'agent_id', + 'job_id', + 'state', + 'last_updated', + 'actions' + ]; + + showItemDialog(id: string) { + this.dataSource.getResult(id).subscribe(resp => { + const dialog = this.infoDialog.open(ResultInfoDialogComponent, { + data: resp, + width: '1000px', + }); + + dialog.afterClosed().subscribe(_ => { + this.router.navigate(['.'], { relativeTo: this.route }) + }) + }) + } +} diff --git a/bin/u_panel/src/gui/fe/src/app/core/models/index.ts b/bin/u_panel/src/gui/fe/src/app/core/models/index.ts deleted file mode 100644 index dad077a..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/models/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -export * from './agent.model'; -export * from './result.model'; -export * from './job.model'; - -export interface UTCDate { - secs_since_epoch: number, - nanos_since_epoch: number -} - -export abstract class ApiModel { } - -export interface Empty extends ApiModel { } - -export type Area = "agents" | "jobs" | "map"; \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/models/job.model.ts b/bin/u_panel/src/gui/fe/src/app/core/models/job.model.ts deleted file mode 100644 index 2b84d70..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/models/job.model.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { ApiModel } from "."; - -export interface JobModel extends ApiModel { - alias: string, - argv: string, - id: string, - exec_type: string, - platform: string, - payload: number[] | null, - schedule: string | null, -} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/services/api.service.ts b/bin/u_panel/src/gui/fe/src/app/core/services/api.service.ts deleted file mode 100644 index 84348e2..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/services/api.service.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { environment } from 'src/environments/environment'; -import { HttpClient } from '@angular/common/http'; -import { firstValueFrom } from 'rxjs'; -import { ApiModel, Empty, Area } from '../models'; - -interface ServerResponse { - status: "ok" | "err", - data: T | string -} - -export class ApiTableService { - area: Area; - - constructor(private http: HttpClient, area: Area) { - this.area = area; - } - - requestUrl = `${environment.server}/cmd/`; - - async req(cmd: string): Promise> { - return await firstValueFrom(this.http.post>(this.requestUrl, cmd)) - } - - async getOne(id: string, area: string = this.area): Promise> { - const resp = await this.req(`${area} read ${id}`) - if (resp.data.length === 0) { - return { - status: 'err', - data: `${id} not found in ${area}` - } - } - return { - status: resp.status, - data: resp.data[0] - } - } - - async getMany(): Promise> { - return await this.req(`${this.area} read`) - } - - async update(item: T): Promise> { - return await this.req(`${this.area} update '${JSON.stringify(item)}'`) - } - - async delete(id: string): Promise> { - return await this.req(`${this.area} delete ${id}`) - } - - async create(item: string): Promise> { - return await this.req(`${this.area} create ${item}`) - } -} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.ts deleted file mode 100644 index c72b8dc..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { Component, OnDestroy, OnInit } from '@angular/core'; -import { TablesComponent } from './table.component'; -import { AgentModel } from '../models'; -import { AgentInfoDialogComponent } from './dialogs/agent_info.component'; -import { HttpErrorResponse } from '@angular/common/http'; -import { AssignJobDialogComponent } from './dialogs'; - -@Component({ - selector: 'agent-table', - templateUrl: './agent.component.html', - styleUrls: ['./table.component.less'] -}) -export class AgentComponent extends TablesComponent implements OnInit { - - //dialogSubscr!: Subscription; - area = 'agents' as const; - - displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions'] - - show_item_dialog(id: string) { - this.data_source!.getOne(id).then(resp => { - if (resp.status === 'ok') { - const dialog = this.infoDialog.open(AgentInfoDialogComponent, { - data: resp.data as AgentModel, - width: '1000px', - }); - - const saveSub = dialog.componentInstance.onSave.subscribe(result => { - this.data_source!.update(result).then(_ => { - this.openSnackBar('Saved', false) - this.loadTableData() - }) - .catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) - }) - - dialog.afterClosed().subscribe(result => { - saveSub.unsubscribe() - this.router.navigate(['.'], { relativeTo: this.route }) - }) - } else { - this.openSnackBar(resp.data) - } - }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) - } - - assignJobs(id: string) { - const dialog = this.infoDialog.open(AssignJobDialogComponent, { - data: id, - width: '1000px', - }); - } -} diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/assign_job.component.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/assign_job.component.ts deleted file mode 100644 index cd278d6..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/assign_job.component.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Component, Inject } from '@angular/core'; -import { MAT_DIALOG_DATA } from '@angular/material/dialog'; -import { HttpClient } from '@angular/common/http'; -import { ApiTableService } from '../../services'; -import { JobModel } from '../../models'; -import { MatListOption } from '@angular/material/list'; - -@Component({ - selector: 'assign-job-dialog', - templateUrl: 'assign-job-dialog.html', - styleUrls: [] -}) -export class AssignJobDialogComponent { - rows: string[] = []; - selected_rows: string[] = []; - - constructor(@Inject(MAT_DIALOG_DATA) public agent_id: string, private http: HttpClient) { - new ApiTableService(http, "jobs").getMany().then(result => { - if (result.status == "ok") { - const jobs = result.data as JobModel[] - this.rows = jobs.map(j => `${j.id} ${j.alias}`) - } else { - alert(result.data as string) - } - }).catch(err => alert(err)) - } - - assignSelectedJobs() { - const job_ids = this.selected_rows.map(row => row.split(' ', 1)[0]).join(' '); - const request = `${this.agent_id} ${job_ids}` - new ApiTableService(this.http, "map").create(request).catch(err => alert(err)) - } -} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/index.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/index.ts deleted file mode 100644 index 4bdb1aa..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './agent_info.component'; -export * from './result_info.component'; -export * from './job_info.component'; -export * from './assign_job.component'; \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job-info-dialog.html b/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job-info-dialog.html deleted file mode 100644 index 7bb97f6..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job-info-dialog.html +++ /dev/null @@ -1,44 +0,0 @@ -

Job info

-

Editing job info

- -
- - ID - - - - Alias - - - - Args - - -
-
- - Type - - - - Platform - - - - Schedule - - -
-
- - Payload - - -
- - - - - - \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job_info.component.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job_info.component.ts deleted file mode 100644 index 3433cbf..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job_info.component.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { Component, Inject } from '@angular/core'; -import { MAT_DIALOG_DATA } from '@angular/material/dialog'; -import { EventEmitter } from '@angular/core'; -import { JobModel } from '../../models/job.model'; - -@Component({ - selector: 'job-info-dialog', - templateUrl: 'job-info-dialog.html', - styleUrls: ['info-dialog.component.less'] -}) -export class JobInfoDialogComponent { - is_preview = true; - decodedPayload: string; - onSave = new EventEmitter(); - - constructor(@Inject(MAT_DIALOG_DATA) public data: JobModel) { - if (data.payload !== null) { - this.decodedPayload = new TextDecoder().decode(new Uint8Array(data.payload)) - } else { - this.decodedPayload = "" - } - } - - updateJob() { - if (this.decodedPayload.length > 0) { - this.data.payload = Array.from(new TextEncoder().encode(this.decodedPayload)) - } - this.onSave.emit(this.data); - } -} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/index.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/index.ts deleted file mode 100644 index 11dfaf2..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './agent.component'; -export * from './job.component'; -export * from './result.component'; \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/job.component.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/job.component.ts deleted file mode 100644 index 5b474c7..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/job.component.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { Component, OnInit } from '@angular/core'; -import { TablesComponent } from './table.component'; -import { JobModel } from '../models'; -import { JobInfoDialogComponent } from './dialogs'; -import { HttpErrorResponse } from '@angular/common/http'; - -@Component({ - selector: 'job-table', - templateUrl: './job.component.html', - styleUrls: ['./table.component.less'] -}) -export class JobComponent extends TablesComponent { - area = 'jobs' as const; - displayedColumns = ['id', 'alias', 'platform', 'schedule', 'exec_type', 'actions'] - - show_item_dialog(id: string | null) { - const show_dlg = (id: string, edit: boolean) => { - this.data_source!.getOne(id).then(resp => { - if (resp.status === 'ok') { - var dialog = this.infoDialog.open(JobInfoDialogComponent, { - data: resp.data as JobModel, - width: '1000px', - }); - if (edit) { - dialog.componentInstance.is_preview = false - } - - const saveSub = dialog.componentInstance.onSave.subscribe(result => { - this.data_source!.update(result) - .then(_ => { - this.openSnackBar("Saved", false) - this.loadTableData() - }) - .catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) - }) - - dialog.afterClosed().subscribe(result => { - saveSub.unsubscribe() - this.router.navigate(['.'], { relativeTo: this.route }) - }) - } else { - this.openSnackBar(resp.data) - } - }).catch((err: any) => this.openSnackBar(err)) - } - - if (id) { - show_dlg(id, false) - } else { - this.data_source!.create('"{}"').then(resp => { - if (resp.status === 'ok') { - show_dlg(resp.data[0], true) - } else { - this.openSnackBar(resp.data) - } - }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) - } - } -} diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/result.component.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/result.component.ts deleted file mode 100644 index ec98ab7..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/result.component.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Component, OnInit } from '@angular/core'; -import { TablesComponent } from './table.component'; -import { ResultModel } from '../models'; -import { ResultInfoDialogComponent } from './dialogs'; -import { HttpErrorResponse } from '@angular/common/http'; - -@Component({ - selector: 'results-table', - templateUrl: './result.component.html', - styleUrls: ['./table.component.less'] -}) -export class ResultComponent extends TablesComponent { - area = 'map' as const; - - displayedColumns = [ - 'id', - 'alias', - 'agent_id', - 'job_id', - 'state', - 'last_updated', - 'actions' - ]; - - show_item_dialog(id: string) { - this.data_source!.getOne(id).then(resp => { - if (resp.status === 'ok') { - const dialog = this.infoDialog.open(ResultInfoDialogComponent, { - data: resp.data as ResultModel, - width: '1000px', - }); - - dialog.afterClosed().subscribe(result => { - this.router.navigate(['.'], { relativeTo: this.route }) - }) - } else { - this.openSnackBar(resp.data) - } - }).catch((err: HttpErrorResponse) => this.openSnackBar(err.message)) - } -} diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/table.component.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/table.component.ts deleted file mode 100644 index 5a7bd23..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/table.component.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { OnInit, Directive } from '@angular/core'; -import { HttpClient } from '@angular/common/http'; -import { ApiTableService } from '../'; -import { MatTableDataSource } from '@angular/material/table'; -import { MatDialog } from '@angular/material/dialog'; -import { ApiModel, Area } from '../models'; -import { ActivatedRoute, Router } from '@angular/router'; -import { interval } from 'rxjs'; -import { MatSnackBar, MatSnackBarConfig } from '@angular/material/snack-bar'; - -@Directive() -export abstract class TablesComponent implements OnInit { - abstract area: Area; - data_source!: ApiTableService; - table_data!: MatTableDataSource; - - isLoadingResults = true; - - constructor( - public httpClient: HttpClient, - public infoDialog: MatDialog, - public route: ActivatedRoute, - public router: Router, - public snackBar: MatSnackBar - ) { - this.table_data = new MatTableDataSource; - } - - ngOnInit() { - this.data_source = new ApiTableService(this.httpClient, this.area); - this.loadTableData(); - this.route.queryParams.subscribe(params => { - const id = params['id'] - const new_agent = params['new'] - if (id) { - this.show_item_dialog(id); - } - if (new_agent) { - this.show_item_dialog(null); - } - }) - //interval(10000).subscribe(_ => this.loadTableData()); - } - - async loadTableData() { - this.isLoadingResults = true; - //possibly needs try/catch - const data = await this.data_source!.getMany(); - this.isLoadingResults = false; - - if (typeof data.data !== 'string') { - this.table_data.data = data.data - } else { - alert(`Error: ${data}`) - }; - } - - apply_filter(event: Event) { - const filterValue = (event.target as HTMLInputElement).value; - this.table_data.filter = filterValue.trim().toLowerCase(); - } - - deleteItem(id: string) { - if (confirm(`Delete ${id}?`)) { - this.data_source!.delete(id).catch(this.openSnackBar) - } - } - - openSnackBar(message: any, error: boolean = true) { - const msg = JSON.stringify(message) - const _config = (duration: number): MatSnackBarConfig => { - return { - horizontalPosition: 'right', - verticalPosition: 'bottom', - duration - } - } - const cfg = error ? _config(0) : _config(2000) - this.snackBar.open(msg, 'Ok', cfg); - } - - abstract displayedColumns: string[]; - abstract show_item_dialog(id: string | null): void; -} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/index.ts b/bin/u_panel/src/gui/fe/src/app/index.ts similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/index.ts rename to bin/u_panel/src/gui/fe/src/app/index.ts diff --git a/bin/u_panel/src/gui/fe/src/app/core/models/agent.model.ts b/bin/u_panel/src/gui/fe/src/app/models/agent.model.ts similarity index 77% rename from bin/u_panel/src/gui/fe/src/app/core/models/agent.model.ts rename to bin/u_panel/src/gui/fe/src/app/models/agent.model.ts index d798f0e..cd8ef76 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/models/agent.model.ts +++ b/bin/u_panel/src/gui/fe/src/app/models/agent.model.ts @@ -1,6 +1,6 @@ -import { UTCDate, ApiModel } from "."; +import { UTCDate } from "."; -export interface AgentModel extends ApiModel { +export interface AgentModel { alias: string | null, hostname: string, host_info: string, diff --git a/bin/u_panel/src/gui/fe/src/app/models/index.ts b/bin/u_panel/src/gui/fe/src/app/models/index.ts new file mode 100644 index 0000000..3ff527f --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/models/index.ts @@ -0,0 +1,24 @@ +import { AgentModel } from './agent.model'; +import { JobModel } from './job.model'; +import { PayloadModel } from './payload.model'; +import { ResultModel } from './result.model'; + +export * from './agent.model'; +export * from './result.model'; +export * from './job.model'; +export * from './payload.model'; + +export interface UTCDate { + secs_since_epoch: number, + nanos_since_epoch: number +} + +export type Area = "agents" | "jobs" | "map" | "payloads"; + +export type ApiModel = AgentModel | JobModel | ResultModel | PayloadModel | Empty; + +export interface Empty { } + +export function getAreaByModel(_: AgentModel): Area { + return "agents" +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/models/job.model.ts b/bin/u_panel/src/gui/fe/src/app/models/job.model.ts new file mode 100644 index 0000000..492db84 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/models/job.model.ts @@ -0,0 +1,16 @@ +import { BriefOrFullPayloadModel } from './' + +export interface JobModel { + alias: string | null, + argv: string, + id: string, + exec_type: string, + target_platforms: string, + payload: string | null, + schedule: string | null, +} + +export interface BriefOrFullJobModel { + job: JobModel, + payload: BriefOrFullPayloadModel | null, +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts b/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts new file mode 100644 index 0000000..4e1e9c7 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts @@ -0,0 +1,17 @@ +export interface PayloadModel { + id: string, + mime_type: string, + name: string, + size: number, +} + +export interface FullPayloadModel { + meta: PayloadModel, + data: number[] +} + +export type BriefOrFullPayloadModel = PayloadModel | FullPayloadModel; + +export function isFullPayload(payload: BriefOrFullPayloadModel): payload is FullPayloadModel { + return (payload as FullPayloadModel).data !== undefined +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/models/result.model.ts b/bin/u_panel/src/gui/fe/src/app/models/result.model.ts similarity index 71% rename from bin/u_panel/src/gui/fe/src/app/core/models/result.model.ts rename to bin/u_panel/src/gui/fe/src/app/models/result.model.ts index c699787..56b0a93 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/models/result.model.ts +++ b/bin/u_panel/src/gui/fe/src/app/models/result.model.ts @@ -1,6 +1,6 @@ -import { UTCDate, ApiModel } from "."; +import { UTCDate } from "."; -export interface ResultModel extends ApiModel { +export interface ResultModel { agent_id: string, alias: string, created: UTCDate, diff --git a/bin/u_panel/src/gui/fe/src/app/services/api.service.ts b/bin/u_panel/src/gui/fe/src/app/services/api.service.ts new file mode 100644 index 0000000..cee6ecb --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/services/api.service.ts @@ -0,0 +1,138 @@ +import { environment } from 'src/environments/environment'; +import { HttpClient, HttpErrorResponse } from '@angular/common/http'; +import { Observable, map, catchError, throwError } from 'rxjs'; +import { ApiModel, getAreaByModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, BriefOrFullJobModel } from '../models'; +import { Injectable, Inject } from '@angular/core'; +import { ErrorService } from './error.service'; + +type Status = "ok" | "err"; + +interface ServerResponse { + status: Status, + data: T | string +} + +@Injectable({ + providedIn: 'root' +}) +export class ApiTableService { + + constructor( + private http: HttpClient, + private errorService: ErrorService + ) { + } + + requestUrl = `${environment.server}/cmd/`; + + req(cmd: string): Observable> { + return this.http.post>(this.requestUrl, cmd).pipe( + catchError(this.errorHandler) + ) + } + + getOne(id: string, area: Area, brief: 'yes' | 'no' | 'auto' | null = null): Observable { + const request = `${area} read ${id}` + (brief !== null ? `-b=${brief}` : '') + const resp = this.req(request).pipe( + map(resp => { + if (resp.data.length === 0) { + return { + status: 'err' as Status, + data: `${id} not found in ${area}` + } + } + return { + status: resp.status, + data: resp.data[0] + } + })); + return this.filterErrStatus(resp) + } + + getAgent(id: string): Observable { + return this.getOne(id, 'agents') + } + + getJob(id: string): Observable { + return this.getOne(id, 'jobs') + } + + getResult(id: string): Observable { + return this.getOne(id, 'map') + } + + getPayload(id: string): Observable { + return this.getOne(id, 'payloads') + } + + getMany(area: Area): Observable { + return this.filterErrStatus(this.req(`${area} read`)) + } + + getAgents(): Observable { + return this.getMany('agents') + } + + getJobs(): Observable { + return this.getMany('jobs') + } + + getResults(): Observable { + return this.getMany('map') + } + + getPayloads(): Observable { + return this.getMany('payloads') + } + + update(item: T, area: Area): Observable { + return this.filterErrStatus(this.req(`${area} update '${JSON.stringify(item)}'`)) + } + + updateAgent(item: AgentModel): Observable { + return this.update(item, 'agents') + } + + updateJob(item: JobModel): Observable { + return this.update(item, 'jobs') + } + + updateResult(item: ResultModel): Observable { + return this.update(item, 'map') + } + + updatePayload(item: PayloadModel): Observable { + return this.update(item, 'payloads') + } + + delete(id: string, area: Area): Observable { + return this.filterErrStatus(this.req(`${area} delete ${id}`)) + } + + create(item: string | null, area: Area): Observable { + if (!item) { + item = '"{}"' + } + return this.filterErrStatus(this.req(`${area} create ${item}`)) + } + + createResult(item: string): Observable { + return this.create(item, 'map') + } + + filterErrStatus(obs: Observable>): Observable { + return obs.pipe( + map(r => { + if (r.status == 'err') { + throw new Error(r.data as string) + } + return r.data as R + }), + catchError(this.errorHandler.bind(this))) + } + + errorHandler(err: HttpErrorResponse, _: R) { + this.errorService.handle(err.message); + return throwError(() => new Error(err.message)); + } +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/services/error.service.ts b/bin/u_panel/src/gui/fe/src/app/services/error.service.ts new file mode 100644 index 0000000..a4d1d13 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/services/error.service.ts @@ -0,0 +1,17 @@ +import { Injectable } from '@angular/core'; +import { Subject } from 'rxjs'; + +@Injectable({ + providedIn: 'root' +}) +export class ErrorService { + error$ = new Subject(); + + handle(msg: string) { + this.error$.next(msg) + } + + clear() { + this.handle('') + } +} diff --git a/bin/u_panel/src/gui/fe/src/app/core/services/index.ts b/bin/u_panel/src/gui/fe/src/app/services/index.ts similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/services/index.ts rename to bin/u_panel/src/gui/fe/src/app/services/index.ts diff --git a/bin/u_panel/src/gui/fe/src/app/core/utils.ts b/bin/u_panel/src/gui/fe/src/app/utils.ts similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/utils.ts rename to bin/u_panel/src/gui/fe/src/app/utils.ts diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index 226c592..9e7234c 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -13,13 +13,13 @@ use u_lib::logging::init_logger; #[actix_web::main] async fn main() -> AnyResult<()> { + init_logger(None); + let env = AccessEnv::load()?; let client = HttpClient::new(&env.u_server, Some(env.admin_auth_token)).await?; let args = Args::from_args(); - - init_logger(None::<&str>); - let result = process_cmd(client, args).await.to_string(); + println!("{result}"); Ok(()) } diff --git a/bin/u_server/Cargo.toml b/bin/u_server/Cargo.toml index 9b800b2..210f7a6 100644 --- a/bin/u_server/Cargo.toml +++ b/bin/u_server/Cargo.toml @@ -20,6 +20,7 @@ tokio = { workspace = true, features = ["macros"] } uuid = { workspace = true, features = ["serde", "v4"] } u_lib = { path = "../../lib/u_lib", features = ["server"] } warp = { version = "0.3.1", features = ["tls"] } +serde_qs = { version = "0.12.0", features = ["warp"] } [dev-dependencies] rstest = "0.12" diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index 053bf26..0f7dfe6 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -3,7 +3,7 @@ use diesel::{pg::PgConnection, prelude::*, result::Error as DslError, Connection use std::mem::drop; use u_lib::{ db::PgAsyncPool, - models::{schema, Agent, AssignedJob, JobModel, JobState, PayloadMeta, ThinJob}, + models::{schema, Agent, AssignedJob, BriefJob, JobModel, JobState, PayloadMeta}, platform::Platform, types::Id, }; @@ -50,7 +50,7 @@ pub struct UDB<'c> { } impl UDB<'_> { - pub fn insert_jobs(&mut self, jobs: &[ThinJob]) -> Result<()> { + pub fn insert_jobs(&mut self, jobs: &[BriefJob]) -> Result<()> { use schema::{jobs, payloads}; let (jobs, payloads_opt): (Vec<_>, Vec<_>) = jobs @@ -76,7 +76,7 @@ impl UDB<'_> { .map_err(with_err_ctx("Can't insert jobs")) } - pub fn get_job(&mut self, id: Id) -> Result> { + pub fn get_job(&mut self, id: Id) -> Result> { use schema::{jobs, payloads}; let maybe_job_with_payload = jobs::table @@ -86,7 +86,7 @@ impl UDB<'_> { .optional() .map_err(with_err_ctx(format!("Can't get job {id}")))?; - Ok(maybe_job_with_payload.map(|(job, payload_meta)| ThinJob { job, payload_meta })) + Ok(maybe_job_with_payload.map(|(job, payload_meta)| BriefJob { job, payload_meta })) } pub fn get_jobs(&mut self) -> Result> { @@ -97,7 +97,25 @@ impl UDB<'_> { .map_err(with_err_ctx("Can't get jobs")) } - pub fn find_job_by_alias(&mut self, alias: &str) -> Result> { + pub fn get_payload_meta(&mut self, id: Id) -> Result> { + use schema::payloads; + + payloads::table + .filter(payloads::id.eq(id)) + .first(self.conn) + .optional() + .map_err(with_err_ctx(format!("Can't get payload {id}"))) + } + + pub fn get_payload_metas(&mut self) -> Result> { + use schema::payloads; + + payloads::table + .load(self.conn) + .map_err(with_err_ctx("Can't get payloads")) + } + + pub fn find_job_by_alias(&mut self, alias: &str) -> Result> { use schema::{jobs, payloads}; let maybe_job_with_payload = jobs::table @@ -107,7 +125,7 @@ impl UDB<'_> { .optional() .map_err(with_err_ctx(format!("Can't get job by alias {alias}")))?; - Ok(maybe_job_with_payload.map(|(job, payload_meta)| ThinJob { job, payload_meta })) + Ok(maybe_job_with_payload.map(|(job, payload_meta)| BriefJob { job, payload_meta })) } pub fn insert_result(&mut self, result: &AssignedJob) -> Result<()> { @@ -150,7 +168,11 @@ impl UDB<'_> { } //TODO: filters possibly could work in a wrong way, check - pub fn get_exact_jobs(&mut self, id: Option, personal: bool) -> Result> { + pub fn get_assigned_jobs( + &mut self, + id: Option, + personal: bool, + ) -> Result> { use schema::results; let mut q = results::table.into_boxed(); diff --git a/bin/u_server/src/error.rs b/bin/u_server/src/error.rs index 0d4a1d4..26eb199 100644 --- a/bin/u_server/src/error.rs +++ b/bin/u_server/src/error.rs @@ -56,7 +56,7 @@ impl RejResponse { pub fn internal() -> Self { Self { - message: "INTERNAL_SERVER_ERROR".to_string(), + message: "INTERNAL SERVER ERROR".to_string(), status: StatusCode::INTERNAL_SERVER_ERROR, } } diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index 0fd739f..e1d6b7c 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -3,22 +3,24 @@ use std::sync::Arc; use crate::db::{PgRepo, UDB}; use crate::error::Error; use serde::Deserialize; +use u_lib::ufs; use u_lib::{ api::retypes, jobs::{join_payload, split_payload}, - messaging::{AsMsg, Reportable}, - misc::OneOrVec, + messaging::Reportable, models::*, types::Id, }; use warp::reject::not_found; use warp::Rejection; +const MAX_READABLE_PAYLOAD_SIZE: i64 = 1024 * 32; + type EndpResult = Result; #[derive(Deserialize)] -pub struct GetJobQuery { - force_payload: bool, +pub struct PayloadFlags { + brief: BriefMode, } pub struct Endpoints; @@ -44,24 +46,32 @@ impl Endpoints { pub async fn get_job( repo: Arc, id: Id, - params: GetJobQuery, + params: Option, ) -> EndpResult { let Some(job) = repo.interact(move |mut db| db.get_job(id)).await? else { return Err(not_found()) }; + let make_full_job = |j| -> Result { + let full_job = join_payload(j).map_err(Error::from)?; + Ok(BriefOrFullJob::Full(full_job)) + }; - if let Some(meta) = &job.payload_meta { - let max_readable_payload_size = 1024 * 8; - if !params.force_payload && meta.size > max_readable_payload_size { - return Ok(FatJob { - job: job.job, - payload_meta: job.payload_meta, - payload_data: None, - }); + Ok(match params.map(|p| p.brief) { + Some(BriefMode::Yes) => BriefOrFullJob::Brief(job), + Some(BriefMode::Auto) | None => { + if job + .payload_meta + .as_ref() + .map(|m| m.size > MAX_READABLE_PAYLOAD_SIZE) + .unwrap_or(false) + { + BriefOrFullJob::Brief(job) + } else { + make_full_job(job)? + } } - } - - Ok(join_payload(job).map_err(Error::from)?) + Some(BriefMode::No) => make_full_job(job)?, + }) } pub async fn get_jobs(repo: Arc) -> EndpResult { @@ -70,15 +80,48 @@ impl Endpoints { .map_err(From::from) } - pub async fn get_agent_jobs( + pub async fn get_assigned_jobs( repo: Arc, id: Option, ) -> EndpResult { - repo.interact(move |mut db| db.get_exact_jobs(id, false)) + repo.interact(move |mut db| db.get_assigned_jobs(id, false)) .await .map_err(From::from) } + pub async fn get_payloads(repo: Arc) -> EndpResult { + repo.interact(move |mut db| db.get_payload_metas()) + .await + .map_err(From::from) + } + + pub async fn get_payload( + repo: Arc, + id: Id, + params: Option, + ) -> EndpResult { + let Some(meta) = repo.interact(move |mut db| db.get_payload_meta(id)).await? else { + return Err(not_found()) + }; + + Ok(match params.map(|p| p.brief) { + Some(BriefMode::Yes) => BriefOrFullPayload::Brief(meta), + None | Some(BriefMode::Auto) if meta.size > MAX_READABLE_PAYLOAD_SIZE => { + BriefOrFullPayload::Brief(meta) + } + _ => { + let payload_data = ufs::read(&meta.name).map_err(|e| { + error!("payload reading failed: {}", e); + Error::from(e.downcast::().expect("wrong error type")) + })?; + BriefOrFullPayload::Full(FullPayload { + meta, + data: payload_data, + }) + } + }) + } + pub async fn get_personal_jobs( repo: Arc, id: Id, @@ -104,7 +147,7 @@ impl Endpoints { } } - let assigned_jobs = db.get_exact_jobs(Some(id), true)?; + let assigned_jobs = db.get_assigned_jobs(Some(id), true)?; for job in &assigned_jobs { db.update_job_status(job.id, JobState::Running)?; @@ -121,12 +164,15 @@ impl Endpoints { pub async fn upload_jobs( repo: Arc, - msg: Vec, + msg: Vec, ) -> EndpResult { let jobs = msg .into_iter() - .map(|meta| Ok(split_payload(meta)?)) - .collect::, Error>>()?; + .map(|meta| match meta { + BriefOrFull::Full(job) => Ok(split_payload(job)?), + BriefOrFull::Brief(job) => Ok(job), + }) + .collect::, Error>>()?; repo.interact(move |mut db| db.insert_jobs(&jobs)) .await @@ -179,13 +225,13 @@ impl Endpoints { .map_err(From::from) } - pub async fn report + AsMsg + Send + Sync + 'static>( + pub async fn report( repo: Arc, - msg: Data, + msg: Vec, agent_id: Id, ) -> EndpResult { repo.transaction(move |mut db| { - for entry in msg.into_vec() { + for entry in msg { match entry { Reportable::Assigned(mut result) => { let result_agent_id = &result.agent_id; @@ -240,8 +286,14 @@ impl Endpoints { Ok(()) } - pub async fn update_job(repo: Arc, job: FatJob) -> EndpResult { - let thin_job = split_payload(job).map_err(Error::from)?; + pub async fn update_job( + repo: Arc, + job: BriefOrFullJob, + ) -> EndpResult { + let thin_job = match job { + BriefOrFullJob::Full(job) => split_payload(job).map_err(Error::from)?, + BriefOrFullJob::Brief(job) => job, + }; repo.interact(move |mut db| db.update_job(&thin_job.job)) .await?; @@ -256,8 +308,4 @@ impl Endpoints { .await?; Ok(()) } - - pub async fn download(_file_id: String) -> EndpResult> { - todo!() - } } diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 3ad73a1..5d06e96 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -1,10 +1,6 @@ #[macro_use] extern crate tracing; -#[cfg(test)] -#[macro_use] -extern crate rstest; - mod db; mod error; mod handlers; @@ -22,11 +18,13 @@ use u_lib::{ use warp::{ body, log::{custom, Info}, - reply::{json, reply, Json, Response}, + reply::{json, Json, Response}, Filter, Rejection, Reply, }; -use crate::handlers::{Endpoints, GetJobQuery}; +const DEFAULT_RESP: &str = "null"; + +use crate::handlers::{Endpoints, PayloadFlags}; fn into_message(msg: M) -> Json { json(&msg) @@ -36,8 +34,15 @@ pub fn init_endpoints( auth_token: &str, db: PgRepo, ) -> impl Filter + Clone { + fn make_optional( + f: impl Filter + Clone, + ) -> impl Filter,), Error = Infallible> + Clone { + f.map(Some) + .or_else(|_| async { Ok::<(Option,), Infallible>((None,)) }) + } + let path = |p: &'static str| warp::post().and(warp::path(p)); - let infallible_none = |_| async { Result::<(Option,), Infallible>::Ok((None,)) }; + let create_qs_cfg = || serde_qs::Config::new(1, true); let with_db = { let adb = Arc::new(db); @@ -46,20 +51,22 @@ pub fn init_endpoints( let get_agents = path("get_agents") .and(with_db.clone()) - .and(warp::path::param::().map(Some).or_else(infallible_none)) + .and(make_optional(warp::path::param::())) .and_then(Endpoints::get_agents) .map(into_message); let upload_jobs = path("upload_jobs") .and(with_db.clone()) - .and(body::json::>()) + .and(body::json::>()) .and_then(Endpoints::upload_jobs) .map(into_message); let get_job = path("get_job") .and(with_db.clone()) .and(warp::path::param::()) - .and(warp::query::()) + .and(make_optional(serde_qs::warp::query::( + create_qs_cfg(), + ))) .and_then(Endpoints::get_job) .map(into_message); @@ -68,10 +75,10 @@ pub fn init_endpoints( .and_then(Endpoints::get_jobs) .map(into_message); - let get_agent_jobs = path("get_agent_jobs") + let get_assigned_jobs = path("get_assigned_jobs") .and(with_db.clone()) - .and(warp::path::param::().map(Some).or_else(infallible_none)) - .and_then(Endpoints::get_agent_jobs) + .and(make_optional(warp::path::param::())) + .and_then(Endpoints::get_assigned_jobs) .map(into_message); let get_personal_jobs = path("get_personal_jobs") @@ -108,7 +115,7 @@ pub fn init_endpoints( let update_job = path("update_job") .and(with_db.clone()) - .and(body::json::()) + .and(body::json::()) .and_then(Endpoints::update_job) .map(ok); @@ -118,12 +125,21 @@ pub fn init_endpoints( .and_then(Endpoints::update_assigned_job) .map(ok); - let download = path("download") - .and(warp::path::param::()) - .and_then(Endpoints::download) - .map(ok); + let get_payloads = path("get_payloads") + .and(with_db.clone()) + .and_then(Endpoints::get_payloads) + .map(into_message); + + let get_payload = path("get_payload") + .and(with_db.clone()) + .and(warp::path::param::()) + .and(make_optional(serde_qs::warp::query::( + create_qs_cfg(), + ))) + .and_then(Endpoints::get_payload) + .map(into_message); - let ping = path("ping").map(reply); + let ping = path("ping").map(|| DEFAULT_RESP); let auth_token = format!("Bearer {auth_token}",).into_boxed_str(); let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); @@ -134,17 +150,14 @@ pub fn init_endpoints( .or(upload_jobs) .or(del) .or(set_jobs) - .or(get_agent_jobs) + .or(get_assigned_jobs) .or(update_agent.or(update_job).or(update_assigned_job)) - .or(download) + .or(get_payloads) + .or(get_payload) .or(ping)) .and(auth_header); - let agent_zone = get_job - .or(get_jobs) - .or(get_personal_jobs) - .or(report) - .or(download); + let agent_zone = get_job.or(get_jobs).or(get_personal_jobs).or(report); auth_zone.or(agent_zone) } @@ -154,7 +167,7 @@ pub async fn preload_jobs(repo: &PgRepo) -> Result<(), ServerError> { let job_alias = "agent_hello"; let if_job_exists = db.find_job_by_alias(job_alias)?; if if_job_exists.is_none() { - let agent_hello = RawJob::builder() + let agent_hello = RawJob::brief_job_builder() .with_type(JobType::Init) .with_alias(job_alias) .build() @@ -221,7 +234,7 @@ fn logger(info: Info<'_>) { } fn ok(_: T) -> impl Reply { - "null" + DEFAULT_RESP } /* diff --git a/integration/Cargo.lock b/integration-tests/Cargo.lock similarity index 100% rename from integration/Cargo.lock rename to integration-tests/Cargo.lock diff --git a/integration/Cargo.toml b/integration-tests/Cargo.toml similarity index 90% rename from integration/Cargo.toml rename to integration-tests/Cargo.toml index 5639c21..eedeb02 100644 --- a/integration/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "integration" +name = "integration-tests" version = "0.1.0" authors = ["plazmoid "] edition = "2021" @@ -7,6 +7,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +ctor = "0.2.0" once_cell = "1.10.0" reqwest = { workspace = true } rstest = "0.17" @@ -20,5 +21,5 @@ u_lib = { path = "../lib/u_lib", features = ["panel", "server"] } [[test]] -name = "integration" +name = "integration-tests" path = "tests/lib.rs" diff --git a/integration/docker-compose.yml b/integration-tests/docker-compose.yml similarity index 81% rename from integration/docker-compose.yml rename to integration-tests/docker-compose.yml index 8d82795..db44663 100644 --- a/integration/docker-compose.yml +++ b/integration-tests/docker-compose.yml @@ -30,10 +30,10 @@ services: environment: RUST_LOG: warp=info,u_server_lib=debug healthcheck: - test: ss -tlpn | grep 63714 - interval: 5s - timeout: 2s - retries: 2 + test: ss -tlpn | grep 63714 + interval: 5s + timeout: 2s + retries: 2 u_db: image: localhost/unki/u_db @@ -51,11 +51,11 @@ services: target: /u_db_entrypoint.sh command: /u_db_entrypoint.sh healthcheck: - # test if db's port is open and db is created - test: ss -tlpn | grep 5432 && psql -lqt -U $${POSTGRES_USER} | grep -qw $${POSTGRES_DATABASE} - interval: 5s - timeout: 5s - retries: 3 + # test if db's port is open and db is created + test: ss -tlpn | grep 5432 && psql -lqt -U $${POSTGRES_USER} | grep -qw $${POSTGRES_DATABASE} + interval: 5s + timeout: 5s + retries: 3 u_agent: user: *user @@ -83,13 +83,13 @@ services: volumes: - ${HOME}/.cargo/registry/:/usr/local/cargo/registry/ - ../__Cargo_integration.toml:/tests/Cargo.toml - - ./:/tests/integration/ + - ./:/tests/integration-tests/ - ../certs:/tests/certs - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_panel:/u_panel - ../lib/u_lib:/tests/lib/u_lib - - ../logs:/tests/integration/logs:rw + - ../logs:/tests/integration-tests/logs:rw working_dir: - /tests/integration/ + /tests/integration-tests/ depends_on: u_agent: condition: service_started @@ -100,4 +100,5 @@ services: - ../.env.private environment: RUST_BACKTRACE: 1 + RUST_LOG: debug,hyper=info,reqwest=info U_SERVER: u_server \ No newline at end of file diff --git a/integration/docker.py b/integration-tests/docker.py similarity index 98% rename from integration/docker.py rename to integration-tests/docker.py index 2103103..06a4fa6 100644 --- a/integration/docker.py +++ b/integration-tests/docker.py @@ -90,7 +90,7 @@ class Compose: ] def __init__(self): - self.container_tpl = 'integration-%s-%d' + self.container_tpl = 'integration-tests-%s-%d' self.cmd_container = self.container_tpl % ('tests_runner', 1) self.ALL_CONTAINERS = [self.container_tpl % (c, 1) for c in self.ALL_IMAGES] diff --git a/integration/integration_tests.py b/integration-tests/integration_tests.py similarity index 75% rename from integration/integration_tests.py rename to integration-tests/integration_tests.py index 4b1ab63..bc98e33 100644 --- a/integration/integration_tests.py +++ b/integration-tests/integration_tests.py @@ -6,7 +6,7 @@ from docker import rebuild_images_if_needed, Compose from pathlib import Path from utils import * -CARGO_INTEGRATION_TOML = Path('../__Cargo_integration.toml') +CARGO_INTEGRATION_TESTS_TOML = Path('../__Cargo_integration.toml') CLUSTER = Compose() @@ -22,12 +22,12 @@ def usage_exit(): fail(usage) -def create_integration_workspace(): - if CARGO_INTEGRATION_TOML.exists(): - CARGO_INTEGRATION_TOML.unlink() +def create_integration_tests_workspace(): + if CARGO_INTEGRATION_TESTS_TOML.exists(): + CARGO_INTEGRATION_TESTS_TOML.unlink() workspace = toml.load('../Cargo.toml') - workspace['workspace']['members'] = ['integration'] - with open(CARGO_INTEGRATION_TOML, 'w') as fo: + workspace['workspace']['members'] = ['integration-tests'] + with open(CARGO_INTEGRATION_TESTS_TOML, 'w') as fo: toml.dump(workspace, fo) @@ -44,7 +44,7 @@ def run_tests(): def _cleanup(): if not preserve_containers and not only_setup_cluster: CLUSTER.down() - CARGO_INTEGRATION_TOML.unlink(missing_ok=True) + CARGO_INTEGRATION_TESTS_TOML.unlink(missing_ok=True) def abort_handler(s, _): warn(f'Received signal: {s}, gracefully stopping...') @@ -53,16 +53,16 @@ def run_tests(): if down_cluster: _cleanup() return - + for s in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): signal.signal(s, abort_handler) rebuild_images_if_needed(force_rebuild) - create_integration_workspace() + create_integration_tests_workspace() try: CLUSTER.up() CLUSTER.is_alive() if not only_setup_cluster: - CLUSTER.run('cargo test --test integration') + CLUSTER.run('cargo test --test integration-tests') except Exception as e: #CLUSTER.print_containers_logs() fail(e) diff --git a/integration/integration_tests.sh b/integration-tests/integration_tests.sh similarity index 86% rename from integration/integration_tests.sh rename to integration-tests/integration_tests.sh index 0b3c416..adf3097 100755 --- a/integration/integration_tests.sh +++ b/integration-tests/integration_tests.sh @@ -3,6 +3,6 @@ set -e export DOCKER_UID=$(id -u) export DOCKER_GID=$(id -g) -rm ../logs/u_agent* +rm ../logs/u_agent* || true [[ "$@" =~ "--release" ]] && export PROFILE=release || export PROFILE=debug python integration_tests.py $@ diff --git a/integration/src/main.rs b/integration-tests/src/main.rs similarity index 100% rename from integration/src/main.rs rename to integration-tests/src/main.rs diff --git a/integration/tests/fixtures/agent.rs b/integration-tests/tests/fixtures/agent.rs similarity index 78% rename from integration/tests/fixtures/agent.rs rename to integration-tests/tests/fixtures/agent.rs index 09a8e30..f8264c8 100644 --- a/integration/tests/fixtures/agent.rs +++ b/integration-tests/tests/fixtures/agent.rs @@ -1,5 +1,6 @@ use super::connections::*; use super::run_async; +use u_lib::unwrap_enum; use u_lib::{api::HttpClient, jobs::split_payload, messaging::Reportable, models::*, types::Id}; pub struct RegisteredAgent { @@ -13,6 +14,7 @@ pub fn registered_agent(client: &HttpClient) -> RegisteredAgent { let agent = Agent::with_current_platform(); let agent_id = agent.id; println!("registering agent {agent_id}"); + debug!("registering agent1 {agent_id}"); let resp = client .get_personal_jobs(agent_id) .await @@ -20,12 +22,13 @@ pub fn registered_agent(client: &HttpClient) -> RegisteredAgent { .pop() .unwrap(); let job_id = resp.job_id; - let job = client.get_job(job_id, true).await.unwrap(); + let job = client.get_job(job_id, BriefMode::No).await.unwrap(); + let job = unwrap_enum!(job, BriefOrFull::Full); assert_eq!(job.job.alias, Some("agent_hello".to_string())); let mut agent_data = AssignedJob::from((&split_payload(job).unwrap().job, resp)); agent_data.set_result(&agent); client - .report(&Reportable::Assigned(agent_data)) + .report([Reportable::Assigned(agent_data)]) .await .unwrap(); RegisteredAgent { id: agent_id } diff --git a/integration/tests/fixtures/connections.rs b/integration-tests/tests/fixtures/connections.rs similarity index 100% rename from integration/tests/fixtures/connections.rs rename to integration-tests/tests/fixtures/connections.rs diff --git a/integration/tests/fixtures/env.rs b/integration-tests/tests/fixtures/env.rs similarity index 100% rename from integration/tests/fixtures/env.rs rename to integration-tests/tests/fixtures/env.rs diff --git a/integration/tests/fixtures/mod.rs b/integration-tests/tests/fixtures/mod.rs similarity index 100% rename from integration/tests/fixtures/mod.rs rename to integration-tests/tests/fixtures/mod.rs diff --git a/integration/tests/helpers/jobs.rs b/integration-tests/tests/helpers/jobs.rs similarity index 100% rename from integration/tests/helpers/jobs.rs rename to integration-tests/tests/helpers/jobs.rs diff --git a/integration/tests/helpers/mod.rs b/integration-tests/tests/helpers/mod.rs similarity index 100% rename from integration/tests/helpers/mod.rs rename to integration-tests/tests/helpers/mod.rs diff --git a/integration/tests/helpers/panel.rs b/integration-tests/tests/helpers/panel.rs similarity index 87% rename from integration/tests/helpers/panel.rs rename to integration-tests/tests/helpers/panel.rs index 5d3a1b7..59a7c94 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration-tests/tests/helpers/panel.rs @@ -10,13 +10,25 @@ pub struct Panel; impl Panel { fn run(args: &[&str]) -> Output { - Command::new(PANEL_BINARY).args(args).output().unwrap() + Command::new(PANEL_BINARY) + .env("RUST_LOG", "u_lib=debug") + .args(args) + .output() + .unwrap() } pub fn output_argv(argv: &[&str]) -> PanelResult { let result = Self::run(argv); let output = ProcOutput::from_output(&result); + let stderr = output.get_stderr(); + if !stderr.is_empty() { + println!( + "\n*** PANEL DEBUG OUTPUT START***\n{}\n*** PANEL DEBUG OUTPUT END ***\n", + String::from_utf8_lossy(stderr) + ); + } + match from_slice(output.get_stdout()) { Ok(r) => r, Err(e) => { diff --git a/integration/tests/integration/api.rs b/integration-tests/tests/integration_tests/api.rs similarity index 59% rename from integration/tests/integration/api.rs rename to integration-tests/tests/integration_tests/api.rs index 17d42cd..7051e9a 100644 --- a/integration/tests/integration/api.rs +++ b/integration-tests/tests/integration_tests/api.rs @@ -14,35 +14,39 @@ // ping(&self) use crate::fixtures::connections::*; -use u_lib::jobs::join_payload; -use u_lib::models::RawJob; +use u_lib::models::{BriefOrFullJob, RawJob}; #[rstest] #[tokio::test] async fn test_jobs_endpoints(client_panel: &HttpClient) { let job_alias = "henlo"; - let job = RawJob::builder() + let mut job = RawJob::brief_job_builder() .with_shell("echo henlo") .with_alias(job_alias) .build() .unwrap(); let job_id = job.job.id; - let mut fat_job = join_payload(job).unwrap(); - client_panel.upload_jobs(&fat_job).await.unwrap(); + client_panel + .upload_jobs([&BriefOrFullJob::Brief(job.clone())]) + .await + .unwrap(); - let fetched_job = client_panel.get_job(job_id, false).await.unwrap(); - assert_eq!(fat_job, fetched_job); + let fetched_job = client_panel.get_brief_job(job_id).await.unwrap(); + assert_eq!(job, fetched_job); - fat_job.job.alias = Some("henlo2".to_string()); - client_panel.update_job(&fat_job).await.unwrap(); + job.job.alias = Some("henlo2".to_string()); + client_panel + .update_job(&BriefOrFullJob::Brief(job.clone())) + .await + .unwrap(); - let fetched_job = client_panel.get_job(job_id, false).await.unwrap(); - assert_eq!(fat_job, fetched_job); + let fetched_job = client_panel.get_brief_job(job_id).await.unwrap(); + assert_eq!(job, fetched_job); client_panel.del(job_id).await.unwrap(); - let not_found_err = client_panel.get_job(job_id, false).await.unwrap_err(); + let not_found_err = client_panel.get_brief_job(job_id).await.unwrap_err(); assert!(not_found_err.to_string().contains("404 Not Found")) } diff --git a/integration/tests/integration/behaviour.rs b/integration-tests/tests/integration_tests/behaviour.rs similarity index 96% rename from integration/tests/integration/behaviour.rs rename to integration-tests/tests/integration_tests/behaviour.rs index 51ea208..99facbb 100644 --- a/integration/tests/integration/behaviour.rs +++ b/integration-tests/tests/integration_tests/behaviour.rs @@ -21,7 +21,7 @@ async fn setup_tasks() { let agents: Vec = Panel::check_output("agents read"); let agent_id = agents[0].id; let job_alias = "passwd_contents"; - let job = RawJob::builder() + let job = RawJob::brief_job_builder() .with_alias(job_alias) .with_raw_payload(b"cat /etc/passwd".as_slice()) .with_shell("/bin/bash {}") @@ -54,7 +54,7 @@ async fn large_payload() { let agent = &Panel::check_output::>("agents read")[0]; let agent_id = agent.id; let job_alias = "large_payload"; - let job = RawJob::builder() + let job = RawJob::brief_job_builder() .with_alias(job_alias) .with_payload_path("./tests/bin/echoer") .with_shell("{} type echo") diff --git a/integration/tests/integration/connection.rs b/integration-tests/tests/integration_tests/connection.rs similarity index 100% rename from integration/tests/integration/connection.rs rename to integration-tests/tests/integration_tests/connection.rs diff --git a/integration/tests/integration/mod.rs b/integration-tests/tests/integration_tests/mod.rs similarity index 100% rename from integration/tests/integration/mod.rs rename to integration-tests/tests/integration_tests/mod.rs diff --git a/integration-tests/tests/lib.rs b/integration-tests/tests/lib.rs new file mode 100644 index 0000000..0f10c3a --- /dev/null +++ b/integration-tests/tests/lib.rs @@ -0,0 +1,14 @@ +mod fixtures; +mod helpers; +mod integration_tests; + +#[macro_use] +extern crate rstest; + +#[macro_use] +extern crate tracing; + +#[ctor::ctor] +fn __init() { + u_lib::logging::init_logger(None); +} diff --git a/integration/utils.py b/integration-tests/utils.py similarity index 100% rename from integration/utils.py rename to integration-tests/utils.py diff --git a/integration/tests/lib.rs b/integration/tests/lib.rs deleted file mode 100644 index 826b82d..0000000 --- a/integration/tests/lib.rs +++ /dev/null @@ -1,6 +0,0 @@ -mod fixtures; -mod helpers; -mod integration; - -#[macro_use] -extern crate rstest; diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index d656832..2a21165 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -7,11 +7,11 @@ use reqwest::{header, header::HeaderMap, Certificate, Client, Identity, Method, use serde::de::DeserializeOwned; use serde_json::{from_str, Value}; +use crate::unwrap_enum; use crate::{ config::{get_self_id, MASTER_PORT}, conv::opt_to_string, messaging::{self, AsMsg}, - misc::OneOrVecRef, models::*, types::Id, UError, UResult, @@ -25,7 +25,9 @@ pub mod retypes { pub type GetPersonalJobs = Vec; pub type Report = (); - pub type GetJob = FatJob; + pub type GetJob = BriefOrFullJob; + pub type GetFullJob = FullJob; + pub type GetBriefJob = BriefJob; pub type GetJobs = Vec; pub type GetAgents = Vec; pub type UpdateAgent = (); @@ -36,6 +38,8 @@ pub mod retypes { pub type SetJobs = Vec; pub type GetAgentJobs = Vec; pub type Ping = (); + pub type GetPayloads = Vec; + pub type GetPayload = BriefOrFullPayload; } #[derive(Clone, Debug)] @@ -109,6 +113,8 @@ impl HttpClient { .post(self.base_url.join(url).unwrap()) .json(payload); + debug!("url = {url}"); + let response = request .send() .await @@ -130,34 +136,47 @@ impl HttpClient { } .map_err(From::from); - debug!("url = {}, resp = {:?}", url, result); + debug!("response = {:?}", result); result } - // get jobs for client - pub async fn get_personal_jobs(&self, agent_id: Id) -> Result> { + /// get jobs for agent + pub async fn get_personal_jobs(&self, agent_id: Id) -> Result { self.req(format!("get_personal_jobs/{}", agent_id)).await } - // send something to server - pub async fn report(&self, payload: impl OneOrVecRef) -> Result<()> { - self.req_with_payload("report", &payload.as_vec()).await + /// send something to server + pub async fn report( + &self, + payload: impl IntoIterator, + ) -> Result { + self.req_with_payload("report", &payload.into_iter().collect::>()) + .await } - // download payload - pub async fn dl(&self, file: &str) -> Result> { + /// download payload + pub async fn _dl(&self, file: &str) -> Result> { self.req(format!("dl/{file}")).await } /// get exact job - pub async fn get_job(&self, job: Id, force_payload: bool) -> Result { - self.req(format!("get_job/{job}?force_payload={force_payload}")) - .await + pub async fn get_job(&self, job: Id, brief: BriefMode) -> Result { + self.req(format!("get_job/{job}?brief={brief}")).await + } + + pub async fn get_full_job(&self, job: Id) -> Result { + let job = self.get_job(job, BriefMode::No).await?; + Ok(unwrap_enum!(job, BriefOrFullJob::Full)) + } + + pub async fn get_brief_job(&self, job: Id) -> Result { + let job = self.get_job(job, BriefMode::Yes).await?; + Ok(unwrap_enum!(job, BriefOrFullJob::Brief)) } /// get all available jobs - pub async fn get_jobs(&self) -> Result> { + pub async fn get_jobs(&self) -> Result { self.req("get_jobs").await } } @@ -166,34 +185,37 @@ impl HttpClient { #[cfg(feature = "panel")] impl HttpClient { /// agent listing - pub async fn get_agents(&self, agent: Option) -> Result> { + pub async fn get_agents(&self, agent: Option) -> Result { self.req(format!("get_agents/{}", opt_to_string(agent))) .await } /// update agent - pub async fn update_agent(&self, agent: &Agent) -> Result<()> { + pub async fn update_agent(&self, agent: &Agent) -> Result { self.req_with_payload("update_agent", agent).await } /// update job - pub async fn update_job(&self, job: &FatJob) -> Result<()> { + pub async fn update_job(&self, job: &BriefOrFullJob) -> Result { self.req_with_payload("update_job", job).await } /// update result - pub async fn update_result(&self, result: &AssignedJob) -> Result<()> { + pub async fn update_result(&self, result: &AssignedJob) -> Result { self.req_with_payload("update_result", result).await } /// create and upload job - pub async fn upload_jobs(&self, payload: impl OneOrVecRef) -> Result<()> { - self.req_with_payload("upload_jobs", &payload.as_vec()) + pub async fn upload_jobs( + &self, + payload: impl IntoIterator, + ) -> Result { + self.req_with_payload("upload_jobs", &payload.into_iter().collect::>()) .await } /// delete something - pub async fn del(&self, item: Id) -> Result<()> { + pub async fn del(&self, item: Id) -> Result { self.req(format!("del/{item}")).await } @@ -201,19 +223,34 @@ impl HttpClient { pub async fn set_jobs( &self, agent: Id, - job_idents: impl OneOrVecRef, - ) -> Result> { - self.req_with_payload(format!("set_jobs/{agent}"), &job_idents.as_vec()) - .await + job_idents: impl IntoIterator>, + ) -> Result { + self.req_with_payload( + format!("set_jobs/{agent}"), + &job_idents + .into_iter() + .map(|i| i.into()) + .collect::>(), + ) + .await } /// get jobs for any agent - pub async fn get_agent_jobs(&self, agent: Option) -> Result> { - self.req(format!("get_agent_jobs/{}", opt_to_string(agent))) + pub async fn get_assigned_jobs(&self, agent: Option) -> Result { + self.req(format!("get_assigned_jobs/{}", opt_to_string(agent))) + .await + } + + pub async fn get_payloads(&self) -> Result { + self.req("get_payloads").await + } + + pub async fn get_payload(&self, payload: Id, brief: BriefMode) -> Result { + self.req(format!("get_payload/{payload}?brief={brief}")) .await } - pub async fn ping(&self) -> Result<()> { + pub async fn ping(&self) -> Result { self.req("ping").await } } diff --git a/lib/u_lib/src/cache.rs b/lib/u_lib/src/cache.rs index c025db4..3d8f4b4 100644 --- a/lib/u_lib/src/cache.rs +++ b/lib/u_lib/src/cache.rs @@ -1,10 +1,10 @@ -use crate::models::ThinJob; +use crate::models::BriefJob; use crate::types::Id; use lazy_static::lazy_static; use parking_lot::{RwLock, RwLockReadGuard}; use std::{collections::HashMap, ops::Deref}; -type Val = ThinJob; +type Val = BriefJob; type Cache = HashMap; lazy_static! { diff --git a/lib/u_lib/src/combined_result.rs b/lib/u_lib/src/combined_result.rs index 2c1c4bb..6aee4a9 100644 --- a/lib/u_lib/src/combined_result.rs +++ b/lib/u_lib/src/combined_result.rs @@ -1,6 +1,5 @@ use std::fmt::Debug; -use crate::misc::OneOrVec; use anyhow::Error; pub struct CombinedResult { @@ -16,17 +15,12 @@ impl CombinedResult { } } - pub fn ok(&mut self, result: impl OneOrVec) { - self.ok.extend(result.into_vec()); + pub fn push_ok(&mut self, result: T) { + self.ok.push(result); } - pub fn err>(&mut self, err: impl OneOrVec) { - self.err.extend( - err.into_vec() - .into_iter() - .map(Into::into) - .collect::>(), - ); + pub fn push_err(&mut self, err: impl Into) { + self.err.push(err.into()); } pub fn unwrap(self) -> Vec { diff --git a/lib/u_lib/src/error/mod.rs b/lib/u_lib/src/error/mod.rs index b9678a2..929d7fd 100644 --- a/lib/u_lib/src/error/mod.rs +++ b/lib/u_lib/src/error/mod.rs @@ -3,8 +3,8 @@ mod chan; pub use chan::*; use crate::ufs; -use reqwest::Error as ReqError; use serde::{Deserialize, Serialize}; +use std::io; use thiserror::Error; use uuid::Uuid; @@ -33,6 +33,9 @@ pub enum UError { #[error(transparent)] FSError(#[from] ufs::Error), + #[error("I/O error: {0}")] + IOError(String), + #[error("Wrong auth token")] WrongToken, @@ -45,12 +48,12 @@ pub enum UError { #[error("Deserialize from json error: {0}, body: {1}")] DeserializeError(String, String), - #[error("{0}\n{1}")] + #[error("{0}\nContext: {1}")] Contexted(Box, String), } -impl From for UError { - fn from(e: ReqError) -> Self { +impl From for UError { + fn from(e: reqwest::Error) -> Self { UError::NetError( e.to_string(), e.url().map(|u| u.to_string()).unwrap_or_default(), @@ -59,6 +62,12 @@ impl From for UError { } } +impl From for UError { + fn from(err: io::Error) -> Self { + UError::IOError(err.to_string()) + } +} + impl From for UError { fn from(e: anyhow::Error) -> Self { let ctx = e @@ -73,7 +82,10 @@ impl From for UError { Ok(err) => UError::Contexted(Box::new(err), ctx), Err(err) => match err.downcast::() { Ok(err) => UError::Contexted(Box::new(UError::FSError(err)), ctx), - Err(err) => UError::Runtime(err.to_string()), + Err(err) => match err.downcast::() { + Ok(err) => UError::Contexted(Box::new(UError::from(err)), ctx), + Err(err) => UError::Runtime(err.to_string()), + }, }, } } diff --git a/lib/u_lib/src/jobs.rs b/lib/u_lib/src/jobs.rs index 94c2c0f..5c01b91 100644 --- a/lib/u_lib/src/jobs.rs +++ b/lib/u_lib/src/jobs.rs @@ -1,8 +1,9 @@ use crate::{ combined_result::CombinedResult, executor::{ExecResult, Waiter}, - misc::OneOrVec, - models::{Agent, AssignedJob, AssignedJobById, FatJob, JobType, RawJob, ThinJob}, + models::{ + Agent, AssignedJob, AssignedJobById, BriefJob, FullJob, FullPayload, JobType, RawJob, + }, proc_output::ProcOutput, ufs, }; @@ -16,9 +17,9 @@ pub struct AnonymousJobBatch { } impl AnonymousJobBatch { - pub fn from_meta_with_id(jobs: impl OneOrVec<(ThinJob, AssignedJobById)>) -> Self { + pub fn from_meta_with_id(jobs: impl IntoIterator) -> Self { let mut waiter = Waiter::new(); - for (job, ids) in jobs.into_vec() { + for (job, ids) in jobs { waiter.push(run_assigned_job(job, ids)); } Self { @@ -27,9 +28,8 @@ impl AnonymousJobBatch { } } - pub fn from_meta(jobs: impl OneOrVec) -> Self { + pub fn from_meta(jobs: impl IntoIterator) -> Self { let jobs_ids: Vec<_> = jobs - .into_vec() .into_iter() .map(|job| { let job_id = job.job.id; @@ -76,30 +76,30 @@ pub struct NamedJobBatch { } impl NamedJobBatch { - pub fn from_shell( - named_jobs: impl OneOrVec<(&'static str, &'static str)>, - ) -> CombinedResult { + pub fn from_shell(named_jobs: Vec<(&'static str, &'static str)>) -> CombinedResult { let mut result = CombinedResult::new(); let jobs: Vec<_> = named_jobs - .into_vec() .into_iter() .filter_map(|(alias, cmd)| { - match RawJob::builder().with_shell(cmd).with_alias(alias).build() { + match RawJob::brief_job_builder() + .with_shell(cmd) + .with_alias(alias) + .build() + { Ok(jpm) => Some(jpm), Err(e) => { - result.err(e); + result.push_err(e); None } } }) .collect(); - result.ok(Self::from_meta(jobs)); + result.push_ok(Self::from_meta(jobs)); result } - pub fn from_meta(named_jobs: impl OneOrVec) -> Self { + pub fn from_meta(named_jobs: Vec) -> Self { let (job_names, jobs): (Vec<_>, Vec<_>) = named_jobs - .into_vec() .into_iter() .map(|job| (job.job.alias.clone().unwrap(), job)) .unzip(); @@ -134,8 +134,8 @@ impl NamedJobBatch { } } -pub async fn run_assigned_job(job: ThinJob, ids: AssignedJobById) -> ExecResult { - let ThinJob { job, payload_meta } = job; +pub async fn run_assigned_job(job: BriefJob, ids: AssignedJobById) -> ExecResult { + let BriefJob { job, payload_meta } = job; let mut result = AssignedJob::from((&job, ids)); match job.exec_type { JobType::Shell => { @@ -181,40 +181,38 @@ pub async fn run_assigned_job(job: ThinJob, ids: AssignedJobById) -> ExecResult Ok(result) } -pub fn split_payload(job: FatJob) -> Result { - let FatJob { - job, - payload_meta, - payload_data, - } = job; - - if let Some(meta) = &payload_meta { - if let Some(data) = payload_data { - if ufs::in_index(&meta.name) { - ufs::edit(&meta.name, data)?; - } else { - ufs::put(&meta.name, data)?; - } +pub fn split_payload(job: FullJob) -> Result { + let FullJob { job, payload } = job; + + if let Some(payload) = &payload { + if ufs::exists_in_index(&payload.meta.name) { + ufs::edit(&payload.meta.name, &payload.data)?; + } else { + ufs::put(&payload.meta.name, &payload.data)?; } } - Ok(ThinJob { job, payload_meta }) -} - -pub fn join_payload(job: ThinJob) -> Result { - let ThinJob { job, payload_meta } = job; - let payload_data = payload_meta - .as_ref() - .map(|p| ufs::read(&p.name)) - .transpose()?; - - Ok(FatJob { + Ok(BriefJob { job, - payload_meta, - payload_data, + payload_meta: payload.map(|p| p.meta), }) } +pub fn join_payload(job: BriefJob) -> Result { + let BriefJob { job, payload_meta } = job; + let payload = match payload_meta { + Some(meta) => { + let payload_data = ufs::read(&meta.name)?; + Some(FullPayload { + meta, + data: payload_data, + }) + } + None => None, + }; + Ok(FullJob { job, payload }) +} + #[cfg(test)] mod tests { use crate::{ @@ -267,12 +265,15 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] expected_result: &str, ) -> TestResult { - let mut job = RawJob::builder().with_shell(cmd); + let mut job = RawJob::brief_job_builder().with_shell(cmd); if let Some(p) = payload { job = job.with_raw_payload(p); } let job = job.build().unwrap(); - let result = AnonymousJobBatch::from_meta(job).wait_one().await.unwrap(); + let result = AnonymousJobBatch::from_meta([job]) + .wait_one() + .await + .unwrap(); let result = result.to_str_result(); assert_eq!(result.trim(), expected_result); Ok(()) @@ -283,8 +284,8 @@ mod tests { const SLEEP_SECS: u64 = 1; let now = SystemTime::now(); let longest_job = RawJob::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); - let longest_job = AnonymousJobBatch::from_meta(longest_job).spawn().await; - let ls = AnonymousJobBatch::from_meta(RawJob::from_shell("ls").unwrap()) + let longest_job = AnonymousJobBatch::from_meta([longest_job]).spawn().await; + let ls = AnonymousJobBatch::from_meta([RawJob::from_shell("ls").unwrap()]) .wait_one() .await .unwrap(); @@ -328,7 +329,10 @@ mod tests { #[tokio::test] async fn test_failing_shell_job() -> TestResult { let job = RawJob::from_shell("lol_kek_puk").unwrap(); - let job_result = AnonymousJobBatch::from_meta(job).wait_one().await.unwrap(); + let job_result = AnonymousJobBatch::from_meta([job]) + .wait_one() + .await + .unwrap(); let output = job_result.to_str_result(); assert!(output.contains("No such file")); assert!(job_result.retcode.is_none()); @@ -344,7 +348,7 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] err_str: &str, ) -> TestResult { - let mut job = RawJob::builder().with_shell(cmd); + let mut job = RawJob::brief_job_builder().with_shell(cmd); if let Some(p) = payload { job = job.with_raw_payload(p); } @@ -357,12 +361,12 @@ mod tests { #[tokio::test] async fn test_different_job_types() -> TestResult { let mut jobs = NamedJobBatch::from_meta(vec![ - RawJob::builder() + RawJob::brief_job_builder() .with_shell("sleep 3") .with_alias("sleeper") .build() .unwrap(), - RawJob::builder() + RawJob::brief_job_builder() .with_type(JobType::Init) .with_alias("gatherer") .build() diff --git a/lib/u_lib/src/logging.rs b/lib/u_lib/src/logging.rs index 9c4f1db..ae32a4f 100644 --- a/lib/u_lib/src/logging.rs +++ b/lib/u_lib/src/logging.rs @@ -1,28 +1,35 @@ use std::env; +use std::io::{stderr, stdout}; use std::path::Path; use tracing_appender::rolling; use tracing_subscriber::{fmt, prelude::*, registry, EnvFilter}; -pub fn init_logger(logfile: Option + Send + Sync + 'static>) { +pub fn init_logger(logfile: Option<&str>) { if env::var("RUST_LOG").is_err() { env::set_var("RUST_LOG", "info") } + let output_layer = if cfg!(test) { + fmt::layer().with_writer(stdout).with_test_writer().boxed() + } else { + fmt::layer().with_writer(stderr).boxed() + }; + let reg = registry() .with(EnvFilter::from_default_env()) - .with(fmt::layer()); + .with(output_layer); match logfile { - Some(file) => reg - .with( + Some(file) => { + let file_path = Path::new(file).with_extension("log"); + reg.with( fmt::layer() - .with_writer(move || { - rolling::never("logs", file.as_ref().with_extension("log")) - }) + .with_writer(move || rolling::never("logs", &file_path)) .with_ansi(false), ) - .init(), + .init() + } None => reg.init(), }; } diff --git a/lib/u_lib/src/messaging.rs b/lib/u_lib/src/messaging.rs index 816e12f..cea2be9 100644 --- a/lib/u_lib/src/messaging.rs +++ b/lib/u_lib/src/messaging.rs @@ -11,13 +11,15 @@ impl AsMsg for Agent {} impl AsMsg for AssignedJob {} impl AsMsg for AssignedJobById {} impl AsMsg for JobModel {} -impl AsMsg for FatJob {} +impl AsMsg for FullJob {} impl AsMsg for Reportable {} -impl AsMsg for String {} -impl AsMsg for ThinJob {} +impl AsMsg for PayloadMeta {} +impl AsMsg for FullPayload {} +impl AsMsg for BriefJob {} +impl AsMsg for BriefOrFull {} impl AsMsg for Id {} -impl AsMsg for i32 {} -impl AsMsg for u8 {} +impl AsMsg for String {} +impl AsMsg for Vec {} impl AsMsg for () {} impl AsMsg for Vec {} diff --git a/lib/u_lib/src/misc.rs b/lib/u_lib/src/misc.rs index 29723a5..2e280cb 100644 --- a/lib/u_lib/src/misc.rs +++ b/lib/u_lib/src/misc.rs @@ -1,35 +1,3 @@ -pub trait OneOrVec { - fn into_vec(self) -> Vec; -} - -impl OneOrVec for T { - fn into_vec(self) -> Vec { - vec![self] - } -} - -impl OneOrVec for Vec { - fn into_vec(self) -> Vec { - self - } -} - -pub trait OneOrVecRef { - fn as_vec(&self) -> Vec<&T>; -} - -impl OneOrVecRef for &T { - fn as_vec(&self) -> Vec<&T> { - vec![self] - } -} - -impl OneOrVecRef for &Vec { - fn as_vec(&self) -> Vec<&T> { - self.iter().collect() - } -} - #[macro_export] macro_rules! unwrap_enum { ($src:expr, $t:path) => { diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index bad92b4..6039ad1 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -1,10 +1,9 @@ use std::fmt; use super::JobType; -use crate::conv::bytes_to_string; #[cfg(feature = "server")] use crate::models::schema::*; -use crate::models::PayloadMeta; +use crate::models::{FullPayload, PayloadMeta}; use crate::platform; use crate::types::Id; use crate::{ufs, UError, UResult}; @@ -12,7 +11,6 @@ use crate::{ufs, UError, UResult}; use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; use std::borrow::Cow; -use std::process::Command; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr( @@ -35,14 +33,13 @@ pub struct JobModel { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct FatJob { +pub struct FullJob { pub job: JobModel, - pub payload_meta: Option, - pub payload_data: Option>, + pub payload: Option, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct ThinJob { +pub struct BriefJob { pub job: JobModel, pub payload_meta: Option, } @@ -108,9 +105,9 @@ impl fmt::Debug for RawJob<'_> { } } -impl From for RawJob<'_> { - fn from(job: ThinJob) -> Self { - let ThinJob { job, payload_meta } = job; +impl From for RawJob<'_> { + fn from(job: BriefJob) -> Self { + let BriefJob { job, payload_meta } = job; RawJob { alias: job.alias, argv: job.argv, @@ -125,15 +122,15 @@ impl From for RawJob<'_> { } impl<'p> RawJob<'p> { - pub fn validated(self) -> UResult { + pub fn validated(self) -> UResult { JobBuilder { inner: self }.build() } - pub fn from_shell(cmd: impl Into) -> UResult { - Self::builder().with_shell(cmd).build() + pub fn from_shell(cmd: impl Into) -> UResult { + Self::brief_job_builder().with_shell(cmd).build() } - pub fn builder() -> JobBuilder<'p> { + pub fn brief_job_builder() -> JobBuilder<'p> { JobBuilder::default() } } @@ -177,43 +174,27 @@ impl<'p> JobBuilder<'p> { self } - pub fn build(self) -> UResult { + pub fn build(self) -> UResult { let mut inner = self.inner; - let raw_into_job = |raw: RawJob| -> UResult { - let payload_id = raw.payload_path.as_ref().map(|_| Id::new_v4()); + let raw_into_job = |raw: RawJob| -> UResult { + let payload_meta = raw + .payload_path + .as_ref() + .map(|payload_ident| PayloadMeta::from_existing_meta(payload_ident)) + .transpose()?; - Ok(ThinJob { + Ok(BriefJob { job: JobModel { alias: raw.alias, argv: raw.argv, id: raw.id, exec_type: raw.exec_type, target_platforms: raw.target_platforms, - payload: payload_id, + payload: payload_meta.as_ref().map(|meta| meta.id), schedule: raw.schedule, }, - payload_meta: raw - .payload_path - .map(|payload_ident| { - let ufs_meta = ufs::read_meta(&payload_ident)?; - let payload_meta = PayloadMeta { - id: payload_id.unwrap(), - mime_type: bytes_to_string( - &Command::new("file") - .arg("-b") - .arg("--mime-type") - .arg(&ufs_meta.path) - .output() - .map_err(|e| UError::JobBuildError(e.to_string()))? - .stdout, - ), - name: payload_ident.clone(), - size: ufs_meta.size as i64, - }; - Ok::<_, UError>(payload_meta) - }) - .transpose()?, + payload_meta, }) }; diff --git a/lib/u_lib/src/models/mod.rs b/lib/u_lib/src/models/mod.rs index 3000f8b..e8f315d 100644 --- a/lib/u_lib/src/models/mod.rs +++ b/lib/u_lib/src/models/mod.rs @@ -4,4 +4,25 @@ mod payload; #[cfg(feature = "server")] pub mod schema; +use crate::messaging::AsMsg; pub use crate::models::{agent::*, jobs::*, payload::*}; +use serde::{Deserialize, Serialize}; +use strum::{Display as StrumDisplay, EnumString}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum BriefOrFull { + Brief(B), + Full(F), +} + +pub type BriefOrFullJob = BriefOrFull; +pub type BriefOrFullPayload = BriefOrFull; + +#[derive(Default, Debug, StrumDisplay, EnumString, Deserialize)] +pub enum BriefMode { + Yes, + #[default] + Auto, + No, +} diff --git a/lib/u_lib/src/models/payload.rs b/lib/u_lib/src/models/payload.rs index 6cac041..88cf6fa 100644 --- a/lib/u_lib/src/models/payload.rs +++ b/lib/u_lib/src/models/payload.rs @@ -1,5 +1,6 @@ -use crate::types::Id; +use crate::{conv::bytes_to_string, types::Id, ufs, UError}; use serde::{Deserialize, Serialize}; +use std::process::Command; #[cfg(feature = "server")] use crate::models::schema::*; @@ -18,3 +19,33 @@ pub struct PayloadMeta { pub name: String, pub size: i64, } + +impl PayloadMeta { + pub fn from_existing_meta(payload_ufs_ident: &str) -> Result { + let ufs_meta = ufs::read_meta(&payload_ufs_ident)?; + let mime_type = bytes_to_string( + &Command::new("file") + .arg("-b") + .arg("--mime-type") + .arg(&ufs_meta.path) + .output() + .map_err(|e| UError::IOError(e.to_string()))? + .stdout, + ) + .trim() + .to_string(); + + Ok(PayloadMeta { + id: Id::new_v4(), + mime_type, + name: payload_ufs_ident.to_owned(), + size: ufs_meta.size as i64, + }) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FullPayload { + pub meta: PayloadMeta, + pub data: Vec, +} diff --git a/lib/u_lib/src/ufs/mod.rs b/lib/u_lib/src/ufs/mod.rs index 51a8ff6..e59d110 100644 --- a/lib/u_lib/src/ufs/mod.rs +++ b/lib/u_lib/src/ufs/mod.rs @@ -1,4 +1,4 @@ -// This module is aiming to store obfuscated payloads, get them by name, +// This module is aiming to store (obfuscated?) payloads, get them by name, // rename, update, delete or prepare to execute via memfd_create (unix) use anyhow::{Context, Result}; @@ -45,8 +45,8 @@ impl FileMeta { } /// Check if file exists in index. -/// File may present in fs but not in index, thus fn will return false. -pub fn in_index(name: impl AsRef) -> bool { +/// File may present in fs but not in index, fn will return false then. +pub fn exists_in_index(name: impl AsRef) -> bool { read_meta(name).is_ok() } @@ -86,7 +86,7 @@ pub fn put(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<()> { let name = name.as_ref(); let data_hash = hash_data(&data); - if in_index(&name) { + if exists_in_index(&name) { return Err(Error::already_exists(&name)).context("put_exists"); } @@ -158,11 +158,11 @@ pub fn rename(old_name: impl AsRef, new_name: impl AsRef) -> Result<() return Ok(()); } - if !in_index(old_name) { + if !exists_in_index(old_name) { return Err(Error::not_found(old_name)).context("rename"); } - if in_index(new_name) { + if exists_in_index(new_name) { return Err(Error::already_exists(new_name)).context("rename"); } @@ -190,7 +190,7 @@ pub fn update_payload_data(name: impl AsRef, data: impl AsRef<[u8]>) -> Res if external { index::remove(name); - } else if in_index(&name) { + } else if exists_in_index(&name) { remove(&name).context("upd")?; } @@ -202,7 +202,7 @@ pub fn put_external(path: impl AsRef) -> Result<()> { let path = path.as_ref(); let path_str = path.as_os_str().to_string_lossy().to_string(); - if in_index(&path_str) { + if exists_in_index(&path_str) { return Ok(()); } -- 2.36.2 From 32e96476cfbfbd2aa60ae5f1ba83c3bbde2bd22c Mon Sep 17 00:00:00 2001 From: plazmoid Date: Wed, 26 Apr 2023 19:02:08 +0300 Subject: [PATCH 06/10] remove redundant brief structs --- Cargo.lock | 27 ++- bin/u_agent/src/lib.rs | 16 +- bin/u_panel/src/argparse.rs | 25 +-- .../job-info-dialog.component.ts | 10 +- .../src/gui/fe/src/app/models/job.model.ts | 6 +- .../gui/fe/src/app/models/payload.model.ts | 12 +- .../gui/fe/src/app/services/api.service.ts | 4 +- bin/u_server/src/db.rs | 29 ++-- bin/u_server/src/error.rs | 26 +++ bin/u_server/src/handlers.rs | 89 ++++------ bin/u_server/src/u_server.rs | 6 +- integration-tests/tests/fixtures/agent.rs | 10 +- integration-tests/tests/helpers/panel.rs | 5 +- .../tests/integration_tests/api.rs | 14 +- .../tests/integration_tests/behaviour.rs | 4 +- lib/u_lib/src/api.rs | 28 ++- lib/u_lib/src/cache.rs | 4 +- lib/u_lib/src/jobs.rs | 66 ++----- lib/u_lib/src/logging.rs | 4 +- lib/u_lib/src/messaging.rs | 7 +- lib/u_lib/src/models/jobs/meta.rs | 102 +++++------ lib/u_lib/src/models/mod.rs | 13 +- lib/u_lib/src/models/payload.rs | 164 +++++++++++++++--- lib/u_lib/src/models/schema.rs | 1 + lib/u_lib/src/ufs/mod.rs | 53 +----- .../2020-10-24-111622_create_all/up.sql | 1 + 26 files changed, 362 insertions(+), 364 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3cdbe68..6f15ca8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2020,9 +2020,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.14" +version = "0.37.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b864d3c18a5785a05953adeed93e2dca37ed30f18e69bba9f30079d51f363f" +checksum = "a0661814f891c57c930a610266415528da53c4933e6dea5fb350cbfe048a9ece" dependencies = [ "bitflags", "errno 0.3.1", @@ -2503,9 +2503,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.27.0" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" dependencies = [ "autocfg", "bytes", @@ -2517,14 +2517,14 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", @@ -2554,9 +2554,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", @@ -2577,9 +2577,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ "bytes", "futures-core", @@ -2606,11 +2606,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "cf9cf6a813d3f40c88b0b6b6f29a5c95c6cdbf97c1f9cc53fb820200f5ad814d" dependencies = [ - "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-attributes", diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 942b1b6..faa3d3f 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -9,8 +9,8 @@ use u_lib::{ cache::JobCache, config::{get_self_id, EndpointsEnv, AGENT_ITERATION_INTERVAL}, error::ErrChan, - executor::pop_completed, - jobs::{split_payload, AnonymousJobBatch}, + executor, + jobs::AnonymousJobBatch, logging::init_logger, messaging::Reportable, models::AssignedJobById, @@ -21,7 +21,7 @@ async fn process_request(jobs: Vec, client: &HttpClient) { for jr in &jobs { if !JobCache::contains(jr.job_id) { info!("Fetching job: {}", &jr.job_id); - let fetched_job = loop { + let mut fetched_job = loop { //todo: use payload cache match client.get_full_job(jr.job_id).await { Ok(result) => break result, @@ -31,10 +31,12 @@ async fn process_request(jobs: Vec, client: &HttpClient) { } } }; - match split_payload(fetched_job) { - Ok(job_payload_meta) => JobCache::insert(job_payload_meta), - Err(e) => ErrChan::send(e, "pld").await, + if let Some(payload) = &mut fetched_job.payload { + if let Err(e) = payload.maybe_split_payload() { + ErrChan::send(e, "pld").await; + } } + JobCache::insert(fetched_job); } } info!( @@ -88,7 +90,7 @@ async fn agent_loop(client: HttpClient) -> ! { Err(err) => ErrChan::send(err, "processing").await, } - let result: Vec = pop_completed() + let result: Vec = executor::pop_completed() .await .into_iter() .map(|result| match result { diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 66bf9d1..0863996 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -2,9 +2,8 @@ use serde_json::{from_str, to_value, Value}; use structopt::StructOpt; use u_lib::{ api::HttpClient, - jobs::join_payload, messaging::AsMsg, - models::{Agent, AssignedJob, BriefMode, BriefOrFullJob, RawJob}, + models::{Agent, AssignedJob, BriefMode, RawJob}, types::Id, types::PanelResult, UError, UResult, @@ -88,14 +87,13 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { JobCRUD::Create { job } => { let raw_job = from_str::(&job) .map_err(|e| UError::DeserializeError(e.to_string(), job))?; - let job = raw_job.validated()?; - let full_job = join_payload(job)?; + let mut job = raw_job.validated()?; - into_value( - client - .upload_jobs([&BriefOrFullJob::Full(full_job)]) - .await?, - ) + if let Some(payload) = &mut job.payload { + payload.join_payload()?; + } + + into_value(client.upload_jobs([&job]).await?) } JobCRUD::RUD(RUD::Read { id }) => match id { Some(id) => into_value(vec![client.get_job(id, args.brief).await?]), @@ -104,10 +102,13 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { JobCRUD::RUD(RUD::Update { item }) => { let raw_job = from_str::(&item) .map_err(|e| UError::DeserializeError(e.to_string(), item))?; - let job = raw_job.validated()?; - let full_job = join_payload(job)?; + let mut job = raw_job.validated()?; + + if let Some(payload) = &mut job.payload { + payload.join_payload()?; + } - into_value(client.update_job(&BriefOrFullJob::Full(full_job)).await?) + into_value(client.update_job(&job).await?) } JobCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), }, diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts index 02a197e..c1ab012 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts @@ -1,9 +1,9 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA } from '@angular/material/dialog'; import { EventEmitter } from '@angular/core'; -import { BriefOrFullJobModel } from '../../../models/job.model'; +import { Job } from '../../../models/job.model'; import { ApiTableService } from 'src/app/services'; -import { BriefOrFullPayloadModel, isFullPayload } from 'src/app/models'; +import { PayloadModel } from 'src/app/models'; @Component({ selector: 'job-info-dialog', @@ -19,7 +19,7 @@ export class JobInfoDialogComponent { onSave = new EventEmitter(); - constructor(@Inject(MAT_DIALOG_DATA) public data: BriefOrFullJobModel, dataSource: ApiTableService) { + constructor(@Inject(MAT_DIALOG_DATA) public data: Job, dataSource: ApiTableService) { if (data.payload !== null) { this.showPayload(data.payload) } @@ -29,8 +29,8 @@ export class JobInfoDialogComponent { }) } - showPayload(payload: BriefOrFullPayloadModel) { - if (isFullPayload(payload)) { + showPayload(payload: PayloadModel) { + if (payload.data !== null) { this.decodedPayload = new TextDecoder().decode(new Uint8Array(payload.data)) } else { this.isTooBigPayload = true diff --git a/bin/u_panel/src/gui/fe/src/app/models/job.model.ts b/bin/u_panel/src/gui/fe/src/app/models/job.model.ts index 492db84..301ad5a 100644 --- a/bin/u_panel/src/gui/fe/src/app/models/job.model.ts +++ b/bin/u_panel/src/gui/fe/src/app/models/job.model.ts @@ -1,4 +1,4 @@ -import { BriefOrFullPayloadModel } from './' +import { PayloadModel } from './' export interface JobModel { alias: string | null, @@ -10,7 +10,7 @@ export interface JobModel { schedule: string | null, } -export interface BriefOrFullJobModel { +export interface Job { job: JobModel, - payload: BriefOrFullPayloadModel | null, + payload: PayloadModel | null, } \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts b/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts index 4e1e9c7..e947ac8 100644 --- a/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts +++ b/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts @@ -3,15 +3,5 @@ export interface PayloadModel { mime_type: string, name: string, size: number, -} - -export interface FullPayloadModel { - meta: PayloadModel, - data: number[] -} - -export type BriefOrFullPayloadModel = PayloadModel | FullPayloadModel; - -export function isFullPayload(payload: BriefOrFullPayloadModel): payload is FullPayloadModel { - return (payload as FullPayloadModel).data !== undefined + data: number[] | null } \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/services/api.service.ts b/bin/u_panel/src/gui/fe/src/app/services/api.service.ts index cee6ecb..4f2dc8a 100644 --- a/bin/u_panel/src/gui/fe/src/app/services/api.service.ts +++ b/bin/u_panel/src/gui/fe/src/app/services/api.service.ts @@ -1,7 +1,7 @@ import { environment } from 'src/environments/environment'; import { HttpClient, HttpErrorResponse } from '@angular/common/http'; import { Observable, map, catchError, throwError } from 'rxjs'; -import { ApiModel, getAreaByModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, BriefOrFullJobModel } from '../models'; +import { ApiModel, getAreaByModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, Job } from '../models'; import { Injectable, Inject } from '@angular/core'; import { ErrorService } from './error.service'; @@ -53,7 +53,7 @@ export class ApiTableService { return this.getOne(id, 'agents') } - getJob(id: string): Observable { + getJob(id: string): Observable { return this.getOne(id, 'jobs') } diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index 0f7dfe6..a417b49 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -3,7 +3,7 @@ use diesel::{pg::PgConnection, prelude::*, result::Error as DslError, Connection use std::mem::drop; use u_lib::{ db::PgAsyncPool, - models::{schema, Agent, AssignedJob, BriefJob, JobModel, JobState, PayloadMeta}, + models::{schema, Agent, AssignedJob, Job, JobModel, JobState, Payload}, platform::Platform, types::Id, }; @@ -50,13 +50,11 @@ pub struct UDB<'c> { } impl UDB<'_> { - pub fn insert_jobs(&mut self, jobs: &[BriefJob]) -> Result<()> { + pub fn insert_jobs(&mut self, jobs: &[Job]) -> Result<()> { use schema::{jobs, payloads}; - let (jobs, payloads_opt): (Vec<_>, Vec<_>) = jobs - .iter() - .map(|j| (&j.job, j.payload_meta.as_ref())) - .unzip(); + let (jobs, payloads_opt): (Vec<_>, Vec<_>) = + jobs.iter().map(|j| (&j.job, j.payload.as_ref())).unzip(); let payloads = payloads_opt .into_iter() @@ -76,17 +74,17 @@ impl UDB<'_> { .map_err(with_err_ctx("Can't insert jobs")) } - pub fn get_job(&mut self, id: Id) -> Result> { + pub fn get_job(&mut self, id: Id) -> Result> { use schema::{jobs, payloads}; let maybe_job_with_payload = jobs::table .left_join(payloads::table) .filter(jobs::id.eq(id)) - .first::<(JobModel, Option)>(self.conn) + .first::<(JobModel, Option)>(self.conn) .optional() .map_err(with_err_ctx(format!("Can't get job {id}")))?; - Ok(maybe_job_with_payload.map(|(job, payload_meta)| BriefJob { job, payload_meta })) + Ok(maybe_job_with_payload.map(|(job, payload)| Job { job, payload })) } pub fn get_jobs(&mut self) -> Result> { @@ -97,7 +95,7 @@ impl UDB<'_> { .map_err(with_err_ctx("Can't get jobs")) } - pub fn get_payload_meta(&mut self, id: Id) -> Result> { + pub fn get_payload(&mut self, id: Id) -> Result> { use schema::payloads; payloads::table @@ -107,7 +105,7 @@ impl UDB<'_> { .map_err(with_err_ctx(format!("Can't get payload {id}"))) } - pub fn get_payload_metas(&mut self) -> Result> { + pub fn get_payloads(&mut self) -> Result> { use schema::payloads; payloads::table @@ -115,17 +113,20 @@ impl UDB<'_> { .map_err(with_err_ctx("Can't get payloads")) } - pub fn find_job_by_alias(&mut self, alias: &str) -> Result> { + pub fn find_job_by_alias(&mut self, alias: &str) -> Result> { use schema::{jobs, payloads}; let maybe_job_with_payload = jobs::table .left_join(payloads::table) .filter(jobs::alias.eq(alias)) - .first::<(JobModel, Option)>(self.conn) + .first::<(JobModel, Option)>(self.conn) .optional() .map_err(with_err_ctx(format!("Can't get job by alias {alias}")))?; - Ok(maybe_job_with_payload.map(|(job, payload_meta)| BriefJob { job, payload_meta })) + Ok(maybe_job_with_payload.map(|(job, payload_meta)| Job { + job, + payload: payload_meta, + })) } pub fn insert_result(&mut self, result: &AssignedJob) -> Result<()> { diff --git a/bin/u_server/src/error.rs b/bin/u_server/src/error.rs index 26eb199..4a958dc 100644 --- a/bin/u_server/src/error.rs +++ b/bin/u_server/src/error.rs @@ -30,6 +30,12 @@ pub enum Error { #[error("Job cannot be ran on this platform. Expected: {0}, got: {1}")] InsuitablePlatform(String, String), + + #[error("{0}\nContext: {1}")] + Contexted(Box, String), + + #[error("Runtime error: {0}")] + Runtime(String), } impl Reject for Error {} @@ -67,3 +73,23 @@ impl Reply for RejResponse { with_status(self.message, self.status).into_response() } } + +impl From for Error { + fn from(e: anyhow::Error) -> Self { + let ctx = e + .chain() + .rev() + .skip(1) + .map(|cause| format!("ctx: {}", cause)) + .collect::>() + .join("\n"); + + match e.downcast::() { + Ok(err) => Error::Contexted(Box::new(err), ctx), + Err(err) => match err.downcast::() { + Ok(err) => Error::Contexted(Box::new(Error::FSError(err)), ctx), + Err(err) => Error::Runtime(err.to_string()), + }, + } + } +} diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index e1d6b7c..1a52b61 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -3,19 +3,10 @@ use std::sync::Arc; use crate::db::{PgRepo, UDB}; use crate::error::Error; use serde::Deserialize; -use u_lib::ufs; -use u_lib::{ - api::retypes, - jobs::{join_payload, split_payload}, - messaging::Reportable, - models::*, - types::Id, -}; +use u_lib::{api::retypes, messaging::Reportable, models::*, types::Id}; use warp::reject::not_found; use warp::Rejection; -const MAX_READABLE_PAYLOAD_SIZE: i64 = 1024 * 32; - type EndpResult = Result; #[derive(Deserialize)] @@ -48,29 +39,24 @@ impl Endpoints { id: Id, params: Option, ) -> EndpResult { - let Some(job) = repo.interact(move |mut db| db.get_job(id)).await? else { + let Some(mut job) = repo.interact(move |mut db| db.get_job(id)).await? else { return Err(not_found()) }; - let make_full_job = |j| -> Result { - let full_job = join_payload(j).map_err(Error::from)?; - Ok(BriefOrFullJob::Full(full_job)) - }; Ok(match params.map(|p| p.brief) { - Some(BriefMode::Yes) => BriefOrFullJob::Brief(job), + Some(BriefMode::Yes) => job, Some(BriefMode::Auto) | None => { - if job - .payload_meta - .as_ref() - .map(|m| m.size > MAX_READABLE_PAYLOAD_SIZE) - .unwrap_or(false) - { - BriefOrFullJob::Brief(job) - } else { - make_full_job(job)? + if let Some(payload) = &mut job.payload { + payload.maybe_join_payload().map_err(Error::from)?; } + job + } + Some(BriefMode::No) => { + if let Some(payload) = &mut job.payload { + payload.join_payload().map_err(Error::from)?; + } + job } - Some(BriefMode::No) => make_full_job(job)?, }) } @@ -90,7 +76,7 @@ impl Endpoints { } pub async fn get_payloads(repo: Arc) -> EndpResult { - repo.interact(move |mut db| db.get_payload_metas()) + repo.interact(move |mut db| db.get_payloads()) .await .map_err(From::from) } @@ -100,24 +86,19 @@ impl Endpoints { id: Id, params: Option, ) -> EndpResult { - let Some(meta) = repo.interact(move |mut db| db.get_payload_meta(id)).await? else { + let Some(mut payload) = repo.interact(move |mut db| db.get_payload(id)).await? else { return Err(not_found()) }; Ok(match params.map(|p| p.brief) { - Some(BriefMode::Yes) => BriefOrFullPayload::Brief(meta), - None | Some(BriefMode::Auto) if meta.size > MAX_READABLE_PAYLOAD_SIZE => { - BriefOrFullPayload::Brief(meta) + Some(BriefMode::Yes) => payload, + None | Some(BriefMode::Auto) => { + payload.maybe_join_payload().map_err(Error::from)?; + payload } _ => { - let payload_data = ufs::read(&meta.name).map_err(|e| { - error!("payload reading failed: {}", e); - Error::from(e.downcast::().expect("wrong error type")) - })?; - BriefOrFullPayload::Full(FullPayload { - meta, - data: payload_data, - }) + payload.join_payload().map_err(Error::from)?; + payload } }) } @@ -162,17 +143,16 @@ impl Endpoints { .map_err(From::from) } - pub async fn upload_jobs( - repo: Arc, - msg: Vec, - ) -> EndpResult { + pub async fn upload_jobs(repo: Arc, msg: Vec) -> EndpResult { let jobs = msg .into_iter() - .map(|meta| match meta { - BriefOrFull::Full(job) => Ok(split_payload(job)?), - BriefOrFull::Brief(job) => Ok(job), + .map(|mut job| { + if let Some(payload) = &mut job.payload { + payload.maybe_split_payload()?; + } + Ok(job) }) - .collect::, Error>>()?; + .collect::, Error>>()?; repo.interact(move |mut db| db.insert_jobs(&jobs)) .await @@ -286,17 +266,12 @@ impl Endpoints { Ok(()) } - pub async fn update_job( - repo: Arc, - job: BriefOrFullJob, - ) -> EndpResult { - let thin_job = match job { - BriefOrFullJob::Full(job) => split_payload(job).map_err(Error::from)?, - BriefOrFullJob::Brief(job) => job, - }; + pub async fn update_job(repo: Arc, mut job: Job) -> EndpResult { + if let Some(payload) = &mut job.payload { + payload.maybe_split_payload().map_err(Error::from)?; + } - repo.interact(move |mut db| db.update_job(&thin_job.job)) - .await?; + repo.interact(move |mut db| db.update_job(&job.job)).await?; Ok(()) } diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 5d06e96..ce1cd9e 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -57,7 +57,7 @@ pub fn init_endpoints( let upload_jobs = path("upload_jobs") .and(with_db.clone()) - .and(body::json::>()) + .and(body::json::>()) .and_then(Endpoints::upload_jobs) .map(into_message); @@ -115,7 +115,7 @@ pub fn init_endpoints( let update_job = path("update_job") .and(with_db.clone()) - .and(body::json::()) + .and(body::json::()) .and_then(Endpoints::update_job) .map(ok); @@ -167,7 +167,7 @@ pub async fn preload_jobs(repo: &PgRepo) -> Result<(), ServerError> { let job_alias = "agent_hello"; let if_job_exists = db.find_job_by_alias(job_alias)?; if if_job_exists.is_none() { - let agent_hello = RawJob::brief_job_builder() + let agent_hello = RawJob::builder() .with_type(JobType::Init) .with_alias(job_alias) .build() diff --git a/integration-tests/tests/fixtures/agent.rs b/integration-tests/tests/fixtures/agent.rs index f8264c8..e0dc8e2 100644 --- a/integration-tests/tests/fixtures/agent.rs +++ b/integration-tests/tests/fixtures/agent.rs @@ -1,7 +1,6 @@ use super::connections::*; use super::run_async; -use u_lib::unwrap_enum; -use u_lib::{api::HttpClient, jobs::split_payload, messaging::Reportable, models::*, types::Id}; +use u_lib::{api::HttpClient, messaging::Reportable, models::*, types::Id}; pub struct RegisteredAgent { pub id: Id, @@ -23,14 +22,17 @@ pub fn registered_agent(client: &HttpClient) -> RegisteredAgent { .unwrap(); let job_id = resp.job_id; let job = client.get_job(job_id, BriefMode::No).await.unwrap(); - let job = unwrap_enum!(job, BriefOrFull::Full); + assert_eq!(job.job.alias, Some("agent_hello".to_string())); - let mut agent_data = AssignedJob::from((&split_payload(job).unwrap().job, resp)); + + let mut agent_data = AssignedJob::from((&job.job, resp)); agent_data.set_result(&agent); + client .report([Reportable::Assigned(agent_data)]) .await .unwrap(); + RegisteredAgent { id: agent_id } }) } diff --git a/integration-tests/tests/helpers/panel.rs b/integration-tests/tests/helpers/panel.rs index 59a7c94..f33085b 100644 --- a/integration-tests/tests/helpers/panel.rs +++ b/integration-tests/tests/helpers/panel.rs @@ -23,10 +23,7 @@ impl Panel { let stderr = output.get_stderr(); if !stderr.is_empty() { - println!( - "\n*** PANEL DEBUG OUTPUT START***\n{}\n*** PANEL DEBUG OUTPUT END ***\n", - String::from_utf8_lossy(stderr) - ); + println!("\n{}\n", String::from_utf8_lossy(stderr)); } match from_slice(output.get_stdout()) { diff --git a/integration-tests/tests/integration_tests/api.rs b/integration-tests/tests/integration_tests/api.rs index 7051e9a..734ea31 100644 --- a/integration-tests/tests/integration_tests/api.rs +++ b/integration-tests/tests/integration_tests/api.rs @@ -14,13 +14,13 @@ // ping(&self) use crate::fixtures::connections::*; -use u_lib::models::{BriefOrFullJob, RawJob}; +use u_lib::models::RawJob; #[rstest] #[tokio::test] async fn test_jobs_endpoints(client_panel: &HttpClient) { let job_alias = "henlo"; - let mut job = RawJob::brief_job_builder() + let mut job = RawJob::builder() .with_shell("echo henlo") .with_alias(job_alias) .build() @@ -28,19 +28,13 @@ async fn test_jobs_endpoints(client_panel: &HttpClient) { let job_id = job.job.id; - client_panel - .upload_jobs([&BriefOrFullJob::Brief(job.clone())]) - .await - .unwrap(); + client_panel.upload_jobs([&job]).await.unwrap(); let fetched_job = client_panel.get_brief_job(job_id).await.unwrap(); assert_eq!(job, fetched_job); job.job.alias = Some("henlo2".to_string()); - client_panel - .update_job(&BriefOrFullJob::Brief(job.clone())) - .await - .unwrap(); + client_panel.update_job(&job).await.unwrap(); let fetched_job = client_panel.get_brief_job(job_id).await.unwrap(); assert_eq!(job, fetched_job); diff --git a/integration-tests/tests/integration_tests/behaviour.rs b/integration-tests/tests/integration_tests/behaviour.rs index 99facbb..51ea208 100644 --- a/integration-tests/tests/integration_tests/behaviour.rs +++ b/integration-tests/tests/integration_tests/behaviour.rs @@ -21,7 +21,7 @@ async fn setup_tasks() { let agents: Vec = Panel::check_output("agents read"); let agent_id = agents[0].id; let job_alias = "passwd_contents"; - let job = RawJob::brief_job_builder() + let job = RawJob::builder() .with_alias(job_alias) .with_raw_payload(b"cat /etc/passwd".as_slice()) .with_shell("/bin/bash {}") @@ -54,7 +54,7 @@ async fn large_payload() { let agent = &Panel::check_output::>("agents read")[0]; let agent_id = agent.id; let job_alias = "large_payload"; - let job = RawJob::brief_job_builder() + let job = RawJob::builder() .with_alias(job_alias) .with_payload_path("./tests/bin/echoer") .with_shell("{} type echo") diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 2a21165..9e5c70b 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -7,7 +7,6 @@ use reqwest::{header, header::HeaderMap, Certificate, Client, Identity, Method, use serde::de::DeserializeOwned; use serde_json::{from_str, Value}; -use crate::unwrap_enum; use crate::{ config::{get_self_id, MASTER_PORT}, conv::opt_to_string, @@ -25,9 +24,8 @@ pub mod retypes { pub type GetPersonalJobs = Vec; pub type Report = (); - pub type GetJob = BriefOrFullJob; - pub type GetFullJob = FullJob; - pub type GetBriefJob = BriefJob; + pub type GetJob = Job; + pub type GetBriefJob = Job; pub type GetJobs = Vec; pub type GetAgents = Vec; pub type UpdateAgent = (); @@ -38,8 +36,8 @@ pub mod retypes { pub type SetJobs = Vec; pub type GetAgentJobs = Vec; pub type Ping = (); - pub type GetPayloads = Vec; - pub type GetPayload = BriefOrFullPayload; + pub type GetPayloads = Vec; + pub type GetPayload = Payload; } #[derive(Clone, Debug)] @@ -113,8 +111,6 @@ impl HttpClient { .post(self.base_url.join(url).unwrap()) .json(payload); - debug!("url = {url}"); - let response = request .send() .await @@ -136,7 +132,7 @@ impl HttpClient { } .map_err(From::from); - debug!("response = {:?}", result); + debug!("url = {url}, response = {result:?}"); result } @@ -165,14 +161,12 @@ impl HttpClient { self.req(format!("get_job/{job}?brief={brief}")).await } - pub async fn get_full_job(&self, job: Id) -> Result { - let job = self.get_job(job, BriefMode::No).await?; - Ok(unwrap_enum!(job, BriefOrFullJob::Full)) + pub async fn get_full_job(&self, job: Id) -> Result { + self.get_job(job, BriefMode::No).await } - pub async fn get_brief_job(&self, job: Id) -> Result { - let job = self.get_job(job, BriefMode::Yes).await?; - Ok(unwrap_enum!(job, BriefOrFullJob::Brief)) + pub async fn get_brief_job(&self, job: Id) -> Result { + self.get_job(job, BriefMode::Yes).await } /// get all available jobs @@ -196,7 +190,7 @@ impl HttpClient { } /// update job - pub async fn update_job(&self, job: &BriefOrFullJob) -> Result { + pub async fn update_job(&self, job: &Job) -> Result { self.req_with_payload("update_job", job).await } @@ -208,7 +202,7 @@ impl HttpClient { /// create and upload job pub async fn upload_jobs( &self, - payload: impl IntoIterator, + payload: impl IntoIterator, ) -> Result { self.req_with_payload("upload_jobs", &payload.into_iter().collect::>()) .await diff --git a/lib/u_lib/src/cache.rs b/lib/u_lib/src/cache.rs index 3d8f4b4..c81414c 100644 --- a/lib/u_lib/src/cache.rs +++ b/lib/u_lib/src/cache.rs @@ -1,10 +1,10 @@ -use crate::models::BriefJob; +use crate::models::Job; use crate::types::Id; use lazy_static::lazy_static; use parking_lot::{RwLock, RwLockReadGuard}; use std::{collections::HashMap, ops::Deref}; -type Val = BriefJob; +type Val = Job; type Cache = HashMap; lazy_static! { diff --git a/lib/u_lib/src/jobs.rs b/lib/u_lib/src/jobs.rs index 5c01b91..4c433cc 100644 --- a/lib/u_lib/src/jobs.rs +++ b/lib/u_lib/src/jobs.rs @@ -1,11 +1,8 @@ use crate::{ combined_result::CombinedResult, executor::{ExecResult, Waiter}, - models::{ - Agent, AssignedJob, AssignedJobById, BriefJob, FullJob, FullPayload, JobType, RawJob, - }, + models::{Agent, AssignedJob, AssignedJobById, Job, JobType, RawJob}, proc_output::ProcOutput, - ufs, }; use std::collections::HashMap; use std::process::exit; @@ -17,7 +14,7 @@ pub struct AnonymousJobBatch { } impl AnonymousJobBatch { - pub fn from_meta_with_id(jobs: impl IntoIterator) -> Self { + pub fn from_meta_with_id(jobs: impl IntoIterator) -> Self { let mut waiter = Waiter::new(); for (job, ids) in jobs { waiter.push(run_assigned_job(job, ids)); @@ -28,7 +25,7 @@ impl AnonymousJobBatch { } } - pub fn from_meta(jobs: impl IntoIterator) -> Self { + pub fn from_meta(jobs: impl IntoIterator) -> Self { let jobs_ids: Vec<_> = jobs .into_iter() .map(|job| { @@ -47,6 +44,7 @@ impl AnonymousJobBatch { /// Spawn jobs pub async fn spawn(mut self) -> Self { + debug!("spawning jobs"); self.waiter = self.waiter.spawn().await; self.is_running = true; self @@ -81,11 +79,7 @@ impl NamedJobBatch { let jobs: Vec<_> = named_jobs .into_iter() .filter_map(|(alias, cmd)| { - match RawJob::brief_job_builder() - .with_shell(cmd) - .with_alias(alias) - .build() - { + match RawJob::builder().with_shell(cmd).with_alias(alias).build() { Ok(jpm) => Some(jpm), Err(e) => { result.push_err(e); @@ -98,7 +92,7 @@ impl NamedJobBatch { result } - pub fn from_meta(named_jobs: Vec) -> Self { + pub fn from_meta(named_jobs: Vec) -> Self { let (job_names, jobs): (Vec<_>, Vec<_>) = named_jobs .into_iter() .map(|job| (job.job.alias.clone().unwrap(), job)) @@ -134,14 +128,14 @@ impl NamedJobBatch { } } -pub async fn run_assigned_job(job: BriefJob, ids: AssignedJobById) -> ExecResult { - let BriefJob { job, payload_meta } = job; +pub async fn run_assigned_job(job: Job, ids: AssignedJobById) -> ExecResult { + let Job { job, payload } = job; let mut result = AssignedJob::from((&job, ids)); match job.exec_type { JobType::Shell => { let (argv, _prepared_payload) = { - if let Some(meta) = payload_meta { - let (prep_exec, prep_exec_path) = ufs::prepare_executable(meta.name)?; + if let Some(payload) = payload { + let (prep_exec, prep_exec_path) = payload.prepare_executable()?; let argv_with_exec = job.argv.replace("{}", &prep_exec_path); (argv_with_exec, Some(prep_exec)) } else { @@ -181,38 +175,6 @@ pub async fn run_assigned_job(job: BriefJob, ids: AssignedJobById) -> ExecResult Ok(result) } -pub fn split_payload(job: FullJob) -> Result { - let FullJob { job, payload } = job; - - if let Some(payload) = &payload { - if ufs::exists_in_index(&payload.meta.name) { - ufs::edit(&payload.meta.name, &payload.data)?; - } else { - ufs::put(&payload.meta.name, &payload.data)?; - } - } - - Ok(BriefJob { - job, - payload_meta: payload.map(|p| p.meta), - }) -} - -pub fn join_payload(job: BriefJob) -> Result { - let BriefJob { job, payload_meta } = job; - let payload = match payload_meta { - Some(meta) => { - let payload_data = ufs::read(&meta.name)?; - Some(FullPayload { - meta, - data: payload_data, - }) - } - None => None, - }; - Ok(FullJob { job, payload }) -} - #[cfg(test)] mod tests { use crate::{ @@ -265,7 +227,7 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] expected_result: &str, ) -> TestResult { - let mut job = RawJob::brief_job_builder().with_shell(cmd); + let mut job = RawJob::builder().with_shell(cmd); if let Some(p) = payload { job = job.with_raw_payload(p); } @@ -348,7 +310,7 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] err_str: &str, ) -> TestResult { - let mut job = RawJob::brief_job_builder().with_shell(cmd); + let mut job = RawJob::builder().with_shell(cmd); if let Some(p) = payload { job = job.with_raw_payload(p); } @@ -361,12 +323,12 @@ mod tests { #[tokio::test] async fn test_different_job_types() -> TestResult { let mut jobs = NamedJobBatch::from_meta(vec![ - RawJob::brief_job_builder() + RawJob::builder() .with_shell("sleep 3") .with_alias("sleeper") .build() .unwrap(), - RawJob::brief_job_builder() + RawJob::builder() .with_type(JobType::Init) .with_alias("gatherer") .build() diff --git a/lib/u_lib/src/logging.rs b/lib/u_lib/src/logging.rs index ae32a4f..018df38 100644 --- a/lib/u_lib/src/logging.rs +++ b/lib/u_lib/src/logging.rs @@ -1,5 +1,5 @@ use std::env; -use std::io::{stderr, stdout}; +use std::io::stderr; use std::path::Path; use tracing_appender::rolling; @@ -11,7 +11,7 @@ pub fn init_logger(logfile: Option<&str>) { } let output_layer = if cfg!(test) { - fmt::layer().with_writer(stdout).with_test_writer().boxed() + fmt::layer().with_test_writer().boxed() } else { fmt::layer().with_writer(stderr).boxed() }; diff --git a/lib/u_lib/src/messaging.rs b/lib/u_lib/src/messaging.rs index cea2be9..721bbb8 100644 --- a/lib/u_lib/src/messaging.rs +++ b/lib/u_lib/src/messaging.rs @@ -11,12 +11,9 @@ impl AsMsg for Agent {} impl AsMsg for AssignedJob {} impl AsMsg for AssignedJobById {} impl AsMsg for JobModel {} -impl AsMsg for FullJob {} impl AsMsg for Reportable {} -impl AsMsg for PayloadMeta {} -impl AsMsg for FullPayload {} -impl AsMsg for BriefJob {} -impl AsMsg for BriefOrFull {} +impl AsMsg for Payload {} +impl AsMsg for Job {} impl AsMsg for Id {} impl AsMsg for String {} impl AsMsg for Vec {} diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 6039ad1..1d79122 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -3,10 +3,10 @@ use std::fmt; use super::JobType; #[cfg(feature = "server")] use crate::models::schema::*; -use crate::models::{FullPayload, PayloadMeta}; +use crate::models::Payload; use crate::platform; use crate::types::Id; -use crate::{ufs, UError, UResult}; +use crate::{UError, UResult}; #[cfg(feature = "server")] use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; @@ -33,15 +33,9 @@ pub struct JobModel { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct FullJob { +pub struct Job { pub job: JobModel, - pub payload: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct BriefJob { - pub job: JobModel, - pub payload_meta: Option, + pub payload: Option, } #[derive(Serialize, Deserialize, Clone)] @@ -105,9 +99,12 @@ impl fmt::Debug for RawJob<'_> { } } -impl From for RawJob<'_> { - fn from(job: BriefJob) -> Self { - let BriefJob { job, payload_meta } = job; +impl From for RawJob<'_> { + fn from(job: Job) -> Self { + let Job { + job, + payload: payload_meta, + } = job; RawJob { alias: job.alias, argv: job.argv, @@ -122,15 +119,15 @@ impl From for RawJob<'_> { } impl<'p> RawJob<'p> { - pub fn validated(self) -> UResult { + pub fn validated(self) -> UResult { JobBuilder { inner: self }.build() } - pub fn from_shell(cmd: impl Into) -> UResult { - Self::brief_job_builder().with_shell(cmd).build() + pub fn from_shell(cmd: impl Into) -> UResult { + Self::builder().with_shell(cmd).build() } - pub fn brief_job_builder() -> JobBuilder<'p> { + pub fn builder() -> JobBuilder<'p> { JobBuilder::default() } } @@ -174,29 +171,40 @@ impl<'p> JobBuilder<'p> { self } - pub fn build(self) -> UResult { + pub fn build(self) -> UResult { let mut inner = self.inner; - let raw_into_job = |raw: RawJob| -> UResult { - let payload_meta = raw - .payload_path - .as_ref() - .map(|payload_ident| PayloadMeta::from_existing_meta(payload_ident)) - .transpose()?; + fn _build(job: RawJob) -> UResult { + let payload = { + let payload_from_path = job + .payload_path + .as_ref() + .map(|path| Payload::from_path(path)) + .transpose()?; + + if payload_from_path.is_none() { + job.raw_payload + .as_ref() + .map(|data| Payload::from_data(data)) + .transpose()? + } else { + payload_from_path + } + }; - Ok(BriefJob { + Ok(Job { job: JobModel { - alias: raw.alias, - argv: raw.argv, - id: raw.id, - exec_type: raw.exec_type, - target_platforms: raw.target_platforms, - payload: payload_meta.as_ref().map(|meta| meta.id), - schedule: raw.schedule, + alias: job.alias, + argv: job.argv, + id: job.id, + exec_type: job.exec_type, + target_platforms: job.target_platforms, + payload: payload.as_ref().map(|p| p.id), + schedule: job.schedule, }, - payload_meta, + payload, }) - }; + } match inner.exec_type { JobType::Shell => { @@ -219,23 +227,14 @@ impl<'p> JobBuilder<'p> { return Err(empty_err.into()); } - if let Some(path) = &inner.payload_path { - ufs::put_external(path)?; - } - - if let Some(raw_payload) = &inner.raw_payload { - match inner.payload_path { - Some(_) => { - return Err(UError::JobBuildError( - "Can't use both raw payload with payload path".to_string(), - )) - } - None => inner.payload_path = Some(ufs::create_anonymous(raw_payload)?), - } + if inner.raw_payload.is_some() && inner.payload_path.is_some() { + return Err(UError::JobBuildError( + "Can't use both raw payload with payload path".to_string(), + )); } match inner.payload_path.as_ref() { - Some(_) => { + Some(_) | None if inner.raw_payload.is_some() => { if !inner.argv.contains("{}") { return Err(UError::JobBuildError( "Argv contains no executable placeholder".into(), @@ -244,7 +243,7 @@ impl<'p> JobBuilder<'p> { } } None => { - if inner.argv.contains("{}") { + if inner.argv.contains("{}") && inner.raw_payload.is_none() { return Err(UError::JobBuildError( "No payload provided, but argv contains executable placeholder" .into(), @@ -252,6 +251,7 @@ impl<'p> JobBuilder<'p> { .into()); } } + _ => (), }; if inner.target_platforms.is_empty() { @@ -265,9 +265,9 @@ impl<'p> JobBuilder<'p> { ))); } - raw_into_job(inner) + _build(inner) } - _ => raw_into_job(inner), + _ => _build(inner), } } } diff --git a/lib/u_lib/src/models/mod.rs b/lib/u_lib/src/models/mod.rs index e8f315d..44d56a8 100644 --- a/lib/u_lib/src/models/mod.rs +++ b/lib/u_lib/src/models/mod.rs @@ -4,21 +4,10 @@ mod payload; #[cfg(feature = "server")] pub mod schema; -use crate::messaging::AsMsg; pub use crate::models::{agent::*, jobs::*, payload::*}; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use strum::{Display as StrumDisplay, EnumString}; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(untagged)] -pub enum BriefOrFull { - Brief(B), - Full(F), -} - -pub type BriefOrFullJob = BriefOrFull; -pub type BriefOrFullPayload = BriefOrFull; - #[derive(Default, Debug, StrumDisplay, EnumString, Deserialize)] pub enum BriefMode { Yes, diff --git a/lib/u_lib/src/models/payload.rs b/lib/u_lib/src/models/payload.rs index 88cf6fa..2b162fe 100644 --- a/lib/u_lib/src/models/payload.rs +++ b/lib/u_lib/src/models/payload.rs @@ -1,6 +1,9 @@ use crate::{conv::bytes_to_string, types::Id, ufs, UError}; +use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; -use std::process::Command; +use std::{fs::File, path::Path, process::Command}; + +const MAX_READABLE_PAYLOAD_SIZE: i64 = 1024 * 32; #[cfg(feature = "server")] use crate::models::schema::*; @@ -13,39 +16,148 @@ use diesel::Identifiable; diesel(table_name = payloads) )] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct PayloadMeta { +pub struct Payload { pub id: Id, pub mime_type: String, pub name: String, pub size: i64, + data: Option>, // when None, payload data is stored in ufs } -impl PayloadMeta { - pub fn from_existing_meta(payload_ufs_ident: &str) -> Result { - let ufs_meta = ufs::read_meta(&payload_ufs_ident)?; - let mime_type = bytes_to_string( - &Command::new("file") - .arg("-b") - .arg("--mime-type") - .arg(&ufs_meta.path) - .output() - .map_err(|e| UError::IOError(e.to_string()))? - .stdout, - ) - .trim() - .to_string(); - - Ok(PayloadMeta { +impl Payload { + pub fn is_human_readable(&self) -> bool { + self.size < MAX_READABLE_PAYLOAD_SIZE && self.mime_type.starts_with("text/") + } + + pub fn maybe_split_payload(&mut self) -> Result<()> { + if self.is_human_readable() { + return Ok(()); + } + + if let Some(data) = self.data.take() { + if ufs::exists_in_index(&self.name) { + ufs::edit(&self.name, data)?; + } else { + ufs::put(&self.name, data)?; + } + } + Ok(()) + } + + pub fn join_payload(&mut self) -> Result<()> { + if self.data.is_none() { + self.data = Some(ufs::read(&self.name)?); + } + Ok(()) + } + + pub fn maybe_join_payload(&mut self) -> Result<()> { + if self.is_human_readable() { + self.join_payload()?; + } + Ok(()) + } + + pub fn from_data(data: impl AsRef<[u8]>) -> Result { + let name = ufs::create_anonymous(data)?; + let meta = ufs::read_meta(&name)?; + + let mut payload = Payload { id: Id::new_v4(), - mime_type, - name: payload_ufs_ident.to_owned(), - size: ufs_meta.size as i64, - }) + mime_type: get_mime_type(&meta.path)?, + name: name.clone(), + size: meta.size as i64, + data: None, + }; + + if payload.is_human_readable() { + payload.join_payload()?; + } + + Ok(payload) + } + + pub fn from_path(payload_path: &str) -> Result { + ufs::put_external(payload_path)?; + + let meta = ufs::read_meta(&payload_path)?; + + let mut payload = Payload { + id: Id::new_v4(), + mime_type: get_mime_type(&meta.path)?, + name: payload_path.to_owned(), + size: meta.size as i64, + data: None, + }; + + if payload.is_human_readable() { + payload.join_payload()?; + } + + Ok(payload) + } + + /// Prepare executable file: unpack, decipher if needed and send under memfd + #[cfg(unix)] + pub fn prepare_executable(&self) -> Result<(File, String)> { + use libc::getpid; + use nix::sys::memfd::*; + use std::ffi::CString; + use std::io::{Read, Write}; + use std::os::fd::FromRawFd; + + const FAKE_EXEC_NAME: &str = "/usr/sbin/lvmetad"; + const BUFFER_LEN: usize = 4096; + + let mut buffer: [u8; BUFFER_LEN] = [0; BUFFER_LEN]; + let mut payload_src = if let Some(data) = &self.data { + Box::new(data.as_slice()) as Box + } else { + let payload_path = ufs::read_meta(&self.name).context("prep")?.path; + let file = File::open(&payload_path).map_err(|e| ufs::Error::new(e, &payload_path))?; + Box::new(file) as Box + }; + + let fd = memfd_create( + CString::new(FAKE_EXEC_NAME).unwrap().as_c_str(), + MemFdCreateFlag::empty(), + ); + + match fd { + Ok(fd) => { + let mut payload_dest = unsafe { File::from_raw_fd(fd) }; + + loop { + let bytes_read = payload_src.read(&mut buffer)?; + payload_dest.write(&buffer[..bytes_read])?; + + if bytes_read != BUFFER_LEN { + break; + } + } + let payload_path = format!("/proc/{}/fd/{}", unsafe { getpid() }, fd); + Ok((payload_dest, payload_path)) + } + Err(e) => Err(ufs::Error::new(e, FAKE_EXEC_NAME)).context("prep"), + } + } + + #[cfg(windows)] + pub fn prepare_executable(&self) -> Result<(File, String)> { + todo!() } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct FullPayload { - pub meta: PayloadMeta, - pub data: Vec, +fn get_mime_type(path: impl AsRef) -> Result { + Ok(bytes_to_string( + &Command::new("file") + .arg("-b") + .arg("--mime-type") + .arg(path.as_ref()) + .output() + .map_err(|e| UError::IOError(e.to_string()))? + .stdout, + ) + .trim() + .to_string()) } diff --git a/lib/u_lib/src/models/schema.rs b/lib/u_lib/src/models/schema.rs index 5fac813..9908f4d 100644 --- a/lib/u_lib/src/models/schema.rs +++ b/lib/u_lib/src/models/schema.rs @@ -59,6 +59,7 @@ diesel::table! { mime_type -> Text, name -> Text, size -> Int8, + data -> Nullable, } } diff --git a/lib/u_lib/src/ufs/mod.rs b/lib/u_lib/src/ufs/mod.rs index e59d110..c6023c5 100644 --- a/lib/u_lib/src/ufs/mod.rs +++ b/lib/u_lib/src/ufs/mod.rs @@ -4,8 +4,8 @@ use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; use std::env::temp_dir; -use std::ffi::{CString, OsString}; -use std::fs::{self, File}; +use std::ffi::OsString; +use std::fs; use std::path::{Path, PathBuf}; use uuid::Uuid; @@ -18,9 +18,9 @@ const OBFUSCATE: bool = cfg!(feature = "agent"); #[derive(Clone, Deserialize, Serialize)] pub struct FileMeta { extension: Option, - external: bool, + external: bool, // if file is present before adding to index hash: Vec, - pub path: PathBuf, + pub path: PathBuf, // actual file path pub size: u64, } @@ -221,51 +221,6 @@ pub fn put_external(path: impl AsRef) -> Result<()> { Ok(()) } -/// Prepare executable file: unpack, decipher if needed and send under memfd -#[cfg(unix)] -pub fn prepare_executable(name: impl AsRef) -> Result<(File, String)> { - use libc::getpid; - use nix::sys::memfd::*; - use std::io::{Read, Write}; - use std::os::fd::FromRawFd; - - const FAKE_EXEC_NAME: &str = "/usr/sbin/lvmetad"; - const BUFFER_LEN: usize = 4096; - - let mut buffer: [u8; BUFFER_LEN] = [0; BUFFER_LEN]; - let payload_meta = read_meta(name).context("prep")?; - - let fd = memfd_create( - CString::new(FAKE_EXEC_NAME).unwrap().as_c_str(), - MemFdCreateFlag::empty(), - ); - - match fd { - Ok(fd) => { - let mut payload_src = - File::open(&payload_meta.path).map_err(|e| Error::new(e, &payload_meta.path))?; - let mut payload_dest = unsafe { File::from_raw_fd(fd) }; - - loop { - let bytes_read = payload_src.read(&mut buffer)?; - payload_dest.write(&buffer[..bytes_read])?; - - if bytes_read != BUFFER_LEN { - break; - } - } - let payload_path = format!("/proc/{}/fd/{}", unsafe { getpid() }, fd); - Ok((payload_dest, payload_path)) - } - Err(e) => Err(Error::new(e, FAKE_EXEC_NAME)).context("prep"), - } -} - -#[cfg(windows)] -pub fn prepare_executable(name: impl AsRef) -> Result<(File, String)> { - todo!() -} - /* pub fn cleanup() { let index = INDEX.read(); diff --git a/migrations/2020-10-24-111622_create_all/up.sql b/migrations/2020-10-24-111622_create_all/up.sql index 7ddd216..49fd6cf 100644 --- a/migrations/2020-10-24-111622_create_all/up.sql +++ b/migrations/2020-10-24-111622_create_all/up.sql @@ -27,6 +27,7 @@ CREATE TABLE IF NOT EXISTS payloads ( mime_type TEXT NOT NULL, name TEXT NOT NULL UNIQUE, size BIGINT NOT NULL, + data BYTEA, PRIMARY KEY(id) ); -- 2.36.2 From 45bba0dd9bc3d0a77ed8ea656ee46e5d3f52df15 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sun, 30 Apr 2023 00:50:19 +0300 Subject: [PATCH 07/10] finish payload crud & tests --- Cargo.lock | 59 +++++---- bin/u_panel/src/argparse.rs | 55 ++++++--- bin/u_server/src/db.rs | 45 ++++--- bin/u_server/src/error.rs | 5 +- bin/u_server/src/handlers.rs | 90 +++++++++++--- bin/u_server/src/u_server.rs | 36 ++++-- images/u_server.Dockerfile | 2 +- integration-tests/docker-compose.yml | 4 +- .../tests/integration_tests/api.rs | 46 ------- .../tests/integration_tests/behaviour.rs | 2 +- .../tests/integration_tests/endpoints.rs | 112 ++++++++++++++++++ .../tests/integration_tests/mod.rs | 2 +- lib/u_lib/src/api.rs | 25 +++- lib/u_lib/src/logging.rs | 6 +- lib/u_lib/src/messaging.rs | 1 + lib/u_lib/src/models/jobs/meta.rs | 32 ++++- lib/u_lib/src/models/payload.rs | 49 ++++++-- lib/u_lib/src/models/schema.rs | 4 +- lib/u_lib/src/ufs/mod.rs | 8 +- .../2020-10-24-111622_create_all/up.sql | 4 +- 20 files changed, 419 insertions(+), 168 deletions(-) delete mode 100644 integration-tests/tests/integration_tests/api.rs create mode 100644 integration-tests/tests/integration_tests/endpoints.rs diff --git a/Cargo.lock b/Cargo.lock index 6f15ca8..c65af22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -815,9 +815,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", "miniz_oxide", @@ -1344,9 +1344,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36eb31c1778188ae1e64398743890d0877fef36d11521ac60406b42016e8c2cf" +checksum = "b64f40e5e03e0d54f03845c8197d0291253cdbedfb1cb46b13c2c117554a9f4c" [[package]] name = "local-channel" @@ -1459,9 +1459,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] @@ -1479,17 +1479,21 @@ dependencies = [ ] [[package]] -name = "multiparty" -version = "0.1.0" +name = "multer" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed1ec6589a6d4a1e0b33b4c0a3f6ee96dfba88ebdb3da51403fd7cf0a24a4b04" +checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" dependencies = [ "bytes", - "futures-core", + "encoding_rs", + "futures-util", + "http", "httparse", + "log", "memchr", - "pin-project-lite", - "try-lock", + "mime", + "spin 0.9.8", + "version_check", ] [[package]] @@ -1873,9 +1877,9 @@ checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" [[package]] name = "reqwest" -version = "0.11.16" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" +checksum = "13293b639a097af28fc8a90f22add145a9c954e49d77da06263d58cf44d5fb91" dependencies = [ "base64 0.21.0", "bytes", @@ -1923,7 +1927,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", @@ -2020,9 +2024,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.15" +version = "0.37.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0661814f891c57c930a610266415528da53c4933e6dea5fb350cbfe048a9ece" +checksum = "bc809f704c03a812ac71f22456c857be34185cac691a4316f27ab0f633bb9009" dependencies = [ "bitflags", "errno 0.3.1", @@ -2278,6 +2282,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "strsim" version = "0.8.0" @@ -2606,10 +2616,11 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.38" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9cf6a813d3f40c88b0b6b6f29a5c95c6cdbf97c1f9cc53fb820200f5ad814d" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ + "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-attributes", @@ -2878,9 +2889,9 @@ checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "uuid" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb" +checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" dependencies = [ "getrandom", "serde", @@ -2938,9 +2949,9 @@ dependencies = [ [[package]] name = "warp" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27e1a710288f0f91a98dd8a74f05b76a10768db245ce183edf64dc1afdc3016c" +checksum = "ba431ef570df1287f7f8b07e376491ad54f84d26ac473489427231e1718e1f69" dependencies = [ "bytes", "futures-channel", @@ -2951,7 +2962,7 @@ dependencies = [ "log", "mime", "mime_guess", - "multiparty", + "multer", "percent-encoding", "pin-project", "rustls-pemfile", diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 0863996..e528aad 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -1,12 +1,7 @@ use serde_json::{from_str, to_value, Value}; use structopt::StructOpt; use u_lib::{ - api::HttpClient, - messaging::AsMsg, - models::{Agent, AssignedJob, BriefMode, RawJob}, - types::Id, - types::PanelResult, - UError, UResult, + api::HttpClient, messaging::AsMsg, models::*, types::Id, types::PanelResult, UError, UResult, }; #[derive(StructOpt, Debug)] @@ -20,17 +15,17 @@ pub struct Args { #[derive(StructOpt, Debug)] enum Cmd { Agents(RUD), - Jobs(JobCRUD), + Jobs(CRUD), Map(MapCRUD), - Payloads(RUD), + Payloads(PayloadCRUD), Ping, Serve, } #[derive(StructOpt, Debug)] -enum JobCRUD { +enum CRUD { Create { - job: String, + item: String, }, #[structopt(flatten)] RUD(RUD), @@ -48,6 +43,23 @@ enum MapCRUD { RUD(RUD), } +#[derive(StructOpt, Debug)] +enum PayloadCRUD { + Create { + item: String, + }, + Read { + id: Option, + }, + Update { + item: String, + }, + Delete { + #[structopt(parse(try_from_str = parse_uuid))] + id: Id, + }, +} + #[derive(StructOpt, Debug)] enum RUD { Read { @@ -84,7 +96,7 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { RUD::Delete { id } => into_value(client.del(id).await?), }, Cmd::Jobs(action) => match action { - JobCRUD::Create { job } => { + CRUD::Create { item: job } => { let raw_job = from_str::(&job) .map_err(|e| UError::DeserializeError(e.to_string(), job))?; let mut job = raw_job.validated()?; @@ -95,11 +107,11 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { into_value(client.upload_jobs([&job]).await?) } - JobCRUD::RUD(RUD::Read { id }) => match id { + CRUD::RUD(RUD::Read { id }) => match id { Some(id) => into_value(vec![client.get_job(id, args.brief).await?]), None => into_value(client.get_jobs().await?), }, - JobCRUD::RUD(RUD::Update { item }) => { + CRUD::RUD(RUD::Update { item }) => { let raw_job = from_str::(&item) .map_err(|e| UError::DeserializeError(e.to_string(), item))?; let mut job = raw_job.validated()?; @@ -110,7 +122,7 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { into_value(client.update_job(&job).await?) } - JobCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), + CRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), }, Cmd::Map(action) => match action { MapCRUD::Create { @@ -126,12 +138,21 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { MapCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), }, Cmd::Payloads(action) => match action { - RUD::Read { id } => match id { + PayloadCRUD::Create { item } => { + let payload = from_str::(&item) + .map_err(|e| UError::DeserializeError(e.to_string(), item))?; + into_value(client.upload_payloads([&payload]).await?) + } + PayloadCRUD::Read { id } => match id { None => into_value(client.get_payloads().await?), Some(id) => into_value(client.get_payload(id, args.brief).await?), }, - RUD::Update { item: _item } => todo!(), - RUD::Delete { id } => into_value(client.del(id).await?), + PayloadCRUD::Update { item } => { + let payload = from_str::(&item) + .map_err(|e| UError::DeserializeError(e.to_string(), item))?; + into_value(client.update_payload(&payload).await?) + } + PayloadCRUD::Delete { id } => into_value(client.del(id).await?), }, Cmd::Ping => into_value(client.ping().await?), Cmd::Serve => { diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index a417b49..9228529 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -50,28 +50,24 @@ pub struct UDB<'c> { } impl UDB<'_> { - pub fn insert_jobs(&mut self, jobs: &[Job]) -> Result<()> { - use schema::{jobs, payloads}; + pub fn insert_jobs(&mut self, jobs: &[JobModel]) -> Result<()> { + use schema::jobs; - let (jobs, payloads_opt): (Vec<_>, Vec<_>) = - jobs.iter().map(|j| (&j.job, j.payload.as_ref())).unzip(); + diesel::insert_into(jobs::table) + .values(jobs) + .execute(self.conn) + .map(drop) + .map_err(with_err_ctx("Can't insert jobs")) + } - let payloads = payloads_opt - .into_iter() - .filter_map(|p| p) - .collect::>(); + pub fn insert_payloads(&mut self, payloads: &[Payload]) -> Result<()> { + use schema::payloads; diesel::insert_into(payloads::table) .values(payloads) .execute(self.conn) .map(drop) - .map_err(with_err_ctx("Can't insert payloads"))?; - - diesel::insert_into(jobs::table) - .values(jobs) - .execute(self.conn) - .map(drop) - .map_err(with_err_ctx("Can't insert jobs")) + .map_err(with_err_ctx("Can't insert payloads")) } pub fn get_job(&mut self, id: Id) -> Result> { @@ -105,6 +101,16 @@ impl UDB<'_> { .map_err(with_err_ctx(format!("Can't get payload {id}"))) } + pub fn get_payload_by_name(&mut self, name: String) -> Result> { + use schema::payloads; + + payloads::table + .filter(payloads::name.eq(&name)) + .first(self.conn) + .optional() + .map_err(with_err_ctx(format!("Can't get payload by name {name}"))) + } + pub fn get_payloads(&mut self) -> Result> { use schema::payloads; @@ -113,7 +119,7 @@ impl UDB<'_> { .map_err(with_err_ctx("Can't get payloads")) } - pub fn find_job_by_alias(&mut self, alias: &str) -> Result> { + pub fn get_job_by_alias(&mut self, alias: &str) -> Result> { use schema::{jobs, payloads}; let maybe_job_with_payload = jobs::table @@ -303,6 +309,13 @@ impl UDB<'_> { Ok(()) } + pub fn update_payload(&mut self, payload: &Payload) -> Result<()> { + payload + .save_changes::(self.conn) + .map_err(with_err_ctx(format!("Can't update payload {payload:?}")))?; + Ok(()) + } + pub fn update_result(&mut self, result: &AssignedJob) -> Result<()> { debug!( "updating result: id = {}, job_id = {}, agent_id = {}", diff --git a/bin/u_server/src/error.rs b/bin/u_server/src/error.rs index 4a958dc..b2c4382 100644 --- a/bin/u_server/src/error.rs +++ b/bin/u_server/src/error.rs @@ -1,6 +1,6 @@ use diesel::result::Error as DslError; use thiserror::Error; -use u_lib::ufs; +use u_lib::{ufs, UError}; use warp::{ http::StatusCode, reject::Reject, @@ -34,6 +34,9 @@ pub enum Error { #[error("{0}\nContext: {1}")] Contexted(Box, String), + #[error(transparent)] + UError(#[from] UError), + #[error("Runtime error: {0}")] Runtime(String), } diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index 1a52b61..2ce2e2e 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -83,15 +83,25 @@ impl Endpoints { pub async fn get_payload( repo: Arc, - id: Id, + name_or_id: String, params: Option, ) -> EndpResult { - let Some(mut payload) = repo.interact(move |mut db| db.get_payload(id)).await? else { - return Err(not_found()) + let mut payload = match repo + .interact(move |mut db| match Id::parse_str(&name_or_id) { + Ok(id) => db.get_payload(id), + Err(_) => db.get_payload_by_name(name_or_id), + }) + .await? + { + Some(p) => p, + None => return Err(not_found()), }; Ok(match params.map(|p| p.brief) { - Some(BriefMode::Yes) => payload, + Some(BriefMode::Yes) => { + payload.data = None; + payload + } None | Some(BriefMode::Auto) => { payload.maybe_join_payload().map_err(Error::from)?; payload @@ -121,7 +131,7 @@ impl Endpoints { db.upsert_agent(&new_agent)?; let job = db - .find_job_by_alias("agent_hello")? + .get_job_by_alias("agent_hello")? .expect("agent_hello job not found"); db.set_jobs_for_agent(id, &[job.job.id])?; @@ -154,7 +164,33 @@ impl Endpoints { }) .collect::, Error>>()?; - repo.interact(move |mut db| db.insert_jobs(&jobs)) + let (jobs, payloads_opt): (Vec<_>, Vec<_>) = + jobs.into_iter().map(|j| (j.job, j.payload)).unzip(); + + let payloads = payloads_opt + .into_iter() + .filter_map(|p| p) + .collect::>(); + + repo.transaction(move |mut db| { + db.insert_payloads(&payloads)?; + db.insert_jobs(&jobs) + }) + .await + .map_err(From::from) + } + + pub async fn upload_payloads( + repo: Arc, + raw_payloads: Vec, + ) -> EndpResult { + let payloads = raw_payloads + .into_iter() + .map(|raw| raw.into_payload()) + .collect::, _>>() + .map_err(Error::from)?; + + repo.interact(move |mut db| db.insert_payloads(&payloads)) .await .map_err(From::from) } @@ -185,7 +221,7 @@ impl Endpoints { .into_iter() .map(|ident| { Id::parse_str(&ident).or_else(|_| { - let job_from_db = db.find_job_by_alias(&ident); + let job_from_db = db.get_job_by_alias(&ident); match job_from_db { Ok(job) => match job { Some(j) => Ok(j.job.id), @@ -262,17 +298,15 @@ impl Endpoints { } pub async fn update_agent(repo: Arc, agent: Agent) -> EndpResult { - repo.interact(move |mut db| db.upsert_agent(&agent)).await?; - Ok(()) + repo.interact(move |mut db| db.upsert_agent(&agent)) + .await + .map_err(From::from) } - pub async fn update_job(repo: Arc, mut job: Job) -> EndpResult { - if let Some(payload) = &mut job.payload { - payload.maybe_split_payload().map_err(Error::from)?; - } - - repo.interact(move |mut db| db.update_job(&job.job)).await?; - Ok(()) + pub async fn update_job(repo: Arc, job: Job) -> EndpResult { + repo.interact(move |mut db| db.update_job(&job.job)) + .await + .map_err(From::from) } pub async fn update_assigned_job( @@ -280,7 +314,27 @@ impl Endpoints { assigned: AssignedJob, ) -> EndpResult { repo.interact(move |mut db| db.update_result(&assigned)) - .await?; - Ok(()) + .await + .map_err(From::from) + } + + pub async fn update_payload( + repo: Arc, + payload: Payload, + ) -> EndpResult { + match payload.data { + Some(data) => { + let mut well_formed_payload = + Payload::from_data(data, Some(&payload.name)).map_err(Error::from)?; + well_formed_payload.id = payload.id; + + debug!("wf payload: {well_formed_payload:?}"); + + repo.interact(move |mut db| db.update_payload(&well_formed_payload)) + .await + .map_err(From::from) + } + None => return Ok(()), + } } } diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index ce1cd9e..87dfe6a 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -22,7 +22,7 @@ use warp::{ Filter, Rejection, Reply, }; -const DEFAULT_RESP: &str = "null"; +const DEFAULT_RESPONSE: &str = "null"; use crate::handlers::{Endpoints, PayloadFlags}; @@ -132,14 +132,26 @@ pub fn init_endpoints( let get_payload = path("get_payload") .and(with_db.clone()) - .and(warp::path::param::()) + .and(warp::path::param::()) .and(make_optional(serde_qs::warp::query::( create_qs_cfg(), ))) .and_then(Endpoints::get_payload) .map(into_message); - let ping = path("ping").map(|| DEFAULT_RESP); + let upload_payloads = path("upload_payloads") + .and(with_db.clone()) + .and(body::json::>()) + .and_then(Endpoints::upload_payloads) + .map(ok); + + let update_payload = path("update_payload") + .and(with_db.clone()) + .and(body::json::()) + .and_then(Endpoints::update_payload) + .map(ok); + + let ping = path("ping").map(|| DEFAULT_RESPONSE); let auth_token = format!("Bearer {auth_token}",).into_boxed_str(); let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); @@ -147,13 +159,17 @@ pub fn init_endpoints( let auth_zone = (get_agents .or(get_job.clone()) .or(get_jobs.clone()) + .or(get_payloads) + .or(get_payload) .or(upload_jobs) + .or(upload_payloads) .or(del) .or(set_jobs) .or(get_assigned_jobs) - .or(update_agent.or(update_job).or(update_assigned_job)) - .or(get_payloads) - .or(get_payload) + .or(update_agent) + .or(update_job) + .or(update_assigned_job) + .or(update_payload) .or(ping)) .and(auth_header); @@ -165,14 +181,14 @@ pub fn init_endpoints( pub async fn preload_jobs(repo: &PgRepo) -> Result<(), ServerError> { repo.interact(|mut db| { let job_alias = "agent_hello"; - let if_job_exists = db.find_job_by_alias(job_alias)?; + let if_job_exists = db.get_job_by_alias(job_alias)?; if if_job_exists.is_none() { let agent_hello = RawJob::builder() .with_type(JobType::Init) .with_alias(job_alias) .build() .unwrap(); - db.insert_jobs(&[agent_hello])?; + db.insert_jobs(&[agent_hello.job])?; } Ok(()) }) @@ -228,13 +244,13 @@ fn logger(info: Info<'_>) { .take(2) .collect::() ) - .unwrap_or_else(|| "NO_AGENT".to_string()), + .unwrap_or_else(|| "NO_AGENT_UID".to_string()), status = info.status() ); } fn ok(_: T) -> impl Reply { - DEFAULT_RESP + DEFAULT_RESPONSE } /* diff --git a/images/u_server.Dockerfile b/images/u_server.Dockerfile index 122ef4d..d9f21d6 100644 --- a/images/u_server.Dockerfile +++ b/images/u_server.Dockerfile @@ -1,3 +1,3 @@ FROM alpine:3.17 -RUN apk add iproute2 bash \ No newline at end of file +RUN apk add iproute2 bash file \ No newline at end of file diff --git a/integration-tests/docker-compose.yml b/integration-tests/docker-compose.yml index db44663..bbd3424 100644 --- a/integration-tests/docker-compose.yml +++ b/integration-tests/docker-compose.yml @@ -28,7 +28,7 @@ services: - ../.env - ../.env.private environment: - RUST_LOG: warp=info,u_server_lib=debug + RUST_LOG: warp=info,u_server_lib=debug,u_lib=debug healthcheck: test: ss -tlpn | grep 63714 interval: 5s @@ -100,5 +100,5 @@ services: - ../.env.private environment: RUST_BACKTRACE: 1 - RUST_LOG: debug,hyper=info,reqwest=info + RUST_LOG: hyper=info,reqwest=info U_SERVER: u_server \ No newline at end of file diff --git a/integration-tests/tests/integration_tests/api.rs b/integration-tests/tests/integration_tests/api.rs deleted file mode 100644 index 734ea31..0000000 --- a/integration-tests/tests/integration_tests/api.rs +++ /dev/null @@ -1,46 +0,0 @@ -// get_personal_jobs(&self, url_param: Id) -// report(&self, payload: impl OneOrVec) -// dl(&self, file: String) -// get_job(&self, job: Id) -// get_jobs(&self) -// get_agents(&self, agent: Option) -// update_agent(&self, agent: Agent) -// update_job(&self, job: FatJob) -// update_result(&self, result: AssignedJob) -// upload_jobs(&self, payload: impl OneOrVec) -// del(&self, item: Id) -// set_jobs(&self, agent: Id, job_idents: impl OneOrVec) -// get_agent_jobs(&self, agent: Option) -// ping(&self) - -use crate::fixtures::connections::*; -use u_lib::models::RawJob; - -#[rstest] -#[tokio::test] -async fn test_jobs_endpoints(client_panel: &HttpClient) { - let job_alias = "henlo"; - let mut job = RawJob::builder() - .with_shell("echo henlo") - .with_alias(job_alias) - .build() - .unwrap(); - - let job_id = job.job.id; - - client_panel.upload_jobs([&job]).await.unwrap(); - - let fetched_job = client_panel.get_brief_job(job_id).await.unwrap(); - assert_eq!(job, fetched_job); - - job.job.alias = Some("henlo2".to_string()); - client_panel.update_job(&job).await.unwrap(); - - let fetched_job = client_panel.get_brief_job(job_id).await.unwrap(); - assert_eq!(job, fetched_job); - - client_panel.del(job_id).await.unwrap(); - - let not_found_err = client_panel.get_brief_job(job_id).await.unwrap_err(); - assert!(not_found_err.to_string().contains("404 Not Found")) -} diff --git a/integration-tests/tests/integration_tests/behaviour.rs b/integration-tests/tests/integration_tests/behaviour.rs index 51ea208..f3d94ed 100644 --- a/integration-tests/tests/integration_tests/behaviour.rs +++ b/integration-tests/tests/integration_tests/behaviour.rs @@ -23,7 +23,7 @@ async fn setup_tasks() { let job_alias = "passwd_contents"; let job = RawJob::builder() .with_alias(job_alias) - .with_raw_payload(b"cat /etc/passwd".as_slice()) + .with_raw_payload("cat /etc/passwd") .with_shell("/bin/bash {}") .with_target_platforms("*linux*") .build() diff --git a/integration-tests/tests/integration_tests/endpoints.rs b/integration-tests/tests/integration_tests/endpoints.rs new file mode 100644 index 0000000..96bf960 --- /dev/null +++ b/integration-tests/tests/integration_tests/endpoints.rs @@ -0,0 +1,112 @@ +// get_personal_jobs(&self, url_param: Id) +// report(&self, payload: impl OneOrVec) +// dl(&self, file: String) +// get_job(&self, job: Id) +// get_jobs(&self) +// get_agents(&self, agent: Option) +// update_agent(&self, agent: Agent) +// update_job(&self, job: FatJob) +// update_result(&self, result: AssignedJob) +// upload_jobs(&self, payload: impl OneOrVec) +// del(&self, item: Id) +// set_jobs(&self, agent: Id, job_idents: impl OneOrVec) +// get_agent_jobs(&self, agent: Option) +// ping(&self) + +use crate::fixtures::connections::*; +use std::iter::repeat; +use u_lib::models::{BriefMode, RawJob, RawPayload, MAX_READABLE_PAYLOAD_SIZE}; + +#[rstest] +#[tokio::test] +async fn jobs_upload_update_get_del(client_panel: &HttpClient) { + let job_alias = "henlo"; + let mut job = RawJob::builder() + .with_shell("/bin/bash {}") + .with_raw_payload("echo henlo") + .with_alias(job_alias) + .build() + .unwrap(); + + let job_id = job.job.id; + + client_panel.upload_jobs([&job]).await.unwrap(); + + let fetched_job = client_panel.get_full_job(job_id).await.unwrap(); + assert_eq!(job, fetched_job); + + // update job's payload by edit existing does nothing, + // editing is only allowed from payload itself + *job.payload.as_mut().unwrap().data.as_mut().unwrap() = b"echo henlo2".to_vec(); + client_panel.update_job(&job).await.unwrap(); + + let fetched_job = client_panel.get_full_job(job_id).await.unwrap(); + assert_eq!( + fetched_job.payload.as_ref().unwrap().data.as_ref().unwrap(), + b"echo henlo" + ); + + client_panel.del(job_id).await.unwrap(); + + let not_found_err = client_panel.get_brief_job(job_id).await.unwrap_err(); + assert!(not_found_err.to_string().contains("404 Not Found")) +} + +#[rstest] +#[tokio::test] +async fn payloads_upload_update_get_del(client_panel: &HttpClient) { + let name = "test1".to_string(); + let data = b"qweasdzxc".to_vec(); + let payload = RawPayload { + name: name.clone(), + data: data.clone(), + }; + + client_panel.upload_payloads([&payload]).await.unwrap(); + + let mut fetched_payload = client_panel + .get_payload(&name, BriefMode::No) + .await + .unwrap(); + let fetched_payload_auto = client_panel + .get_payload(&name, BriefMode::Auto) + .await + .unwrap(); + + assert_eq!(fetched_payload, fetched_payload_auto); + assert_eq!(fetched_payload.data.unwrap(), data); + + let new_size = MAX_READABLE_PAYLOAD_SIZE + 1; + let big_data = repeat(1u8).take(new_size as usize).collect::>(); + + fetched_payload.data = Some(big_data.clone()); + client_panel.update_payload(&fetched_payload).await.unwrap(); + + let fetched_big_payload = client_panel + .get_payload(&name, BriefMode::Yes) + .await + .unwrap(); + let fetched_big_payload_auto = client_panel + .get_payload(&name, BriefMode::Auto) + .await + .unwrap(); + + assert_eq!(fetched_big_payload, fetched_big_payload_auto); + assert_eq!(fetched_big_payload.size, new_size); + assert!(fetched_big_payload.data.is_none()); + + let fetched_big_payload_full = client_panel + .get_payload(&name, BriefMode::No) + .await + .unwrap(); + + assert_eq!(fetched_big_payload_full.data.unwrap(), big_data); + + client_panel.del(fetched_big_payload_full.id).await.unwrap(); + + let not_found_err = client_panel + .get_payload(&name, BriefMode::Yes) + .await + .unwrap_err(); + assert!(not_found_err.to_string().contains("404 Not Found")) +} diff --git a/integration-tests/tests/integration_tests/mod.rs b/integration-tests/tests/integration_tests/mod.rs index 117235a..bbc23d4 100644 --- a/integration-tests/tests/integration_tests/mod.rs +++ b/integration-tests/tests/integration_tests/mod.rs @@ -1,3 +1,3 @@ -mod api; mod behaviour; mod connection; +mod endpoints; diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 9e5c70b..deca48a 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -31,7 +31,9 @@ pub mod retypes { pub type UpdateAgent = (); pub type UpdateJob = (); pub type UpdateResult = (); + pub type UpdatePayload = (); pub type UploadJobs = (); + pub type UploadPayloads = (); pub type Del = (); pub type SetJobs = Vec; pub type GetAgentJobs = Vec; @@ -199,12 +201,24 @@ impl HttpClient { self.req_with_payload("update_result", result).await } + pub async fn update_payload(&self, payload: &Payload) -> Result { + self.req_with_payload("update_payload", payload).await + } + /// create and upload job pub async fn upload_jobs( &self, - payload: impl IntoIterator, + jobs: impl IntoIterator, ) -> Result { - self.req_with_payload("upload_jobs", &payload.into_iter().collect::>()) + self.req_with_payload("upload_jobs", &jobs.into_iter().collect::>()) + .await + } + + pub async fn upload_payloads( + &self, + payload: impl IntoIterator, + ) -> Result { + self.req_with_payload("upload_payloads", &payload.into_iter().collect::>()) .await } @@ -239,7 +253,12 @@ impl HttpClient { self.req("get_payloads").await } - pub async fn get_payload(&self, payload: Id, brief: BriefMode) -> Result { + pub async fn get_payload( + &self, + payload: impl AsRef, + brief: BriefMode, + ) -> Result { + let payload = payload.as_ref(); self.req(format!("get_payload/{payload}?brief={brief}")) .await } diff --git a/lib/u_lib/src/logging.rs b/lib/u_lib/src/logging.rs index 018df38..e552f18 100644 --- a/lib/u_lib/src/logging.rs +++ b/lib/u_lib/src/logging.rs @@ -10,10 +10,12 @@ pub fn init_logger(logfile: Option<&str>) { env::set_var("RUST_LOG", "info") } + let layer = fmt::layer().with_line_number(true); + let output_layer = if cfg!(test) { - fmt::layer().with_test_writer().boxed() + layer.with_test_writer().boxed() } else { - fmt::layer().with_writer(stderr).boxed() + layer.with_writer(stderr).boxed() }; let reg = registry() diff --git a/lib/u_lib/src/messaging.rs b/lib/u_lib/src/messaging.rs index 721bbb8..b7a0318 100644 --- a/lib/u_lib/src/messaging.rs +++ b/lib/u_lib/src/messaging.rs @@ -13,6 +13,7 @@ impl AsMsg for AssignedJobById {} impl AsMsg for JobModel {} impl AsMsg for Reportable {} impl AsMsg for Payload {} +impl AsMsg for RawPayload {} impl AsMsg for Job {} impl AsMsg for Id {} impl AsMsg for String {} diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 1d79122..c419cf5 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -27,7 +27,7 @@ pub struct JobModel { pub exec_type: JobType, /// target triple pub target_platforms: String, - pub payload: Option, + pub payload_id: Option, /// cron-like string pub schedule: Option, } @@ -144,8 +144,8 @@ impl<'p> JobBuilder<'p> { self } - pub fn with_raw_payload(mut self, raw_payload: impl Into>) -> Self { - self.inner.raw_payload = Some(raw_payload.into()); + pub fn with_raw_payload(mut self, raw_payload: impl AsPayload<'p>) -> Self { + self.inner.raw_payload = Some(raw_payload.as_payload()); self.inner.payload_path = None; self } @@ -185,7 +185,7 @@ impl<'p> JobBuilder<'p> { if payload_from_path.is_none() { job.raw_payload .as_ref() - .map(|data| Payload::from_data(data)) + .map(|data| Payload::from_data(data, None)) .transpose()? } else { payload_from_path @@ -199,7 +199,7 @@ impl<'p> JobBuilder<'p> { id: job.id, exec_type: job.exec_type, target_platforms: job.target_platforms, - payload: payload.as_ref().map(|p| p.id), + payload_id: payload.as_ref().map(|p| p.id), schedule: job.schedule, }, payload, @@ -271,3 +271,25 @@ impl<'p> JobBuilder<'p> { } } } + +pub trait AsPayload<'p> { + fn as_payload(&self) -> Cow<'p, [u8]>; +} + +impl<'p> AsPayload<'p> for &'p str { + fn as_payload(&self) -> Cow<'p, [u8]> { + Cow::Borrowed(self.as_bytes()) + } +} + +impl<'p, const N: usize> AsPayload<'p> for &'p [u8; N] { + fn as_payload(&self) -> Cow<'p, [u8]> { + Cow::Borrowed(*self) + } +} + +impl<'p> AsPayload<'p> for &'p [u8] { + fn as_payload(&self) -> Cow<'p, [u8]> { + Cow::Borrowed(self) + } +} diff --git a/lib/u_lib/src/models/payload.rs b/lib/u_lib/src/models/payload.rs index 2b162fe..211a5aa 100644 --- a/lib/u_lib/src/models/payload.rs +++ b/lib/u_lib/src/models/payload.rs @@ -3,17 +3,30 @@ use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; use std::{fs::File, path::Path, process::Command}; -const MAX_READABLE_PAYLOAD_SIZE: i64 = 1024 * 32; +pub const MAX_READABLE_PAYLOAD_SIZE: i64 = 1024 * 32; #[cfg(feature = "server")] use crate::models::schema::*; #[cfg(feature = "server")] use diesel::Identifiable; +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RawPayload { + pub name: String, + pub data: Vec, +} + +impl RawPayload { + pub fn into_payload(self) -> Result { + Payload::from_data(self.data, Some(&self.name)) + } +} + #[cfg_attr( feature = "server", - derive(Insertable, Queryable, Identifiable), - diesel(table_name = payloads) + derive(Insertable, Queryable, Identifiable, AsChangeset), + diesel(table_name = payloads), + diesel(treat_none_as_null = true) )] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct Payload { @@ -21,7 +34,7 @@ pub struct Payload { pub mime_type: String, pub name: String, pub size: i64, - data: Option>, // when None, payload data is stored in ufs + pub data: Option>, // when None, payload data is stored in ufs } impl Payload { @@ -58,21 +71,27 @@ impl Payload { Ok(()) } - pub fn from_data(data: impl AsRef<[u8]>) -> Result { - let name = ufs::create_anonymous(data)?; - let meta = ufs::read_meta(&name)?; + pub fn from_data(data: impl AsRef<[u8]>, name: Option<&str>) -> Result { + let name = match name { + Some(name) => { + ufs::put(name, data).context("fr_put")?; + name.to_string() + } + None => ufs::create_anonymous(data).context("fr_anon")?, + }; + let meta = ufs::read_meta(&name).context("fr_me")?; + + debug!("from_data {meta:?}"); let mut payload = Payload { id: Id::new_v4(), - mime_type: get_mime_type(&meta.path)?, - name: name.clone(), + mime_type: get_mime_type(&meta.path).context(format!("fr_mi {:?}", &meta.path))?, + name, size: meta.size as i64, data: None, }; - if payload.is_human_readable() { - payload.join_payload()?; - } + payload.maybe_join_payload().context("fr_ma")?; Ok(payload) } @@ -149,11 +168,15 @@ impl Payload { } fn get_mime_type(path: impl AsRef) -> Result { + let path = path.as_ref(); + + debug!("mime of {path:?}"); + Ok(bytes_to_string( &Command::new("file") .arg("-b") .arg("--mime-type") - .arg(path.as_ref()) + .arg(path) .output() .map_err(|e| UError::IOError(e.to_string()))? .stdout, diff --git a/lib/u_lib/src/models/schema.rs b/lib/u_lib/src/models/schema.rs index 9908f4d..53a01fc 100644 --- a/lib/u_lib/src/models/schema.rs +++ b/lib/u_lib/src/models/schema.rs @@ -46,7 +46,7 @@ diesel::table! { id -> Uuid, exec_type -> Jobtype, target_platforms -> Text, - payload -> Nullable, + payload_id -> Nullable, schedule -> Nullable, } } @@ -82,7 +82,7 @@ diesel::table! { } } -diesel::joinable!(jobs -> payloads (payload)); +diesel::joinable!(jobs -> payloads (payload_id)); diesel::joinable!(results -> agents (agent_id)); diesel::joinable!(results -> jobs (job_id)); diff --git a/lib/u_lib/src/ufs/mod.rs b/lib/u_lib/src/ufs/mod.rs index c6023c5..96f0bbd 100644 --- a/lib/u_lib/src/ufs/mod.rs +++ b/lib/u_lib/src/ufs/mod.rs @@ -15,7 +15,7 @@ pub use error::Error; const OBFUSCATE: bool = cfg!(feature = "agent"); -#[derive(Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct FileMeta { extension: Option, external: bool, // if file is present before adding to index @@ -86,9 +86,7 @@ pub fn put(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<()> { let name = name.as_ref(); let data_hash = hash_data(&data); - if exists_in_index(&name) { - return Err(Error::already_exists(&name)).context("put_exists"); - } + debug!("put: {name}"); let path = match index::get_by_hash(&data_hash) { Some((_, meta)) => meta.path, @@ -105,6 +103,8 @@ pub fn put(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<()> { path }; + debug!("put: path: {path:?}"); + fs::write(&path, &data) .map_err(|e| Error::new(e, name)) .context("put_write")?; diff --git a/migrations/2020-10-24-111622_create_all/up.sql b/migrations/2020-10-24-111622_create_all/up.sql index 49fd6cf..819c7b6 100644 --- a/migrations/2020-10-24-111622_create_all/up.sql +++ b/migrations/2020-10-24-111622_create_all/up.sql @@ -38,10 +38,10 @@ CREATE TABLE IF NOT EXISTS jobs ( id UUID NOT NULL, exec_type JobType NOT NULL DEFAULT 'shell', target_platforms TEXT NOT NULL, - payload UUID, + payload_id UUID, schedule TEXT, - FOREIGN KEY(payload) REFERENCES payloads(id) ON DELETE SET NULL, + FOREIGN KEY(payload_id) REFERENCES payloads(id) ON DELETE SET NULL, PRIMARY KEY(id) ); -- 2.36.2 From 69b1d3d9014548086830520a37bcefc256403b39 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Wed, 3 May 2023 16:34:15 +0300 Subject: [PATCH 08/10] add payload-overview component & use try_from rawjob instead of builder --- Cargo.lock | 8 +- bin/u_panel/src/argparse.rs | 14 +- bin/u_panel/src/gui/fe/src/app/app.module.ts | 4 +- .../job-info-dialog.component.html | 22 +- .../job-info-dialog.component.ts | 29 +- .../payload-info-dialog.component.html | 13 +- .../payload-info-dialog.component.ts | 5 +- .../payload-overview.component.html | 9 + .../payload-overview.component.less | 0 .../payload-overview.component.ts | 22 ++ .../agent-table/agent-table.component.ts | 4 +- .../tables/base-table/base-table.component.ts | 5 +- .../tables/job-table/job-table.component.ts | 76 +++-- .../payload-table/payload-table.component.ts | 6 +- .../result-table/result-table.component.ts | 4 +- .../src/gui/fe/src/app/models/index.ts | 4 - .../src/gui/fe/src/app/models/job.model.ts | 6 +- .../gui/fe/src/app/services/api.service.ts | 23 +- bin/u_server/src/db.rs | 20 +- bin/u_server/src/handlers.rs | 14 +- bin/u_server/src/u_server.rs | 8 +- integration-tests/tests/fixtures/agent.rs | 4 +- .../tests/integration_tests/behaviour.rs | 8 +- .../tests/integration_tests/endpoints.rs | 16 +- lib/u_lib/src/api.rs | 45 +-- lib/u_lib/src/cache.rs | 2 +- lib/u_lib/src/jobs.rs | 40 +-- lib/u_lib/src/messaging.rs | 2 +- lib/u_lib/src/models/jobs/assigned.rs | 6 +- lib/u_lib/src/models/jobs/meta.rs | 298 ++++++++---------- lib/u_lib/src/models/jobs/misc.rs | 3 +- lib/u_lib/src/models/payload.rs | 10 +- 32 files changed, 364 insertions(+), 366 deletions(-) create mode 100644 bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html create mode 100644 bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.less create mode 100644 bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts diff --git a/Cargo.lock b/Cargo.lock index c65af22..06f20e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1700,9 +1700,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "platforms" @@ -2024,9 +2024,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.17" +version = "0.37.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc809f704c03a812ac71f22456c857be34185cac691a4316f27ab0f633bb9009" +checksum = "8bbfc1d1c7c40c01715f47d71444744a81669ca84e8b63e25a55e169b1f86433" dependencies = [ "bitflags", "errno 0.3.1", diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index e528aad..4a0b8cf 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -99,7 +99,7 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { CRUD::Create { item: job } => { let raw_job = from_str::(&job) .map_err(|e| UError::DeserializeError(e.to_string(), job))?; - let mut job = raw_job.validated()?; + let mut job = raw_job.try_into_job()?; if let Some(payload) = &mut job.payload { payload.join_payload()?; @@ -112,13 +112,13 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { None => into_value(client.get_jobs().await?), }, CRUD::RUD(RUD::Update { item }) => { - let raw_job = from_str::(&item) + let raw_job = from_str::(&item) .map_err(|e| UError::DeserializeError(e.to_string(), item))?; - let mut job = raw_job.validated()?; + let job = raw_job.validate()?; - if let Some(payload) = &mut job.payload { - payload.join_payload()?; - } + // if let Some(payload) = &mut job.payload { + // payload.join_payload()?; + // } into_value(client.update_job(&job).await?) } @@ -145,7 +145,7 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { } PayloadCRUD::Read { id } => match id { None => into_value(client.get_payloads().await?), - Some(id) => into_value(client.get_payload(id, args.brief).await?), + Some(id) => into_value(vec![client.get_payload(id, args.brief).await?]), }, PayloadCRUD::Update { item } => { let payload = from_str::(&item) diff --git a/bin/u_panel/src/gui/fe/src/app/app.module.ts b/bin/u_panel/src/gui/fe/src/app/app.module.ts index 71d356f..43ce309 100644 --- a/bin/u_panel/src/gui/fe/src/app/app.module.ts +++ b/bin/u_panel/src/gui/fe/src/app/app.module.ts @@ -27,6 +27,7 @@ import { MatTooltipModule } from '@angular/material/tooltip'; import { MatSnackBarModule } from '@angular/material/snack-bar'; import { MatListModule } from '@angular/material/list'; import { GlobalErrorComponent } from './components/global-error/global-error.component'; +import { PayloadOverviewComponent } from './components/payload-overview/payload-overview.component'; @NgModule({ declarations: [ @@ -40,7 +41,8 @@ import { GlobalErrorComponent } from './components/global-error/global-error.com AssignJobDialogComponent, PayloadComponent, PayloadInfoDialogComponent, - GlobalErrorComponent + GlobalErrorComponent, + PayloadOverviewComponent ], imports: [ BrowserModule, diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html index dc79aa7..ffe8910 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html @@ -4,46 +4,40 @@
ID - + Alias - + Args - +
Type - + Platform - + Schedule - +
Payload - + {{ pld[1] }} - - Payload data - - -
+ diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts index c1ab012..0f6eef0 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts @@ -1,7 +1,7 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA } from '@angular/material/dialog'; import { EventEmitter } from '@angular/core'; -import { Job } from '../../../models/job.model'; +import { Job, JobModel } from '../../../models/job.model'; import { ApiTableService } from 'src/app/services'; import { PayloadModel } from 'src/app/models'; @@ -11,36 +11,19 @@ import { PayloadModel } from 'src/app/models'; styleUrls: ['../base-info-dialog.component.less'] }) export class JobInfoDialogComponent { - isPreview = true; - isTooBigPayload = false; - decodedPayload = ""; //[id, name] - allPayloads: [string, string][] = []; + isPreview = true; + allPayloads: [string | null, string][] = [[null, "none"]]; - onSave = new EventEmitter(); + onSave = new EventEmitter(); constructor(@Inject(MAT_DIALOG_DATA) public data: Job, dataSource: ApiTableService) { - if (data.payload !== null) { - this.showPayload(data.payload) - } - dataSource.getPayloads().subscribe(resp => { - this.allPayloads = resp.map(r => [r.id, r.name]) + this.allPayloads = this.allPayloads.concat(resp.map(r => [r.id, r.name])) }) } - showPayload(payload: PayloadModel) { - if (payload.data !== null) { - this.decodedPayload = new TextDecoder().decode(new Uint8Array(payload.data)) - } else { - this.isTooBigPayload = true - } - } - updateJob() { - // if (this.decodedPayload.length > 0) { - // this.data.payload = Array.from(new TextEncoder().encode(this.decodedPayload)) - // } - // this.onSave.emit(this.data); + this.onSave.emit(this.data.meta); } } \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html index 5780429..fabd0b1 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html @@ -1,23 +1,26 @@ -

Result

+

Payload

ID - + Name - + MIME-type - + Size - +
+
+ +
diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts index ba879a4..0cafab0 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts @@ -5,10 +5,9 @@ import { PayloadModel } from 'src/app/models/payload.model'; @Component({ selector: 'payload-info-dialog', templateUrl: 'payload-info-dialog.component.html', - styleUrls: [] + styleUrls: ['../base-info-dialog.component.less'] }) export class PayloadInfoDialogComponent { - constructor(@Inject(MAT_DIALOG_DATA) public data: PayloadModel) { } - + constructor(@Inject(MAT_DIALOG_DATA) public payload: PayloadModel) { } } \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html new file mode 100644 index 0000000..b73739b --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html @@ -0,0 +1,9 @@ +
+ + Payload data + + + +
\ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.less b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.less new file mode 100644 index 0000000..e69de29 diff --git a/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts new file mode 100644 index 0000000..1a66c86 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts @@ -0,0 +1,22 @@ +import { Component, Input, OnInit } from '@angular/core'; +import { PayloadModel } from 'src/app/models'; + +@Component({ + selector: 'payload-overview', + templateUrl: './payload-overview.component.html', + styleUrls: ['./payload-overview.component.less'] +}) +export class PayloadOverviewComponent implements OnInit { + @Input() payload!: PayloadModel; + @Input("preview") isPreview = true; + isTooBigPayload = false; + decodedPayload = ""; + + ngOnInit() { + if (this.payload.data !== null) { + this.decodedPayload = new TextDecoder().decode(new Uint8Array(this.payload.data)) + } else { + this.isTooBigPayload = true + } + } +} diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts index 2a98125..89e0b4b 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts @@ -1,5 +1,5 @@ import { Component, OnInit } from '@angular/core'; -import { TablesComponent } from '../base-table/base-table.component'; +import { TableComponent } from '../base-table/base-table.component'; import { AgentModel, Area } from '../../../models'; import { AssignJobDialogComponent, AgentInfoDialogComponent } from '../../dialogs'; @@ -8,7 +8,7 @@ import { AssignJobDialogComponent, AgentInfoDialogComponent } from '../../dialog templateUrl: './agent-table.component.html', styleUrls: ['../base-table/base-table.component.less'], }) -export class AgentComponent extends TablesComponent implements OnInit { +export class AgentComponent extends TableComponent implements OnInit { area = 'agents' as Area displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions'] diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts index aa2183a..5900a79 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts @@ -6,7 +6,7 @@ import { ApiModel, Area } from '../../../models'; import { ActivatedRoute, Router } from '@angular/router'; @Directive() -export abstract class TablesComponent implements OnInit { +export abstract class TableComponent implements OnInit { abstract area: Area; table_data: MatTableDataSource = new MatTableDataSource; isLoadingResults = true; @@ -49,7 +49,8 @@ export abstract class TablesComponent implements OnInit { deleteItem(id: string) { if (confirm(`Delete ${id}?`)) { - this.dataSource.delete(id, this.area) + this.dataSource.delete(id, this.area).subscribe(_ => { }) + this.loadTableData() } } diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.ts index 4c77d3b..4419141 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.ts @@ -1,7 +1,8 @@ import { Component, OnInit } from '@angular/core'; -import { TablesComponent } from '../base-table/base-table.component'; -import { Area, JobModel } from '../../../models'; +import { TableComponent } from '../base-table/base-table.component'; +import { Area, JobModel, Job } from '../../../models'; import { JobInfoDialogComponent } from '../../dialogs'; +import { Observable } from 'rxjs'; @Component({ selector: 'job-table', @@ -9,42 +10,63 @@ import { JobInfoDialogComponent } from '../../dialogs'; styleUrls: ['../base-table/base-table.component.less'], providers: [{ provide: 'area', useValue: 'jobs' }] }) -export class JobComponent extends TablesComponent { +export class JobComponent extends TableComponent { area = 'jobs' as Area; displayedColumns = ['id', 'alias', 'platform', 'schedule', 'exec_type', 'actions'] showItemDialog(id: string | null) { - const show_dlg = (id: string, edit: boolean) => { - this.dataSource.getJob(id).subscribe(resp => { - var dialog = this.infoDialog.open(JobInfoDialogComponent, { - data: resp, - width: '1000px', - }); - if (edit) { - dialog.componentInstance.isPreview = false - } + const is_new_job = id === null; + + var dialogData$: Observable; + + if (is_new_job) { + dialogData$ = new Observable(subscriber => { + var defaultJob: Job = { + meta: { + alias: null, + argv: '', + exec_type: 'shell', + target_platforms: '*', + payload_id: null, + schedule: null + }, + payload: null + }; + subscriber.next(defaultJob) + }) + } else { + dialogData$ = this.dataSource.getJob(id) + } + + dialogData$.subscribe(dialogData => { + const dialog = this.infoDialog.open(JobInfoDialogComponent, { + data: dialogData, + width: '1000px', + }); + + dialog.componentInstance.isPreview = !is_new_job; - const saveSub = dialog.componentInstance.onSave.subscribe(result => { + const saveSub = dialog.componentInstance.onSave.subscribe(result => { + if (is_new_job) { + this.dataSource.create(dialogData.meta, this.area) + .subscribe(_ => { + alert("Created") + this.loadTableData() + }) + } else { this.dataSource.updateJob(result) .subscribe(_ => { - alert("Saved") + alert("Updated") this.loadTableData() }) - }) - - dialog.afterClosed().subscribe(result => { - saveSub.unsubscribe() - this.router.navigate(['.'], { relativeTo: this.route }) - }) + } + dialog.close() }) - } - if (id) { - show_dlg(id, false) - } else { - this.dataSource.create(null, this.area).subscribe(resp => { - show_dlg(resp[0], true) + dialog.afterClosed().subscribe(result => { + saveSub.unsubscribe() + this.router.navigate(['.'], { relativeTo: this.route }) }) - } + }) } } diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts index 7cd3ae4..9e7b6da 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts @@ -2,7 +2,7 @@ import { Component } from '@angular/core'; import { Area } from 'src/app/models'; import { PayloadModel } from 'src/app/models/payload.model'; import { PayloadInfoDialogComponent } from '../../dialogs'; -import { TablesComponent } from '../base-table/base-table.component'; +import { TableComponent } from '../base-table/base-table.component'; @Component({ selector: 'payload-table', @@ -10,9 +10,9 @@ import { TablesComponent } from '../base-table/base-table.component'; styleUrls: ['../base-table/base-table.component.less'], providers: [{ provide: 'area', useValue: 'payloads' }] }) -export class PayloadComponent extends TablesComponent { +export class PayloadComponent extends TableComponent { area = 'payloads' as Area - displayedColumns = ["name", "mime_type", "size"]; + displayedColumns = ["name", "mime_type", "size", 'actions']; showItemDialog(id: string) { this.dataSource.getPayload(id).subscribe(resp => { diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts index 5d1a4cf..511c62e 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts @@ -1,5 +1,5 @@ import { Component, OnInit } from '@angular/core'; -import { TablesComponent } from '../base-table/base-table.component'; +import { TableComponent } from '../base-table/base-table.component'; import { Area, ResultModel } from '../../../models'; import { ResultInfoDialogComponent } from '../../dialogs'; @@ -9,7 +9,7 @@ import { ResultInfoDialogComponent } from '../../dialogs'; styleUrls: ['../base-table/base-table.component.less'], providers: [{ provide: 'area', useValue: 'map' }] }) -export class ResultComponent extends TablesComponent { +export class ResultComponent extends TableComponent { area = 'map' as Area displayedColumns = [ 'id', diff --git a/bin/u_panel/src/gui/fe/src/app/models/index.ts b/bin/u_panel/src/gui/fe/src/app/models/index.ts index 3ff527f..dbe93ca 100644 --- a/bin/u_panel/src/gui/fe/src/app/models/index.ts +++ b/bin/u_panel/src/gui/fe/src/app/models/index.ts @@ -18,7 +18,3 @@ export type Area = "agents" | "jobs" | "map" | "payloads"; export type ApiModel = AgentModel | JobModel | ResultModel | PayloadModel | Empty; export interface Empty { } - -export function getAreaByModel(_: AgentModel): Area { - return "agents" -} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/models/job.model.ts b/bin/u_panel/src/gui/fe/src/app/models/job.model.ts index 301ad5a..96d97b7 100644 --- a/bin/u_panel/src/gui/fe/src/app/models/job.model.ts +++ b/bin/u_panel/src/gui/fe/src/app/models/job.model.ts @@ -3,14 +3,14 @@ import { PayloadModel } from './' export interface JobModel { alias: string | null, argv: string, - id: string, + id?: string, exec_type: string, target_platforms: string, - payload: string | null, + payload_id: string | null, schedule: string | null, } export interface Job { - job: JobModel, + meta: JobModel, payload: PayloadModel | null, } \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/services/api.service.ts b/bin/u_panel/src/gui/fe/src/app/services/api.service.ts index 4f2dc8a..be9174c 100644 --- a/bin/u_panel/src/gui/fe/src/app/services/api.service.ts +++ b/bin/u_panel/src/gui/fe/src/app/services/api.service.ts @@ -1,7 +1,7 @@ import { environment } from 'src/environments/environment'; import { HttpClient, HttpErrorResponse } from '@angular/common/http'; import { Observable, map, catchError, throwError } from 'rxjs'; -import { ApiModel, getAreaByModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, Job } from '../models'; +import { ApiModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, Job } from '../models'; import { Injectable, Inject } from '@angular/core'; import { ErrorService } from './error.service'; @@ -26,9 +26,7 @@ export class ApiTableService { requestUrl = `${environment.server}/cmd/`; req(cmd: string): Observable> { - return this.http.post>(this.requestUrl, cmd).pipe( - catchError(this.errorHandler) - ) + return this.http.post>(this.requestUrl, cmd) } getOne(id: string, area: Area, brief: 'yes' | 'no' | 'auto' | null = null): Observable { @@ -73,11 +71,11 @@ export class ApiTableService { return this.getMany('agents') } - getJobs(): Observable { + getJobs(): Observable { return this.getMany('jobs') } - getResults(): Observable { + getResults(): Observable { return this.getMany('map') } @@ -109,11 +107,12 @@ export class ApiTableService { return this.filterErrStatus(this.req(`${area} delete ${id}`)) } - create(item: string | null, area: Area): Observable { - if (!item) { - item = '"{}"' + create(item: T | null, area: Area): Observable { + var serialized = '"{}"' + if (item) { + serialized = JSON.stringify(item); } - return this.filterErrStatus(this.req(`${area} create ${item}`)) + return this.filterErrStatus(this.req(`${area} create '${serialized}'`)) } createResult(item: string): Observable { @@ -131,8 +130,8 @@ export class ApiTableService { catchError(this.errorHandler.bind(this))) } - errorHandler(err: HttpErrorResponse, _: R) { - this.errorService.handle(err.message); + errorHandler(err: HttpErrorResponse, caught: any) { + this.errorService.handle(caught.data ?? err.message); return throwError(() => new Error(err.message)); } } \ No newline at end of file diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index 9228529..81f3fd5 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -3,7 +3,7 @@ use diesel::{pg::PgConnection, prelude::*, result::Error as DslError, Connection use std::mem::drop; use u_lib::{ db::PgAsyncPool, - models::{schema, Agent, AssignedJob, Job, JobModel, JobState, Payload}, + models::{schema, Agent, AssignedJob, Job, JobMeta, JobState, Payload}, platform::Platform, types::Id, }; @@ -50,7 +50,7 @@ pub struct UDB<'c> { } impl UDB<'_> { - pub fn insert_jobs(&mut self, jobs: &[JobModel]) -> Result<()> { + pub fn insert_jobs(&mut self, jobs: &[JobMeta]) -> Result<()> { use schema::jobs; diesel::insert_into(jobs::table) @@ -67,7 +67,7 @@ impl UDB<'_> { .values(payloads) .execute(self.conn) .map(drop) - .map_err(with_err_ctx("Can't insert payloads")) + .map_err(with_err_ctx(format!("Can't insert payloads {payloads:?}"))) } pub fn get_job(&mut self, id: Id) -> Result> { @@ -76,14 +76,14 @@ impl UDB<'_> { let maybe_job_with_payload = jobs::table .left_join(payloads::table) .filter(jobs::id.eq(id)) - .first::<(JobModel, Option)>(self.conn) + .first::<(JobMeta, Option)>(self.conn) .optional() .map_err(with_err_ctx(format!("Can't get job {id}")))?; - Ok(maybe_job_with_payload.map(|(job, payload)| Job { job, payload })) + Ok(maybe_job_with_payload.map(|(job, payload)| Job { meta: job, payload })) } - pub fn get_jobs(&mut self) -> Result> { + pub fn get_jobs(&mut self) -> Result> { use schema::jobs; jobs::table @@ -125,12 +125,12 @@ impl UDB<'_> { let maybe_job_with_payload = jobs::table .left_join(payloads::table) .filter(jobs::alias.eq(alias)) - .first::<(JobModel, Option)>(self.conn) + .first::<(JobMeta, Option)>(self.conn) .optional() .map_err(with_err_ctx(format!("Can't get job by alias {alias}")))?; Ok(maybe_job_with_payload.map(|(job, payload_meta)| Job { - job, + meta: job, payload: payload_meta, })) } @@ -303,8 +303,8 @@ impl UDB<'_> { Ok(()) } - pub fn update_job(&mut self, job: &JobModel) -> Result<()> { - job.save_changes::(self.conn) + pub fn update_job(&mut self, job: &JobMeta) -> Result<()> { + job.save_changes::(self.conn) .map_err(with_err_ctx(format!("Can't update job {job:?}")))?; Ok(()) } diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index 2ce2e2e..4cb1b4a 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -134,7 +134,7 @@ impl Endpoints { .get_job_by_alias("agent_hello")? .expect("agent_hello job not found"); - db.set_jobs_for_agent(id, &[job.job.id])?; + db.set_jobs_for_agent(id, &[job.meta.id])?; } } @@ -165,7 +165,7 @@ impl Endpoints { .collect::, Error>>()?; let (jobs, payloads_opt): (Vec<_>, Vec<_>) = - jobs.into_iter().map(|j| (j.job, j.payload)).unzip(); + jobs.into_iter().map(|j| (j.meta, j.payload)).unzip(); let payloads = payloads_opt .into_iter() @@ -224,7 +224,7 @@ impl Endpoints { let job_from_db = db.get_job_by_alias(&ident); match job_from_db { Ok(job) => match job { - Some(j) => Ok(j.job.id), + Some(j) => Ok(j.meta.id), None => { Err(Error::ProcessingError(format!("unknown ident {ident}"))) } @@ -303,8 +303,8 @@ impl Endpoints { .map_err(From::from) } - pub async fn update_job(repo: Arc, job: Job) -> EndpResult { - repo.interact(move |mut db| db.update_job(&job.job)) + pub async fn update_job(repo: Arc, job: JobMeta) -> EndpResult { + repo.interact(move |mut db| db.update_job(&job.validate()?)) .await .map_err(From::from) } @@ -325,11 +325,9 @@ impl Endpoints { match payload.data { Some(data) => { let mut well_formed_payload = - Payload::from_data(data, Some(&payload.name)).map_err(Error::from)?; + Payload::from_data(data, Some(payload.name)).map_err(Error::from)?; well_formed_payload.id = payload.id; - debug!("wf payload: {well_formed_payload:?}"); - repo.interact(move |mut db| db.update_payload(&well_formed_payload)) .await .map_err(From::from) diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 87dfe6a..42e9f36 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -115,7 +115,7 @@ pub fn init_endpoints( let update_job = path("update_job") .and(with_db.clone()) - .and(body::json::()) + .and(body::json::()) .and_then(Endpoints::update_job) .map(ok); @@ -183,12 +183,12 @@ pub async fn preload_jobs(repo: &PgRepo) -> Result<(), ServerError> { let job_alias = "agent_hello"; let if_job_exists = db.get_job_by_alias(job_alias)?; if if_job_exists.is_none() { - let agent_hello = RawJob::builder() + let agent_hello = RawJob::default() .with_type(JobType::Init) .with_alias(job_alias) - .build() + .try_into_job() .unwrap(); - db.insert_jobs(&[agent_hello.job])?; + db.insert_jobs(&[agent_hello.meta])?; } Ok(()) }) diff --git a/integration-tests/tests/fixtures/agent.rs b/integration-tests/tests/fixtures/agent.rs index e0dc8e2..b441d7a 100644 --- a/integration-tests/tests/fixtures/agent.rs +++ b/integration-tests/tests/fixtures/agent.rs @@ -23,9 +23,9 @@ pub fn registered_agent(client: &HttpClient) -> RegisteredAgent { let job_id = resp.job_id; let job = client.get_job(job_id, BriefMode::No).await.unwrap(); - assert_eq!(job.job.alias, Some("agent_hello".to_string())); + assert_eq!(job.meta.alias, Some("agent_hello".to_string())); - let mut agent_data = AssignedJob::from((&job.job, resp)); + let mut agent_data = AssignedJob::from((&job.meta, resp)); agent_data.set_result(&agent); client diff --git a/integration-tests/tests/integration_tests/behaviour.rs b/integration-tests/tests/integration_tests/behaviour.rs index f3d94ed..51726a1 100644 --- a/integration-tests/tests/integration_tests/behaviour.rs +++ b/integration-tests/tests/integration_tests/behaviour.rs @@ -21,12 +21,12 @@ async fn setup_tasks() { let agents: Vec = Panel::check_output("agents read"); let agent_id = agents[0].id; let job_alias = "passwd_contents"; - let job = RawJob::builder() + let job = RawJob::default() .with_alias(job_alias) .with_raw_payload("cat /etc/passwd") .with_shell("/bin/bash {}") .with_target_platforms("*linux*") - .build() + .try_into_job() .unwrap(); Panel::check_status(["jobs", "create", &to_string(&RawJob::from(job)).unwrap()]); @@ -54,12 +54,12 @@ async fn large_payload() { let agent = &Panel::check_output::>("agents read")[0]; let agent_id = agent.id; let job_alias = "large_payload"; - let job = RawJob::builder() + let job = RawJob::default() .with_alias(job_alias) .with_payload_path("./tests/bin/echoer") .with_shell("{} type echo") .with_target_platforms(&agent.platform) - .build() + .try_into_job() .unwrap(); Panel::check_status(["jobs", "create", &to_string(&RawJob::from(job)).unwrap()]); diff --git a/integration-tests/tests/integration_tests/endpoints.rs b/integration-tests/tests/integration_tests/endpoints.rs index 96bf960..40cddd1 100644 --- a/integration-tests/tests/integration_tests/endpoints.rs +++ b/integration-tests/tests/integration_tests/endpoints.rs @@ -21,30 +21,30 @@ use u_lib::models::{BriefMode, RawJob, RawPayload, MAX_READABLE_PAYLOAD_SIZE}; #[tokio::test] async fn jobs_upload_update_get_del(client_panel: &HttpClient) { let job_alias = "henlo"; - let mut job = RawJob::builder() + let mut job = RawJob::default() .with_shell("/bin/bash {}") .with_raw_payload("echo henlo") .with_alias(job_alias) - .build() + .try_into_job() .unwrap(); - let job_id = job.job.id; + let job_id = job.meta.id; client_panel.upload_jobs([&job]).await.unwrap(); let fetched_job = client_panel.get_full_job(job_id).await.unwrap(); assert_eq!(job, fetched_job); - // update job's payload by edit existing does nothing, - // editing is only allowed from payload itself - *job.payload.as_mut().unwrap().data.as_mut().unwrap() = b"echo henlo2".to_vec(); - client_panel.update_job(&job).await.unwrap(); + let new_alias = "henlo2".to_string(); + job.meta.alias = Some(new_alias.clone()); + client_panel.update_job(&job.meta).await.unwrap(); let fetched_job = client_panel.get_full_job(job_id).await.unwrap(); assert_eq!( fetched_job.payload.as_ref().unwrap().data.as_ref().unwrap(), b"echo henlo" ); + assert_eq!(fetched_job.meta.alias, Some(new_alias)); client_panel.del(job_id).await.unwrap(); @@ -58,7 +58,7 @@ async fn payloads_upload_update_get_del(client_panel: &HttpClient) { let name = "test1".to_string(); let data = b"qweasdzxc".to_vec(); let payload = RawPayload { - name: name.clone(), + name: Some(name.clone()), data: data.clone(), }; diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index deca48a..35a7048 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -2,7 +2,7 @@ use std::fmt::Debug; use std::net::SocketAddr; use std::{collections::HashMap, time::Duration}; -use anyhow::{Context, Result}; +use anyhow::{anyhow, Context, Result}; use reqwest::{header, header::HeaderMap, Certificate, Client, Identity, Method, Url}; use serde::de::DeserializeOwned; use serde_json::{from_str, Value}; @@ -26,7 +26,7 @@ pub mod retypes { pub type Report = (); pub type GetJob = Job; pub type GetBriefJob = Job; - pub type GetJobs = Vec; + pub type GetJobs = Vec; pub type GetAgents = Vec; pub type UpdateAgent = (); pub type UpdateJob = (); @@ -58,18 +58,17 @@ impl HttpClient { default_headers.insert(header::AUTHORIZATION, format!("Bearer {pwd}")); } - // todo: don't rely only on dns resolve - let client = { - let client = Client::builder() - .identity(identity) - .default_headers(HeaderMap::try_from(&default_headers).unwrap()) - .add_root_certificate(Certificate::from_pem(ROOT_CA_CERT).unwrap()) - .timeout(Duration::from_secs(20)); + let client = Client::builder() + .identity(identity) + .default_headers(HeaderMap::try_from(&default_headers).unwrap()) + .add_root_certificate(Certificate::from_pem(ROOT_CA_CERT).unwrap()) + .timeout(Duration::from_secs(20)); + async fn resolve(domain_name: &str) -> Result { let dns_response = Client::new() .request( Method::GET, - format!("https://1.1.1.1/dns-query?name={server}&type=A"), + format!("https://1.1.1.1/dns-query?name={domain_name}&type=A"), ) .header(header::ACCEPT, "application/dns-json") .send() @@ -77,16 +76,22 @@ impl HttpClient { .text() .await?; - match from_str::(&dns_response).unwrap()["Answer"] - .get(0) + let ip = from_str::(&dns_response)? + .get("Answer") + .and_then(|a| a.get(0)) .and_then(|a| a.get("data")) - { - Some(ip) => { - let raw_addr = format!("{}:{MASTER_PORT}", ip.as_str().unwrap()); - let addr: SocketAddr = raw_addr.parse().unwrap(); - client.resolve(server, addr) - } - None => client, + .map(|ip| ip.to_owned()) + .ok_or_else(|| anyhow!("can't extract dns answer"))?; + + let raw_addr = format!("{}:{MASTER_PORT}", ip.as_str().unwrap()); + Ok(raw_addr.parse().unwrap()) + } + + let client = match resolve(server).await { + Ok(addr) => client.resolve(server, addr), + Err(e) => { + warn!("DNS error: {e}"); + client } } .build() @@ -192,7 +197,7 @@ impl HttpClient { } /// update job - pub async fn update_job(&self, job: &Job) -> Result { + pub async fn update_job(&self, job: &JobMeta) -> Result { self.req_with_payload("update_job", job).await } diff --git a/lib/u_lib/src/cache.rs b/lib/u_lib/src/cache.rs index c81414c..3031b0f 100644 --- a/lib/u_lib/src/cache.rs +++ b/lib/u_lib/src/cache.rs @@ -15,7 +15,7 @@ pub struct JobCache; impl JobCache { pub fn insert(job: Val) { - JOB_CACHE.write().insert(job.job.id, job); + JOB_CACHE.write().insert(job.meta.id, job); } pub fn contains(id: Id) -> bool { diff --git a/lib/u_lib/src/jobs.rs b/lib/u_lib/src/jobs.rs index 4c433cc..899aa67 100644 --- a/lib/u_lib/src/jobs.rs +++ b/lib/u_lib/src/jobs.rs @@ -29,7 +29,7 @@ impl AnonymousJobBatch { let jobs_ids: Vec<_> = jobs .into_iter() .map(|job| { - let job_id = job.job.id; + let job_id = job.meta.id; ( job, AssignedJobById { @@ -79,7 +79,11 @@ impl NamedJobBatch { let jobs: Vec<_> = named_jobs .into_iter() .filter_map(|(alias, cmd)| { - match RawJob::builder().with_shell(cmd).with_alias(alias).build() { + match RawJob::default() + .with_shell(cmd) + .with_alias(alias) + .try_into_job() + { Ok(jpm) => Some(jpm), Err(e) => { result.push_err(e); @@ -95,7 +99,7 @@ impl NamedJobBatch { pub fn from_meta(named_jobs: Vec) -> Self { let (job_names, jobs): (Vec<_>, Vec<_>) = named_jobs .into_iter() - .map(|job| (job.job.alias.clone().unwrap(), job)) + .map(|job| (job.meta.alias.clone().unwrap(), job)) .unzip(); Self { runner: Some(AnonymousJobBatch::from_meta(jobs)), @@ -129,17 +133,17 @@ impl NamedJobBatch { } pub async fn run_assigned_job(job: Job, ids: AssignedJobById) -> ExecResult { - let Job { job, payload } = job; - let mut result = AssignedJob::from((&job, ids)); - match job.exec_type { + let Job { meta, payload } = job; + let mut result = AssignedJob::from((&meta, ids)); + match meta.exec_type { JobType::Shell => { let (argv, _prepared_payload) = { if let Some(payload) = payload { let (prep_exec, prep_exec_path) = payload.prepare_executable()?; - let argv_with_exec = job.argv.replace("{}", &prep_exec_path); + let argv_with_exec = meta.argv.replace("{}", &prep_exec_path); (argv_with_exec, Some(prep_exec)) } else { - (job.argv, None) + (meta.argv, None) } }; @@ -227,11 +231,11 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] expected_result: &str, ) -> TestResult { - let mut job = RawJob::builder().with_shell(cmd); + let mut raw_job = RawJob::default().with_shell(cmd); if let Some(p) = payload { - job = job.with_raw_payload(p); + raw_job = raw_job.with_raw_payload(p); } - let job = job.build().unwrap(); + let job = raw_job.try_into_job().unwrap(); let result = AnonymousJobBatch::from_meta([job]) .wait_one() .await @@ -310,11 +314,11 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] err_str: &str, ) -> TestResult { - let mut job = RawJob::builder().with_shell(cmd); + let mut raw_job = RawJob::default().with_shell(cmd); if let Some(p) = payload { - job = job.with_raw_payload(p); + raw_job = raw_job.with_raw_payload(p); } - let err = job.build().unwrap_err(); + let err = raw_job.try_into_job().unwrap_err(); let err_msg = unwrap_enum!(err, UError::JobBuildError); assert!(err_msg.contains(err_str)); Ok(()) @@ -323,15 +327,15 @@ mod tests { #[tokio::test] async fn test_different_job_types() -> TestResult { let mut jobs = NamedJobBatch::from_meta(vec![ - RawJob::builder() + RawJob::default() .with_shell("sleep 3") .with_alias("sleeper") - .build() + .try_into_job() .unwrap(), - RawJob::builder() + RawJob::default() .with_type(JobType::Init) .with_alias("gatherer") - .build() + .try_into_job() .unwrap(), ]) .wait() diff --git a/lib/u_lib/src/messaging.rs b/lib/u_lib/src/messaging.rs index b7a0318..b1e1ecf 100644 --- a/lib/u_lib/src/messaging.rs +++ b/lib/u_lib/src/messaging.rs @@ -10,7 +10,7 @@ pub trait AsMsg: Clone + Serialize + Debug {} impl AsMsg for Agent {} impl AsMsg for AssignedJob {} impl AsMsg for AssignedJobById {} -impl AsMsg for JobModel {} +impl AsMsg for JobMeta {} impl AsMsg for Reportable {} impl AsMsg for Payload {} impl AsMsg for RawPayload {} diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index 946fc4b..fef995f 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -1,4 +1,4 @@ -use super::{JobModel, JobState, JobType}; +use super::{JobMeta, JobState, JobType}; #[cfg(feature = "server")] use crate::models::schema::*; use crate::{ @@ -60,8 +60,8 @@ pub struct AssignedJobById { pub job_id: Id, } -impl From<(&JobModel, AssignedJobById)> for AssignedJob { - fn from((job, ids): (&JobModel, AssignedJobById)) -> Self { +impl From<(&JobMeta, AssignedJobById)> for AssignedJob { + fn from((job, ids): (&JobMeta, AssignedJobById)) -> Self { let AssignedJobById { agent_id, id, diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index c419cf5..3611ad7 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -12,34 +12,14 @@ use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; use std::borrow::Cow; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr( feature = "server", derive(Queryable, Identifiable, Insertable, AsChangeset), - diesel(table_name = jobs) + diesel(table_name = jobs), + diesel(treat_none_as_null = true) )] -pub struct JobModel { - pub alias: Option, - /// string like `bash -c {} -a 1 --arg2`, - /// where {} is replaced by executable's tmp path - pub argv: String, - pub id: Id, - pub exec_type: JobType, - /// target triple - pub target_platforms: String, - pub payload_id: Option, - /// cron-like string - pub schedule: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Job { - pub job: JobModel, - pub payload: Option, -} - -#[derive(Serialize, Deserialize, Clone)] -pub struct RawJob<'p> { +pub struct JobMeta { #[serde(default)] pub alias: Option, @@ -59,17 +39,14 @@ pub struct RawJob<'p> { pub target_platforms: String, #[serde(default)] - pub payload_path: Option, - - #[serde(default)] - pub raw_payload: Option>, + pub payload_id: Option, /// cron-like string #[serde(default)] pub schedule: Option, } -impl Default for RawJob<'_> { +impl Default for JobMeta { fn default() -> Self { Self { alias: None, @@ -77,198 +54,181 @@ impl Default for RawJob<'_> { id: Id::new_v4(), exec_type: JobType::default(), target_platforms: String::new(), - payload_path: None, - raw_payload: None, + payload_id: None, schedule: None, } } } -impl fmt::Debug for RawJob<'_> { +impl fmt::Debug for JobMeta { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RawJob") + f.debug_struct("JobMeta") .field("alias", &self.alias) .field("argv", &self.argv) .field("id", &self.id.to_string()) .field("exec_type", &self.exec_type) - .field("platform", &self.target_platforms) - .field("payload_path", &self.payload_path) - .field("raw_payload", &self.raw_payload) + .field("target_platforms", &self.target_platforms) + .field("payload_id", &self.payload_id.map(|id| id.to_string())) .field("schedule", &self.schedule) .finish() } } +impl JobMeta { + pub fn validate(mut self) -> UResult { + fn mk_err(msg: impl Into) -> UError { + UError::JobBuildError(msg.into()) + } + + const ARGV_STR_LEN: usize = 2048; + + if self.argv.is_empty() { + // TODO: fix detecting + self.argv = String::from("echo 'hello, world!'") + } else if self.argv.len() > ARGV_STR_LEN { + return Err(mk_err(format!( + "argv length limit ({ARGV_STR_LEN}) exceeded" + ))); + } + + let argv_parts = shlex::split(&self.argv).ok_or_else(|| mk_err("Shlex failed"))?; + let empty_err = mk_err("Empty argv"); + + if argv_parts.get(0).ok_or(empty_err.clone())?.is_empty() { + return Err(empty_err); + } + + if self.payload_id.is_some() && !self.argv.contains("{}") { + return Err(mk_err("Argv contains no executable placeholder")); + } + + if self.argv.contains("{}") && self.payload_id.is_none() { + return Err(mk_err( + "No payload provided, but argv contains executable placeholder", + )); + } + + if self.target_platforms.is_empty() { + self.target_platforms = "*".to_string(); + } + + if !platform::is_valid_glob(&self.target_platforms) { + return Err(mk_err(format!( + "Unknown platform '{}'", + self.target_platforms + ))); + } + + Ok(self) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Job { + pub meta: JobMeta, + pub payload: Option, +} + +#[derive(Serialize, Deserialize, Clone, Default)] +pub struct RawJob<'p> { + #[serde(default)] + payload_path: Option, + + #[serde(default)] + raw_payload: Option>, + + #[serde(default, flatten)] + meta: JobMeta, +} + impl From for RawJob<'_> { fn from(job: Job) -> Self { - let Job { - job, - payload: payload_meta, - } = job; + let Job { meta, payload } = job; + RawJob { - alias: job.alias, - argv: job.argv, - id: job.id, - exec_type: job.exec_type, - target_platforms: job.target_platforms, - payload_path: payload_meta.map(|m| m.name), + payload_path: payload.map(|m| m.name), raw_payload: None, - schedule: job.schedule, + meta, } } } impl<'p> RawJob<'p> { - pub fn validated(self) -> UResult { - JobBuilder { inner: self }.build() + pub fn try_into_job(self) -> UResult { + Job::try_from(self) } pub fn from_shell(cmd: impl Into) -> UResult { - Self::builder().with_shell(cmd).build() + Self::default().with_shell(cmd).try_into_job() } - pub fn builder() -> JobBuilder<'p> { - JobBuilder::default() - } -} - -#[derive(Default)] -pub struct JobBuilder<'p> { - inner: RawJob<'p>, -} - -impl<'p> JobBuilder<'p> { pub fn with_shell(mut self, shell_cmd: impl Into) -> Self { - self.inner.argv = shell_cmd.into(); - self.inner.exec_type = JobType::Shell; + self.meta.argv = shell_cmd.into(); + self.meta.exec_type = JobType::Shell; self } - pub fn with_raw_payload(mut self, raw_payload: impl AsPayload<'p>) -> Self { - self.inner.raw_payload = Some(raw_payload.as_payload()); - self.inner.payload_path = None; + pub fn with_alias(mut self, alias: impl Into) -> Self { + self.meta.alias = Some(alias.into()); self } - pub fn with_payload_path(mut self, path: impl Into) -> Self { - self.inner.payload_path = Some(path.into()); - self.inner.raw_payload = None; + pub fn with_type(mut self, e_type: JobType) -> Self { + self.meta.exec_type = e_type; self } - pub fn with_alias(mut self, alias: impl Into) -> Self { - self.inner.alias = Some(alias.into()); + pub fn with_target_platforms(mut self, platform: impl Into) -> Self { + self.meta.target_platforms = platform.into(); self } - pub fn with_type(mut self, e_type: JobType) -> Self { - self.inner.exec_type = e_type; + pub fn with_raw_payload(mut self, raw_payload: impl AsPayload<'p>) -> Self { + self.raw_payload = Some(raw_payload.as_payload()); + self.payload_path = None; self } - pub fn with_target_platforms(mut self, platform: impl Into) -> Self { - self.inner.target_platforms = platform.into(); + pub fn with_payload_path(mut self, path: impl Into) -> Self { + self.payload_path = Some(path.into()); + self.raw_payload = None; self } +} - pub fn build(self) -> UResult { - let mut inner = self.inner; +impl TryFrom> for Job { + type Error = UError; - fn _build(job: RawJob) -> UResult { - let payload = { - let payload_from_path = job - .payload_path - .as_ref() - .map(|path| Payload::from_path(path)) - .transpose()?; - - if payload_from_path.is_none() { - job.raw_payload - .as_ref() - .map(|data| Payload::from_data(data, None)) - .transpose()? - } else { - payload_from_path - } - }; - - Ok(Job { - job: JobModel { - alias: job.alias, - argv: job.argv, - id: job.id, - exec_type: job.exec_type, - target_platforms: job.target_platforms, - payload_id: payload.as_ref().map(|p| p.id), - schedule: job.schedule, - }, - payload, - }) + fn try_from(mut raw: RawJob) -> Result { + if raw.raw_payload.is_some() && raw.payload_path.is_some() { + return Err(UError::JobBuildError( + "Can't use both raw payload with payload path".to_string(), + )); } - match inner.exec_type { - JobType::Shell => { - const ARGV_STR_LEN: usize = 2048; - - if inner.argv.is_empty() { - // TODO: fix detecting - inner.argv = String::from("echo 'hello, world!'") - } else if inner.argv.len() > ARGV_STR_LEN { - return Err(UError::JobBuildError(format!( - "argv length limit ({ARGV_STR_LEN}) exceeded" - ))); - } - - let argv_parts = shlex::split(&inner.argv) - .ok_or(UError::JobBuildError("Shlex failed".into()))?; - let empty_err = UError::JobBuildError("Empty argv".into()); - - if argv_parts.get(0).ok_or(empty_err.clone())?.is_empty() { - return Err(empty_err.into()); - } - - if inner.raw_payload.is_some() && inner.payload_path.is_some() { - return Err(UError::JobBuildError( - "Can't use both raw payload with payload path".to_string(), - )); - } - - match inner.payload_path.as_ref() { - Some(_) | None if inner.raw_payload.is_some() => { - if !inner.argv.contains("{}") { - return Err(UError::JobBuildError( - "Argv contains no executable placeholder".into(), - ) - .into()); - } - } - None => { - if inner.argv.contains("{}") && inner.raw_payload.is_none() { - return Err(UError::JobBuildError( - "No payload provided, but argv contains executable placeholder" - .into(), - ) - .into()); - } - } - _ => (), - }; - - if inner.target_platforms.is_empty() { - inner.target_platforms = "*".to_string(); - } - - if !platform::is_valid_glob(&inner.target_platforms) { - return Err(UError::JobBuildError(format!( - "Unknown platform '{}'", - inner.target_platforms - ))); - } - - _build(inner) + let payload = { + let payload_from_path = raw + .payload_path + .as_ref() + .map(|path| Payload::from_path(path)) + .transpose()?; + + if payload_from_path.is_none() { + raw.raw_payload + .as_ref() + .map(|data| Payload::from_data(data, None)) + .transpose()? + } else { + payload_from_path } - _ => _build(inner), - } + }; + + raw.meta.payload_id = payload.as_ref().map(|p| p.id); + + Ok(Job { + meta: raw.meta.validate()?, + payload, + }) } } diff --git a/lib/u_lib/src/models/jobs/misc.rs b/lib/u_lib/src/models/jobs/misc.rs index 94de0b1..faf108e 100644 --- a/lib/u_lib/src/models/jobs/misc.rs +++ b/lib/u_lib/src/models/jobs/misc.rs @@ -26,7 +26,8 @@ pub enum JobState { Finished, } -#[derive(Default, Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Display)] +#[derive(Default, Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] #[cfg_attr( feature = "server", derive(DbEnum), diff --git a/lib/u_lib/src/models/payload.rs b/lib/u_lib/src/models/payload.rs index 211a5aa..9b982bf 100644 --- a/lib/u_lib/src/models/payload.rs +++ b/lib/u_lib/src/models/payload.rs @@ -12,13 +12,13 @@ use diesel::Identifiable; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RawPayload { - pub name: String, + pub name: Option, pub data: Vec, } impl RawPayload { pub fn into_payload(self) -> Result { - Payload::from_data(self.data, Some(&self.name)) + Payload::from_data(self.data, self.name) } } @@ -71,11 +71,11 @@ impl Payload { Ok(()) } - pub fn from_data(data: impl AsRef<[u8]>, name: Option<&str>) -> Result { + pub fn from_data(data: impl AsRef<[u8]>, name: Option) -> Result { let name = match name { Some(name) => { - ufs::put(name, data).context("fr_put")?; - name.to_string() + ufs::put(&name, data).context("fr_put")?; + name } None => ufs::create_anonymous(data).context("fr_anon")?, }; -- 2.36.2 From 886d4833fbc1183cda96f2206b1f8dc66f47667c Mon Sep 17 00:00:00 2001 From: plazmoid Date: Fri, 6 Oct 2023 02:50:47 +0300 Subject: [PATCH 09/10] implement payload table --- .cargo/config.toml | 11 +- Cargo.lock | 1146 ++++++++--------- Cargo.toml | 1 + Makefile.toml | 13 +- bin/u_panel/src/argparse.rs | 4 +- bin/u_panel/src/gui/fe/src/app/app.module.ts | 4 +- .../job-info-dialog.component.html | 4 +- .../job-info-dialog.component.ts | 1 - .../new-payload-dialog.component.html | 20 + .../new-payload-dialog.component.ts | 43 + .../payload-info-dialog.component.html | 45 +- .../payload-info-dialog.component.ts | 9 +- .../result-info-dialog.component.html | 8 +- .../payload-overview.component.html | 13 +- .../payload-overview.component.ts | 7 +- .../tables/base-table/base-table.component.ts | 4 +- .../payload-table.component.html | 3 + .../payload-table/payload-table.component.ts | 49 +- .../gui/fe/src/app/models/payload.model.ts | 5 + .../src/gui/fe/src/app/models/result.model.ts | 2 +- .../gui/fe/src/app/services/api.service.ts | 11 +- bin/u_panel/src/gui/mod.rs | 4 +- bin/u_server/src/db.rs | 17 +- bin/u_server/src/handlers.rs | 103 +- bin/u_server/src/u_server.rs | 9 +- images/tests_runner.Dockerfile | 2 +- integration-tests/tests/fixtures/agent.rs | 2 +- .../tests/integration_tests/endpoints.rs | 31 +- lib/u_lib/src/api.rs | 52 +- lib/u_lib/src/models/jobs/meta.rs | 2 +- lib/u_lib/src/models/mod.rs | 2 +- lib/u_lib/src/models/payload.rs | 7 +- .../2020-10-24-111622_create_all/up.sql | 2 +- 33 files changed, 853 insertions(+), 783 deletions(-) create mode 100644 bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.html create mode 100644 bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.ts diff --git a/.cargo/config.toml b/.cargo/config.toml index 433fea9..241bfac 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,6 +1,15 @@ [build] rustflags = [ - "-L", "/home/ortem/src/rust/unki/static/lib", + "-L/usr/lib/musl/lib", + "-L/home/ortem/src/rust/unki/static/lib", "--remap-path-prefix=/home/ortem/src/rust/unki=src", "--remap-path-prefix=/home/ortem/.cargo=cargo" ] +target = "x86_64-unknown-linux-musl" + +[env] +STATIC_PREFIX = "static" +PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true" +PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = { value = "static/bin/pg_config", relative = true } +OPENSSL_STATIC = "true" +OPENSSL_DIR = { value = "static", relative = true } diff --git a/Cargo.lock b/Cargo.lock index 06f20e1..40d3168 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,19 +4,19 @@ version = 3 [[package]] name = "actix-codec" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a7559404a7f3573127aab53c08ce37a6c6a315c374a31070f3c91cd1b4a7fe" +checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "bytes", "futures-core", "futures-sink", - "log", "memchr", "pin-project-lite", "tokio", "tokio-util", + "tracing", ] [[package]] @@ -36,17 +36,17 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.3.1" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2079246596c18b4a33e274ae10c0e50613f4d32a4198e09c7b93771013fed74" +checksum = "a92ef85799cba03f76e4f7c10f533e66d87c9a7e7055f3391f09000ad8351bc9" dependencies = [ "actix-codec", "actix-rt", "actix-service", "actix-utils", - "ahash 0.8.3", - "base64 0.21.0", - "bitflags", + "ahash", + "base64", + "bitflags 2.4.0", "brotli", "bytes", "bytestring", @@ -75,12 +75,12 @@ dependencies = [ [[package]] name = "actix-macros" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" +checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 1.0.109", + "syn 2.0.38", ] [[package]] @@ -98,9 +98,9 @@ dependencies = [ [[package]] name = "actix-rt" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" +checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" dependencies = [ "futures-core", "tokio", @@ -108,9 +108,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e8613a75dd50cc45f473cee3c34d59ed677c0f7b44480ce3b8247d7dc519327" +checksum = "3eb13e7eef0423ea6eab0e59f6c72e7cb46d33691ad56a726b3cd07ddec2c2d4" dependencies = [ "actix-rt", "actix-service", @@ -118,8 +118,7 @@ dependencies = [ "futures-core", "futures-util", "mio", - "num_cpus", - "socket2", + "socket2 0.5.4", "tokio", "tracing", ] @@ -147,9 +146,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.3.1" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3cb42f9566ab176e1ef0b8b3a896529062b4efc6be0123046095914c4c1c96" +checksum = "0e4a5b5e29603ca8c94a77c65cf874718ceb60292c5a5c3e5f4ace041af462b9" dependencies = [ "actix-codec", "actix-http", @@ -160,7 +159,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web-codegen", - "ahash 0.7.6", + "ahash", "bytes", "bytestring", "cfg-if 1.0.0", @@ -169,7 +168,6 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "http", "itoa", "language-tags", "log", @@ -181,21 +179,30 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2", - "time 0.3.20", + "socket2 0.5.4", + "time", "url", ] [[package]] name = "actix-web-codegen" -version = "4.2.0" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9" +checksum = "eb1f50ebbb30eca122b188319a4398b3f7bb4a8cdf50ecfb73bfc6a3c3ce54f5" dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.38", +] + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", ] [[package]] @@ -210,17 +217,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" -[[package]] -name = "ahash" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" -dependencies = [ - "getrandom", - "once_cell", - "version_check", -] - [[package]] name = "ahash" version = "0.8.3" @@ -235,9 +231,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" dependencies = [ "memchr", ] @@ -257,6 +253,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -283,13 +285,13 @@ checksum = "a26fa4d7e3f2eebadf743988fc8aec9fa9a9e82611acafd77c1462ed6262440a" [[package]] name = "async-trait" -version = "0.1.68" +version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.38", ] [[package]] @@ -310,16 +312,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] -name = "base64" -version = "0.13.1" +name = "backtrace" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] [[package]] name = "base64" -version = "0.21.0" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "bincode" @@ -336,6 +347,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + [[package]] name = "block-buffer" version = "0.10.4" @@ -347,9 +364,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.3.4" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -358,9 +375,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.3.4" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -368,9 +385,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.1" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "byteorder" @@ -380,9 +397,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "bytestring" @@ -395,11 +412,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", + "libc", ] [[package]] @@ -416,17 +434,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.24" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", - "time 0.1.45", "wasm-bindgen", - "winapi", + "windows-targets", ] [[package]] @@ -437,23 +454,13 @@ checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ "ansi_term", "atty", - "bitflags", + "bitflags 1.3.2", "strsim", "textwrap", "unicode-width", "vec_map", ] -[[package]] -name = "codespan-reporting" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] - [[package]] name = "convert_case" version = "0.4.0" @@ -467,7 +474,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding", - "time 0.3.20", + "time", "version_check", ] @@ -489,9 +496,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -528,9 +535,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.14" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if 1.0.0", @@ -541,9 +548,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if 1.0.0", ] @@ -560,56 +567,12 @@ dependencies = [ [[package]] name = "ctor" -version = "0.2.0" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4056f63fce3b82d852c3da92b08ea59959890813a7f4ce9c0ff85b10cf301b" +checksum = "37e366bff8cd32dd8754b0991fb66b279dc48f598c3a18914852a6673deef583" dependencies = [ "quote", - "syn 2.0.15", -] - -[[package]] -name = "cxx" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn 2.0.15", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", + "syn 2.0.38", ] [[package]] @@ -621,6 +584,12 @@ dependencies = [ "libc", ] +[[package]] +name = "data-encoding" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" + [[package]] name = "deadpool" version = "0.9.5" @@ -647,22 +616,28 @@ dependencies = [ [[package]] name = "deadpool-runtime" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaa37046cc0f6c3cc6090fbdbf73ef0b8ef4cfcc37f6befc0020f63e8cf121e1" +checksum = "63dfa964fe2a66f3fde91fc70b267fe193d822c7e603e2a675a49a7f46ad3f49" dependencies = [ "tokio", ] [[package]] name = "deadpool-sync" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1bea344b64b32537fde6e0f0179b1ede34d435636719dd40fe6a0f28218a61c" +checksum = "f8db70494c13cae4ce67b4b4dafdaf828cf0df7237ab5b9e2fcabee4965d0a0a" dependencies = [ - "deadpool", + "deadpool-runtime", ] +[[package]] +name = "deranged" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" + [[package]] name = "derive_more" version = "0.99.17" @@ -678,11 +653,11 @@ dependencies = [ [[package]] name = "diesel" -version = "2.0.4" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72eb77396836a4505da85bae0712fa324b74acfe1876d7c2f7e694ef3d0ee373" +checksum = "2268a214a6f118fce1838edba3d1561cf0e78d8de785475957a580a7f8c69d33" dependencies = [ - "bitflags", + "bitflags 2.4.0", "byteorder", "diesel_derives", "itoa", @@ -693,44 +668,53 @@ dependencies = [ [[package]] name = "diesel-derive-enum" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b10c03b954333d05bfd5be1d8a74eae2c9ca77b86e0f1c3a1ea29c49da1d6c2" +checksum = "81c5131a2895ef64741dad1d483f358c2a229a3a2d1b256778cdc5e146db64d4" dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.38", ] [[package]] name = "diesel_derives" -version = "2.0.2" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad74fdcf086be3d4fdd142f67937678fe60ed431c3b2f08599e7687269410c4" +checksum = "ef8337737574f55a468005a83499da720f20c65586241ffea339db9ecdfd2b44" dependencies = [ - "proc-macro-error", + "diesel_table_macro_syntax", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.38", ] [[package]] name = "diesel_migrations" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9ae22beef5e9d6fab9225ddb073c1c6c1a7a6ded5019d5da11d1e5c5adc34e2" +checksum = "6036b3f0120c5961381b570ee20a02432d7e2d27ea60de9578799cf9156914ac" dependencies = [ "diesel", "migrations_internals", "migrations_macros", ] +[[package]] +name = "diesel_table_macro_syntax" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" +dependencies = [ + "syn 2.0.38", +] + [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -750,15 +734,15 @@ checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if 1.0.0", ] @@ -772,6 +756,12 @@ dependencies = [ "serde", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.2.8" @@ -785,13 +775,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -806,18 +796,15 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" dependencies = [ "crc32fast", "miniz_oxide", @@ -846,9 +833,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] @@ -909,7 +896,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.38", ] [[package]] @@ -960,15 +947,21 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", ] +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + [[package]] name = "guess_host_triple" version = "0.1.3" @@ -983,9 +976,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.18" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes", "fnv", @@ -993,7 +986,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -1006,14 +999,19 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" + [[package]] name = "headers" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" +checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" dependencies = [ - "base64 0.13.1", - "bitflags", + "base64", "bytes", "headers-core", "http", @@ -1057,18 +1055,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "http" @@ -1100,15 +1089,15 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -1121,7 +1110,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -1143,9 +1132,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1157,19 +1146,18 @@ dependencies = [ [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", ] [[package]] name = "idna" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1177,9 +1165,9 @@ dependencies = [ [[package]] name = "include-flate" -version = "0.1.4" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfdcb449c721557c1cf89bbd3412bf33fa963289e26e9badbd824a960912e148" +checksum = "c2e11569346406931d20276cc460215ee2826e7cad43aa986999cb244dd7adb0" dependencies = [ "include-flate-codegen-exports", "lazy_static", @@ -1216,16 +1204,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", ] [[package]] -name = "instant" -version = "0.1.12" +name = "indexmap" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ - "cfg-if 1.0.0", + "equivalent", + "hashbrown 0.14.1", ] [[package]] @@ -1245,28 +1234,17 @@ dependencies = [ "uuid", ] -[[package]] -name = "io-lifetimes" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" -dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "ipnet" -version = "2.7.2" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" @@ -1279,18 +1257,18 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] [[package]] name = "keccak" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3afef3b6eff9ce9d8ff9b3601125eec7f0c8cbac7abd14f355d053fa56c98768" +checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" dependencies = [ "cpufeatures", ] @@ -1309,15 +1287,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.142" +version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" [[package]] name = "libflate" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97822bf791bd4d5b403713886a5fbe8bf49520fe78e323b0dc480ca1a03e50b0" +checksum = "5ff4ae71b685bbad2f2f391fe74f6b7659a34871c08b210fdc039e43bee07d18" dependencies = [ "adler32", "crc32fast", @@ -1333,30 +1311,20 @@ dependencies = [ "rle-decode-fast", ] -[[package]] -name = "link-cplusplus" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] - [[package]] name = "linux-raw-sys" -version = "0.3.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b64f40e5e03e0d54f03845c8197d0291253cdbedfb1cb46b13c2c117554a9f4c" +checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" [[package]] name = "local-channel" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f303ec0e94c6c54447f84f3b0ef7af769858a9c4ef56ef2a986d3dcd4c3fc9c" +checksum = "e0a493488de5f18c8ffcba89eebb8532ffc562dc400490eb65b84893fae0b178" dependencies = [ "futures-core", "futures-sink", - "futures-util", "local-waker", ] @@ -1368,9 +1336,9 @@ checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -1378,12 +1346,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "matchers" @@ -1391,29 +1356,29 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] [[package]] name = "migrations_internals" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c493c09323068c01e54c685f7da41a9ccf9219735c3766fbfd6099806ea08fbc" +checksum = "0f23f71580015254b020e856feac3df5878c2c7a8812297edd6c0a485ac9dada" dependencies = [ "serde", "toml", @@ -1421,9 +1386,9 @@ dependencies = [ [[package]] name = "migrations_macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a8ff27a350511de30cdabb77147501c36ef02e0451d957abea2f30caffb2b58" +checksum = "cce3325ac70e67bbab5bd837a31cae01f1a6db64e0e744a33cb03a543469ef08" dependencies = [ "migrations_internals", "proc-macro2", @@ -1468,14 +1433,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "wasi", + "windows-sys", ] [[package]] @@ -1520,7 +1485,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cc", "cfg-if 0.1.10", "libc", @@ -1547,47 +1512,46 @@ dependencies = [ ] [[package]] -name = "num-integer" -version = "0.1.45" +name = "num-traits" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", - "num-traits", ] [[package]] -name = "num-traits" -version = "0.2.15" +name = "num_cpus" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "autocfg", + "hermit-abi 0.3.3", + "libc", ] [[package]] -name = "num_cpus" -version = "1.15.0" +name = "object" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ - "hermit-abi 0.2.6", - "libc", + "memchr", ] [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "openssl" -version = "0.10.52" +version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags", + "bitflags 2.4.0", "cfg-if 1.0.0", "foreign-types", "libc", @@ -1604,7 +1568,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.38", ] [[package]] @@ -1615,9 +1579,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.87" +version = "0.9.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" dependencies = [ "cc", "libc", @@ -1643,54 +1607,54 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.2.16", + "redox_syscall", "smallvec", - "windows-sys 0.45.0", + "windows-targets", ] [[package]] name = "paste" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.38", ] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -1706,9 +1670,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "platforms" -version = "3.0.2" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d7ddaed09e0eb771a79ab0fd64609ba0afb0a8366421957936ad14cbd13630" +checksum = "4503fa043bf02cee09a9582e9554b4c6403b2ef55e4612e96561d294419429f8" [[package]] name = "ppv-lite86" @@ -1757,18 +1721,18 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.26" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -1805,9 +1769,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ "either", "rayon-core", @@ -1815,23 +1779,12 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", ] [[package]] @@ -1840,18 +1793,19 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.8.1" +version = "1.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.1", + "regex-automata 0.3.9", + "regex-syntax 0.7.5", ] [[package]] @@ -1863,6 +1817,17 @@ dependencies = [ "regex-syntax 0.6.29", ] +[[package]] +name = "regex-automata" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.7.5", +] + [[package]] name = "regex-syntax" version = "0.6.29" @@ -1871,17 +1836,17 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.1" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "reqwest" -version = "0.11.17" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13293b639a097af28fc8a90f22add145a9c954e49d77da06263d58cf44d5fb91" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ - "base64 0.21.0", + "base64", "bytes", "encoding_rs", "futures-core", @@ -1902,6 +1867,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-native-tls", "tower-service", @@ -1980,9 +1946,9 @@ dependencies = [ [[package]] name = "rust-embed" -version = "6.6.1" +version = "6.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b68543d5527e158213414a92832d2aab11a84d2571a5eb021ebe22c43aab066" +checksum = "a36224c3276f8c4ebc8c20f158eca7ca4359c8db89991c4925132aaaf6702661" dependencies = [ "include-flate", "rust-embed-impl", @@ -1992,27 +1958,33 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "6.5.0" +version = "6.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4e0f0ced47ded9a68374ac145edd65a6c1fa13a96447b873660b2a568a0fd7" +checksum = "49b94b81e5b2c284684141a2fb9e2a31be90638caf040bf9afbc5a0416afe1ac" dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 1.0.109", + "syn 2.0.38", "walkdir", ] [[package]] name = "rust-embed-utils" -version = "7.5.0" +version = "7.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512b0ab6853f7e14e3c8754acb43d6f748bb9ced66aa5915a6553ac8213f7731" +checksum = "9d38ff6bf570dc3bb7100fce9f7b60c33fa71d80e88da3f2580df4ff2bdded74" dependencies = [ "sha2", "walkdir", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc_version" version = "0.4.0" @@ -2024,44 +1996,53 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.18" +version = "0.38.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bbfc1d1c7c40c01715f47d71444744a81669ca84e8b63e25a55e169b1f86433" +checksum = "f25469e9ae0f3d0047ca8b93fc56843f38e6774f0914a107ff8b41be8be8e0b7" dependencies = [ - "bitflags", - "errno 0.3.1", - "io-lifetimes", + "bitflags 2.4.0", + "errno 0.3.4", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "rustls" -version = "0.20.8" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", "ring", + "rustls-webpki", "sct", - "webpki", ] [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.0", + "base64", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +dependencies = [ + "ring", + "untrusted", ] [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "same-file" @@ -2074,11 +2055,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -2089,15 +2070,9 @@ checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" [[package]] name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "scratch" -version = "1.0.5" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" @@ -2111,11 +2086,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.8.2" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -2124,9 +2099,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" dependencies = [ "core-foundation-sys", "libc", @@ -2134,35 +2109,35 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.17" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" [[package]] name = "serde" -version = "1.0.160" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.38", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -2183,6 +2158,15 @@ dependencies = [ "warp", ] +[[package]] +name = "serde_spanned" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2197,9 +2181,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if 1.0.0", "cpufeatures", @@ -2208,9 +2192,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if 1.0.0", "cpufeatures", @@ -2219,9 +2203,9 @@ dependencies = [ [[package]] name = "sha3" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54c2bb1a323307527314a36bfb73f24febb08ce2b8a554bf4ffd6f51ad15198c" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" dependencies = [ "digest", "keccak", @@ -2229,18 +2213,18 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] [[package]] name = "shlex" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" [[package]] name = "signal-hook-registry" @@ -2253,18 +2237,18 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "socket2" @@ -2276,6 +2260,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "spin" version = "0.5.2" @@ -2373,9 +2367,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.15" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -2398,25 +2392,37 @@ dependencies = [ ] [[package]] -name = "tempfile" -version = "3.5.0" +name = "system-configuration" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ - "cfg-if 1.0.0", - "fastrand", - "redox_syscall 0.3.5", - "rustix", - "windows-sys 0.45.0", + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", ] [[package]] -name = "termcolor" -version = "1.2.0" +name = "system-configuration-sys" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" dependencies = [ - "winapi-util", + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +dependencies = [ + "cfg-if 1.0.0", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", ] [[package]] @@ -2460,21 +2466,11 @@ dependencies = [ [[package]] name = "time" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "time" -version = "0.3.20" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" dependencies = [ + "deranged", "itoa", "serde", "time-core", @@ -2483,15 +2479,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -2513,11 +2509,11 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.0" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", "mio", @@ -2525,9 +2521,9 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.4", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -2538,7 +2534,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.38", ] [[package]] @@ -2553,13 +2549,12 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.23.4" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls", "tokio", - "webpki", ] [[package]] @@ -2575,9 +2570,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.18.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" +checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", @@ -2587,9 +2582,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", @@ -2601,11 +2596,36 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.11" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" dependencies = [ "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.0.2", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", ] [[package]] @@ -2634,26 +2654,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" dependencies = [ "crossbeam-channel", - "time 0.3.20", + "time", "tracing-subscriber", ] [[package]] name = "tracing-attributes" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.38", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", "valuable", @@ -2696,13 +2716,13 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "tungstenite" -version = "0.18.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" +checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" dependencies = [ - "base64 0.13.1", "byteorder", "bytes", + "data-encoding", "http", "httparse", "log", @@ -2715,9 +2735,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "u_agent" @@ -2824,9 +2844,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ "version_check", ] @@ -2839,9 +2859,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -2860,9 +2880,9 @@ checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "untrusted" @@ -2872,9 +2892,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -2889,9 +2909,9 @@ checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "uuid" -version = "1.3.2" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" dependencies = [ "getrandom", "serde", @@ -2929,9 +2949,9 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", @@ -2939,19 +2959,18 @@ dependencies = [ [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] [[package]] name = "warp" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba431ef570df1287f7f8b07e376491ad54f84d26ac473489427231e1718e1f69" +checksum = "c1e92e22e03ff1230c03a1a8ee37d2f89cd489e2e541b7550d6afad96faed169" dependencies = [ "bytes", "futures-channel", @@ -2979,12 +2998,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2993,9 +3006,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -3003,24 +3016,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.38", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.34" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3030,9 +3043,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3040,43 +3053,33 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.38", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "winapi" version = "0.3.9" @@ -3095,9 +3098,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -3114,31 +3117,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets 0.48.0", -] - -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", + "windows-targets", ] [[package]] @@ -3147,146 +3126,99 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets", ] [[package]] name = "windows-targets" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-targets" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" -dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" +name = "winnow" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +dependencies = [ + "memchr", +] [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi", + "cfg-if 1.0.0", + "windows-sys", ] [[package]] name = "zstd" -version = "0.12.3+zstd.1.5.2" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.5+zstd.1.5.4" +version = "6.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" +checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" dependencies = [ "libc", "zstd-sys", diff --git a/Cargo.toml b/Cargo.toml index ed5ae87..372e08a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "lib/u_lib", "integration-tests", ] +resolver = "2" [workspace.dependencies] anyhow = "=1.0.63" diff --git a/Makefile.toml b/Makefile.toml index 0a103a1..67ef1dc 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -22,13 +22,6 @@ default_to_workspace = false [env] TARGET = "x86_64-unknown-linux-musl" CARGO = "cargo" -ROOTDIR = "${CARGO_MAKE_WORKING_DIRECTORY}" -STATIC_PREFIX = "${ROOTDIR}/static" -PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true" -PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = "${STATIC_PREFIX}/bin/pg_config" -OPENSSL_STATIC = "true" -OPENSSL_DIR = "${STATIC_PREFIX}" - [tasks.build_static_libs] script = "./scripts/build_musl_libs.sh" @@ -69,6 +62,12 @@ clear = true [tasks.run] disabled = true +[tasks.run_front] +script = ''' +cd ./bin/u_panel/src/gui/fe +ng serve +''' + [tasks.unit-tests] command = "${CARGO}" args = ["test", "--target", "${TARGET}", "--lib", "--", "${@}"] diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 4a0b8cf..1455635 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -9,7 +9,7 @@ pub struct Args { #[structopt(subcommand)] cmd: Cmd, #[structopt(short, long, default_value)] - brief: BriefMode, + brief: Brief, } #[derive(StructOpt, Debug)] @@ -141,7 +141,7 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { PayloadCRUD::Create { item } => { let payload = from_str::(&item) .map_err(|e| UError::DeserializeError(e.to_string(), item))?; - into_value(client.upload_payloads([&payload]).await?) + into_value(client.upload_payload(&payload).await?) } PayloadCRUD::Read { id } => match id { None => into_value(client.get_payloads().await?), diff --git a/bin/u_panel/src/gui/fe/src/app/app.module.ts b/bin/u_panel/src/gui/fe/src/app/app.module.ts index 43ce309..5925085 100644 --- a/bin/u_panel/src/gui/fe/src/app/app.module.ts +++ b/bin/u_panel/src/gui/fe/src/app/app.module.ts @@ -28,6 +28,7 @@ import { MatSnackBarModule } from '@angular/material/snack-bar'; import { MatListModule } from '@angular/material/list'; import { GlobalErrorComponent } from './components/global-error/global-error.component'; import { PayloadOverviewComponent } from './components/payload-overview/payload-overview.component'; +import { NewPayloadDialogComponent } from './components/dialogs/new-payload-dialog/new-payload-dialog.component'; @NgModule({ declarations: [ @@ -42,7 +43,8 @@ import { PayloadOverviewComponent } from './components/payload-overview/payload- PayloadComponent, PayloadInfoDialogComponent, GlobalErrorComponent, - PayloadOverviewComponent + PayloadOverviewComponent, + NewPayloadDialogComponent ], imports: [ BrowserModule, diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html index ffe8910..f581b5d 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html @@ -37,7 +37,9 @@ - +
+ +
diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts index 0f6eef0..04b29c2 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts @@ -3,7 +3,6 @@ import { MAT_DIALOG_DATA } from '@angular/material/dialog'; import { EventEmitter } from '@angular/core'; import { Job, JobModel } from '../../../models/job.model'; import { ApiTableService } from 'src/app/services'; -import { PayloadModel } from 'src/app/models'; @Component({ selector: 'job-info-dialog', diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.html new file mode 100644 index 0000000..2480108 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.html @@ -0,0 +1,20 @@ +

New payload

+ +
+ + Name + + + +
+
+ + Data + + +
+ + + + + \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.ts new file mode 100644 index 0000000..69551bf --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.ts @@ -0,0 +1,43 @@ +import { Component, EventEmitter, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { NewPayloadModel } from 'src/app/models/payload.model'; + +@Component({ + selector: 'new-payload-dialog', + templateUrl: 'new-payload-dialog.component.html', + styleUrls: ['../base-info-dialog.component.less'] +}) +export class NewPayloadDialogComponent { + decodedPayload = ""; + uploadMode = false; + onSave = new EventEmitter(); + + constructor(@Inject(MAT_DIALOG_DATA) public payload: NewPayloadModel) { } + + save() { + if (this.payload.data.length == 0) { + this.payload.data = Array.from(new TextEncoder().encode(this.decodedPayload)); + } + this.onSave.emit(this.payload); + } + + onFileSelected(event: any) { + const file: File = event.target.files[0]; + if (file) { + this.uploadMode = true + const reader = new FileReader(); + reader.onload = e => { + this.payload.name = file.name; + const result = e.target?.result; + if (result instanceof ArrayBuffer) { + const d = Array.from(new Uint8Array(result)); + this.payload.data = d; + console.log(this.payload.data) + } else { + alert!("no file") + } + } + reader.readAsArrayBuffer(file) + } + } +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html index fabd0b1..cc3ce41 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html @@ -1,27 +1,34 @@ -

Payload

+

Payload

+

Editing payload

-
- - ID - - - - Name - - - - MIME-type - - - - Size - - +
+
+ + ID + + + + Name + + +
+
+ + MIME-type + + + + Size + + +
- +
+ + \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts index 0cafab0..680f996 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts @@ -1,4 +1,4 @@ -import { Component, Inject } from '@angular/core'; +import { Component, EventEmitter, Inject } from '@angular/core'; import { MAT_DIALOG_DATA } from '@angular/material/dialog'; import { PayloadModel } from 'src/app/models/payload.model'; @@ -8,6 +8,13 @@ import { PayloadModel } from 'src/app/models/payload.model'; styleUrls: ['../base-info-dialog.component.less'] }) export class PayloadInfoDialogComponent { + isPreview = true; + + onSave = new EventEmitter(); constructor(@Inject(MAT_DIALOG_DATA) public payload: PayloadModel) { } + + updatePayload() { + this.onSave.emit(this.payload); + } } \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.html index 6ce43c2..50580d9 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.html +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.html @@ -39,13 +39,7 @@
-

- - Result - - -

+
diff --git a/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html index b73739b..4e4fc7c 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html +++ b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html @@ -1,9 +1,6 @@ -
- - Payload data - - - -
\ No newline at end of file + + \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts index 1a66c86..e961bb2 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts @@ -1,5 +1,4 @@ import { Component, Input, OnInit } from '@angular/core'; -import { PayloadModel } from 'src/app/models'; @Component({ selector: 'payload-overview', @@ -7,14 +6,14 @@ import { PayloadModel } from 'src/app/models'; styleUrls: ['./payload-overview.component.less'] }) export class PayloadOverviewComponent implements OnInit { - @Input() payload!: PayloadModel; + @Input() payload: number[] | null = null; @Input("preview") isPreview = true; isTooBigPayload = false; decodedPayload = ""; ngOnInit() { - if (this.payload.data !== null) { - this.decodedPayload = new TextDecoder().decode(new Uint8Array(this.payload.data)) + if (this.payload !== null) { + this.decodedPayload = new TextDecoder().decode(new Uint8Array(this.payload)) } else { this.isTooBigPayload = true } diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts index 5900a79..42a9d93 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts @@ -22,11 +22,11 @@ export abstract class TableComponent implements OnInit { this.loadTableData(); this.route.queryParams.subscribe(params => { const id = params['id'] - const new_agent = params['new'] + const new_item = params['new'] if (id) { this.showItemDialog(id); } - if (new_agent) { + if (new_item) { this.showItemDialog(null); } }) diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.html b/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.html index ce52d84..78cb243 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.html +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.html @@ -9,6 +9,9 @@ + + diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts index 9e7b6da..fd6461c 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts @@ -1,7 +1,8 @@ import { Component } from '@angular/core'; import { Area } from 'src/app/models'; -import { PayloadModel } from 'src/app/models/payload.model'; +import { NewPayloadModel, PayloadModel } from 'src/app/models/payload.model'; import { PayloadInfoDialogComponent } from '../../dialogs'; +import { NewPayloadDialogComponent } from '../../dialogs/new-payload-dialog/new-payload-dialog.component'; import { TableComponent } from '../base-table/base-table.component'; @Component({ @@ -14,17 +15,53 @@ export class PayloadComponent extends TableComponent { area = 'payloads' as Area displayedColumns = ["name", "mime_type", "size", 'actions']; - showItemDialog(id: string) { - this.dataSource.getPayload(id).subscribe(resp => { - const dialog = this.infoDialog.open(PayloadInfoDialogComponent, { - data: resp, + showItemDialog(id: string | null) { + if (id === null) { + const payload: NewPayloadModel = { + name: "", + data: [] + } + const dialog = this.infoDialog.open(NewPayloadDialogComponent, { + data: payload, width: '1000px', }); + dialog.componentInstance.onSave.subscribe(result => { + this.dataSource.createPayload(result) + .subscribe(_ => { + alert("Created") + this.loadTableData() + }) + + dialog.close() + }) + dialog.afterClosed().subscribe(_ => { this.router.navigate(['.'], { relativeTo: this.route }) }) - }) + + } else { + this.dataSource.getPayload(id as string).subscribe(resp => { + const dialog = this.infoDialog.open(PayloadInfoDialogComponent, { + data: resp, + width: '1000px', + }); + + const saveSub = dialog.componentInstance.onSave.subscribe(result => { + this.dataSource.updatePayload(result) + .subscribe(_ => { + alert("Updated") + this.loadTableData() + }) + dialog.close() + }) + + dialog.afterClosed().subscribe(_ => { + saveSub.unsubscribe() + this.router.navigate(['.'], { relativeTo: this.route }) + }) + }) + } } } diff --git a/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts b/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts index e947ac8..368c56c 100644 --- a/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts +++ b/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts @@ -4,4 +4,9 @@ export interface PayloadModel { name: string, size: number, data: number[] | null +} + +export interface NewPayloadModel { + name: string, + data: number[] } \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/models/result.model.ts b/bin/u_panel/src/gui/fe/src/app/models/result.model.ts index 56b0a93..4234b5e 100644 --- a/bin/u_panel/src/gui/fe/src/app/models/result.model.ts +++ b/bin/u_panel/src/gui/fe/src/app/models/result.model.ts @@ -6,7 +6,7 @@ export interface ResultModel { created: UTCDate, id: string, job_id: string, - result: number[], + result: number[] | null, state: "Queued" | "Running" | "Finished", retcode: number | null, updated: UTCDate, diff --git a/bin/u_panel/src/gui/fe/src/app/services/api.service.ts b/bin/u_panel/src/gui/fe/src/app/services/api.service.ts index be9174c..705b903 100644 --- a/bin/u_panel/src/gui/fe/src/app/services/api.service.ts +++ b/bin/u_panel/src/gui/fe/src/app/services/api.service.ts @@ -1,7 +1,7 @@ import { environment } from 'src/environments/environment'; import { HttpClient, HttpErrorResponse } from '@angular/common/http'; import { Observable, map, catchError, throwError } from 'rxjs'; -import { ApiModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, Job } from '../models'; +import { ApiModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, Job, NewPayloadModel } from '../models'; import { Injectable, Inject } from '@angular/core'; import { ErrorService } from './error.service'; @@ -119,6 +119,10 @@ export class ApiTableService { return this.create(item, 'map') } + createPayload(item: NewPayloadModel): Observable { + return this.create(item, 'payloads') + } + filterErrStatus(obs: Observable>): Observable { return obs.pipe( map(r => { @@ -131,7 +135,8 @@ export class ApiTableService { } errorHandler(err: HttpErrorResponse, caught: any) { - this.errorService.handle(caught.data ?? err.message); - return throwError(() => new Error(err.message)); + var error = err.error.data !== undefined ? JSON.stringify(err.error.data) : err.message; + this.errorService.handle(error); + return throwError(() => new Error()); } } \ No newline at end of file diff --git a/bin/u_panel/src/gui/mod.rs b/bin/u_panel/src/gui/mod.rs index 729a7ad..c3de15f 100644 --- a/bin/u_panel/src/gui/mod.rs +++ b/bin/u_panel/src/gui/mod.rs @@ -65,10 +65,8 @@ async fn send_cmd( let response = if result.is_ok() { HttpResponse::Ok().body(result_string) - } else if result.is_err() { - HttpResponse::BadRequest().body(result_string) } else { - unreachable!() + HttpResponse::BadRequest().body(result_string) }; Ok(response) diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index 81f3fd5..2b6248c 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -60,14 +60,14 @@ impl UDB<'_> { .map_err(with_err_ctx("Can't insert jobs")) } - pub fn insert_payloads(&mut self, payloads: &[Payload]) -> Result<()> { + pub fn insert_payload(&mut self, payload: &Payload) -> Result<()> { use schema::payloads; diesel::insert_into(payloads::table) - .values(payloads) + .values(payload) .execute(self.conn) .map(drop) - .map_err(with_err_ctx(format!("Can't insert payloads {payloads:?}"))) + .map_err(with_err_ctx(format!("Can't insert payload {payload:?}"))) } pub fn get_job(&mut self, id: Id) -> Result> { @@ -119,6 +119,17 @@ impl UDB<'_> { .map_err(with_err_ctx("Can't get payloads")) } + pub fn payload_exists(&mut self, payload_id: Id) -> Result { + use schema::payloads; + + payloads::table + .find(payload_id) + .first::(self.conn) + .optional() + .map(|r| r.is_some()) + .map_err(with_err_ctx("Can't check payload {payload_id}")) + } + pub fn get_job_by_alias(&mut self, alias: &str) -> Result> { use schema::{jobs, payloads}; diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index 4cb1b4a..36b3454 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use crate::db::{PgRepo, UDB}; use crate::error::Error; use serde::Deserialize; -use u_lib::{api::retypes, messaging::Reportable, models::*, types::Id}; +use u_lib::{api::api_types, messaging::Reportable, models::*, types::Id}; use warp::reject::not_found; use warp::Rejection; @@ -11,13 +11,13 @@ type EndpResult = Result; #[derive(Deserialize)] pub struct PayloadFlags { - brief: BriefMode, + brief: Brief, } pub struct Endpoints; impl Endpoints { - pub async fn get_agents(repo: Arc, id: Option) -> EndpResult { + pub async fn get_agents(repo: Arc, id: Option) -> EndpResult { repo.interact(move |mut db| { Ok(match id { Some(id) => { @@ -38,20 +38,20 @@ impl Endpoints { repo: Arc, id: Id, params: Option, - ) -> EndpResult { + ) -> EndpResult { let Some(mut job) = repo.interact(move |mut db| db.get_job(id)).await? else { - return Err(not_found()) + return Err(not_found()); }; Ok(match params.map(|p| p.brief) { - Some(BriefMode::Yes) => job, - Some(BriefMode::Auto) | None => { + Some(Brief::Yes) => job, + Some(Brief::Auto) | None => { if let Some(payload) = &mut job.payload { payload.maybe_join_payload().map_err(Error::from)?; } job } - Some(BriefMode::No) => { + Some(Brief::No) => { if let Some(payload) = &mut job.payload { payload.join_payload().map_err(Error::from)?; } @@ -60,7 +60,7 @@ impl Endpoints { }) } - pub async fn get_jobs(repo: Arc) -> EndpResult { + pub async fn get_jobs(repo: Arc) -> EndpResult { repo.interact(move |mut db| db.get_jobs()) .await .map_err(From::from) @@ -69,13 +69,13 @@ impl Endpoints { pub async fn get_assigned_jobs( repo: Arc, id: Option, - ) -> EndpResult { + ) -> EndpResult { repo.interact(move |mut db| db.get_assigned_jobs(id, false)) .await .map_err(From::from) } - pub async fn get_payloads(repo: Arc) -> EndpResult { + pub async fn get_payloads(repo: Arc) -> EndpResult { repo.interact(move |mut db| db.get_payloads()) .await .map_err(From::from) @@ -85,7 +85,7 @@ impl Endpoints { repo: Arc, name_or_id: String, params: Option, - ) -> EndpResult { + ) -> EndpResult { let mut payload = match repo .interact(move |mut db| match Id::parse_str(&name_or_id) { Ok(id) => db.get_payload(id), @@ -98,11 +98,11 @@ impl Endpoints { }; Ok(match params.map(|p| p.brief) { - Some(BriefMode::Yes) => { + Some(Brief::Yes) => { payload.data = None; payload } - None | Some(BriefMode::Auto) => { + None | Some(Brief::Auto) => { payload.maybe_join_payload().map_err(Error::from)?; payload } @@ -116,7 +116,7 @@ impl Endpoints { pub async fn get_personal_jobs( repo: Arc, id: Id, - ) -> EndpResult { + ) -> EndpResult { repo.transaction(move |mut db| { let agent = db.get_agent(id)?; match agent { @@ -153,19 +153,33 @@ impl Endpoints { .map_err(From::from) } - pub async fn upload_jobs(repo: Arc, msg: Vec) -> EndpResult { - let jobs = msg - .into_iter() - .map(|mut job| { - if let Some(payload) = &mut job.payload { - payload.maybe_split_payload()?; + pub async fn upload_jobs( + repo: Arc, + jobs: Vec, + ) -> EndpResult { + let mut checked_jobs = vec![]; + for mut job in jobs { + debug!("{job:?}"); + if let Some(payload) = &mut job.payload { + payload.maybe_split_payload().map_err(Error::from)?; + } else if let Some(pld_id) = job.meta.payload_id { + if repo + .interact(move |mut db| db.payload_exists(pld_id)) + .await? + { + checked_jobs.push(job) + } else { + Err(Error::ProcessingError(format!( + "Payload {pld_id} not found" + )))? } - Ok(job) - }) - .collect::, Error>>()?; + } + } - let (jobs, payloads_opt): (Vec<_>, Vec<_>) = - jobs.into_iter().map(|j| (j.meta, j.payload)).unzip(); + let (jobs, payloads_opt): (Vec<_>, Vec<_>) = checked_jobs + .into_iter() + .map(|j| (j.meta, j.payload)) + .unzip(); let payloads = payloads_opt .into_iter() @@ -173,7 +187,9 @@ impl Endpoints { .collect::>(); repo.transaction(move |mut db| { - db.insert_payloads(&payloads)?; + for payload in payloads { + db.insert_payload(&payload)?; + } db.insert_jobs(&jobs) }) .await @@ -182,15 +198,11 @@ impl Endpoints { pub async fn upload_payloads( repo: Arc, - raw_payloads: Vec, - ) -> EndpResult { - let payloads = raw_payloads - .into_iter() - .map(|raw| raw.into_payload()) - .collect::, _>>() - .map_err(Error::from)?; + raw_payload: RawPayload, + ) -> EndpResult { + let payloads = raw_payload.into_payload().map_err(Error::from)?; - repo.interact(move |mut db| db.insert_payloads(&payloads)) + repo.interact(move |mut db| db.insert_payload(&payloads)) .await .map_err(From::from) } @@ -215,7 +227,7 @@ impl Endpoints { repo: Arc, agent_id: Id, job_idents: Vec, - ) -> EndpResult { + ) -> EndpResult { repo.transaction(move |mut db| { let assigned_job_idents = job_idents .into_iter() @@ -245,7 +257,7 @@ impl Endpoints { repo: Arc, msg: Vec, agent_id: Id, - ) -> EndpResult { + ) -> EndpResult { repo.transaction(move |mut db| { for entry in msg { match entry { @@ -297,13 +309,16 @@ impl Endpoints { .map_err(From::from) } - pub async fn update_agent(repo: Arc, agent: Agent) -> EndpResult { + pub async fn update_agent( + repo: Arc, + agent: Agent, + ) -> EndpResult { repo.interact(move |mut db| db.upsert_agent(&agent)) .await .map_err(From::from) } - pub async fn update_job(repo: Arc, job: JobMeta) -> EndpResult { + pub async fn update_job(repo: Arc, job: JobMeta) -> EndpResult { repo.interact(move |mut db| db.update_job(&job.validate()?)) .await .map_err(From::from) @@ -312,7 +327,7 @@ impl Endpoints { pub async fn update_assigned_job( repo: Arc, assigned: AssignedJob, - ) -> EndpResult { + ) -> EndpResult { repo.interact(move |mut db| db.update_result(&assigned)) .await .map_err(From::from) @@ -321,7 +336,8 @@ impl Endpoints { pub async fn update_payload( repo: Arc, payload: Payload, - ) -> EndpResult { + ) -> EndpResult { + debug!("update payload: {payload:?}"); match payload.data { Some(data) => { let mut well_formed_payload = @@ -332,7 +348,10 @@ impl Endpoints { .await .map_err(From::from) } - None => return Ok(()), + None => repo + .interact(move |mut db| db.update_payload(&payload)) + .await + .map_err(From::from), } } } diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 42e9f36..bae4a2d 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -5,6 +5,7 @@ mod db; mod error; mod handlers; +use crate::handlers::{Endpoints, PayloadFlags}; use db::PgRepo; use error::{Error as ServerError, RejResponse}; use std::{convert::Infallible, sync::Arc}; @@ -24,8 +25,6 @@ use warp::{ const DEFAULT_RESPONSE: &str = "null"; -use crate::handlers::{Endpoints, PayloadFlags}; - fn into_message(msg: M) -> Json { json(&msg) } @@ -139,9 +138,9 @@ pub fn init_endpoints( .and_then(Endpoints::get_payload) .map(into_message); - let upload_payloads = path("upload_payloads") + let upload_payload = path("upload_payload") .and(with_db.clone()) - .and(body::json::>()) + .and(body::json::()) .and_then(Endpoints::upload_payloads) .map(ok); @@ -162,7 +161,7 @@ pub fn init_endpoints( .or(get_payloads) .or(get_payload) .or(upload_jobs) - .or(upload_payloads) + .or(upload_payload) .or(del) .or(set_jobs) .or(get_assigned_jobs) diff --git a/images/tests_runner.Dockerfile b/images/tests_runner.Dockerfile index 35355c5..7819ddf 100644 --- a/images/tests_runner.Dockerfile +++ b/images/tests_runner.Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.67 +FROM rust:1.72 RUN rustup target add x86_64-unknown-linux-musl RUN mkdir -p /tests && chmod 777 /tests diff --git a/integration-tests/tests/fixtures/agent.rs b/integration-tests/tests/fixtures/agent.rs index b441d7a..db67502 100644 --- a/integration-tests/tests/fixtures/agent.rs +++ b/integration-tests/tests/fixtures/agent.rs @@ -21,7 +21,7 @@ pub fn registered_agent(client: &HttpClient) -> RegisteredAgent { .pop() .unwrap(); let job_id = resp.job_id; - let job = client.get_job(job_id, BriefMode::No).await.unwrap(); + let job = client.get_job(job_id, Brief::No).await.unwrap(); assert_eq!(job.meta.alias, Some("agent_hello".to_string())); diff --git a/integration-tests/tests/integration_tests/endpoints.rs b/integration-tests/tests/integration_tests/endpoints.rs index 40cddd1..45c9f39 100644 --- a/integration-tests/tests/integration_tests/endpoints.rs +++ b/integration-tests/tests/integration_tests/endpoints.rs @@ -15,7 +15,7 @@ use crate::fixtures::connections::*; use std::iter::repeat; -use u_lib::models::{BriefMode, RawJob, RawPayload, MAX_READABLE_PAYLOAD_SIZE}; +use u_lib::models::{Brief, RawJob, RawPayload, MAX_READABLE_PAYLOAD_SIZE}; #[rstest] #[tokio::test] @@ -62,16 +62,10 @@ async fn payloads_upload_update_get_del(client_panel: &HttpClient) { data: data.clone(), }; - client_panel.upload_payloads([&payload]).await.unwrap(); + client_panel.upload_payload(&payload).await.unwrap(); - let mut fetched_payload = client_panel - .get_payload(&name, BriefMode::No) - .await - .unwrap(); - let fetched_payload_auto = client_panel - .get_payload(&name, BriefMode::Auto) - .await - .unwrap(); + let mut fetched_payload = client_panel.get_payload(&name, Brief::No).await.unwrap(); + let fetched_payload_auto = client_panel.get_payload(&name, Brief::Auto).await.unwrap(); assert_eq!(fetched_payload, fetched_payload_auto); assert_eq!(fetched_payload.data.unwrap(), data); @@ -82,30 +76,21 @@ async fn payloads_upload_update_get_del(client_panel: &HttpClient) { fetched_payload.data = Some(big_data.clone()); client_panel.update_payload(&fetched_payload).await.unwrap(); - let fetched_big_payload = client_panel - .get_payload(&name, BriefMode::Yes) - .await - .unwrap(); - let fetched_big_payload_auto = client_panel - .get_payload(&name, BriefMode::Auto) - .await - .unwrap(); + let fetched_big_payload = client_panel.get_payload(&name, Brief::Yes).await.unwrap(); + let fetched_big_payload_auto = client_panel.get_payload(&name, Brief::Auto).await.unwrap(); assert_eq!(fetched_big_payload, fetched_big_payload_auto); assert_eq!(fetched_big_payload.size, new_size); assert!(fetched_big_payload.data.is_none()); - let fetched_big_payload_full = client_panel - .get_payload(&name, BriefMode::No) - .await - .unwrap(); + let fetched_big_payload_full = client_panel.get_payload(&name, Brief::No).await.unwrap(); assert_eq!(fetched_big_payload_full.data.unwrap(), big_data); client_panel.del(fetched_big_payload_full.id).await.unwrap(); let not_found_err = client_panel - .get_payload(&name, BriefMode::Yes) + .get_payload(&name, Brief::Yes) .await .unwrap_err(); assert!(not_found_err.to_string().contains("404 Not Found")) diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 35a7048..c89b57d 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -19,7 +19,7 @@ use crate::{ const AGENT_IDENTITY: &[u8] = include_bytes!("../../../certs/alice.p12"); const ROOT_CA_CERT: &[u8] = include_bytes!("../../../certs/ca.crt"); -pub mod retypes { +pub mod api_types { use super::*; pub type GetPersonalJobs = Vec; @@ -145,7 +145,7 @@ impl HttpClient { } /// get jobs for agent - pub async fn get_personal_jobs(&self, agent_id: Id) -> Result { + pub async fn get_personal_jobs(&self, agent_id: Id) -> Result { self.req(format!("get_personal_jobs/{}", agent_id)).await } @@ -153,7 +153,7 @@ impl HttpClient { pub async fn report( &self, payload: impl IntoIterator, - ) -> Result { + ) -> Result { self.req_with_payload("report", &payload.into_iter().collect::>()) .await } @@ -164,20 +164,20 @@ impl HttpClient { } /// get exact job - pub async fn get_job(&self, job: Id, brief: BriefMode) -> Result { + pub async fn get_job(&self, job: Id, brief: Brief) -> Result { self.req(format!("get_job/{job}?brief={brief}")).await } - pub async fn get_full_job(&self, job: Id) -> Result { - self.get_job(job, BriefMode::No).await + pub async fn get_full_job(&self, job: Id) -> Result { + self.get_job(job, Brief::No).await } - pub async fn get_brief_job(&self, job: Id) -> Result { - self.get_job(job, BriefMode::Yes).await + pub async fn get_brief_job(&self, job: Id) -> Result { + self.get_job(job, Brief::Yes).await } /// get all available jobs - pub async fn get_jobs(&self) -> Result { + pub async fn get_jobs(&self) -> Result { self.req("get_jobs").await } } @@ -186,27 +186,27 @@ impl HttpClient { #[cfg(feature = "panel")] impl HttpClient { /// agent listing - pub async fn get_agents(&self, agent: Option) -> Result { + pub async fn get_agents(&self, agent: Option) -> Result { self.req(format!("get_agents/{}", opt_to_string(agent))) .await } /// update agent - pub async fn update_agent(&self, agent: &Agent) -> Result { + pub async fn update_agent(&self, agent: &Agent) -> Result { self.req_with_payload("update_agent", agent).await } /// update job - pub async fn update_job(&self, job: &JobMeta) -> Result { + pub async fn update_job(&self, job: &JobMeta) -> Result { self.req_with_payload("update_job", job).await } /// update result - pub async fn update_result(&self, result: &AssignedJob) -> Result { + pub async fn update_result(&self, result: &AssignedJob) -> Result { self.req_with_payload("update_result", result).await } - pub async fn update_payload(&self, payload: &Payload) -> Result { + pub async fn update_payload(&self, payload: &Payload) -> Result { self.req_with_payload("update_payload", payload).await } @@ -214,21 +214,17 @@ impl HttpClient { pub async fn upload_jobs( &self, jobs: impl IntoIterator, - ) -> Result { + ) -> Result { self.req_with_payload("upload_jobs", &jobs.into_iter().collect::>()) .await } - pub async fn upload_payloads( - &self, - payload: impl IntoIterator, - ) -> Result { - self.req_with_payload("upload_payloads", &payload.into_iter().collect::>()) - .await + pub async fn upload_payload(&self, payload: &RawPayload) -> Result { + self.req_with_payload("upload_payload", payload).await } /// delete something - pub async fn del(&self, item: Id) -> Result { + pub async fn del(&self, item: Id) -> Result { self.req(format!("del/{item}")).await } @@ -237,7 +233,7 @@ impl HttpClient { &self, agent: Id, job_idents: impl IntoIterator>, - ) -> Result { + ) -> Result { self.req_with_payload( format!("set_jobs/{agent}"), &job_idents @@ -249,26 +245,26 @@ impl HttpClient { } /// get jobs for any agent - pub async fn get_assigned_jobs(&self, agent: Option) -> Result { + pub async fn get_assigned_jobs(&self, agent: Option) -> Result { self.req(format!("get_assigned_jobs/{}", opt_to_string(agent))) .await } - pub async fn get_payloads(&self) -> Result { + pub async fn get_payloads(&self) -> Result { self.req("get_payloads").await } pub async fn get_payload( &self, payload: impl AsRef, - brief: BriefMode, - ) -> Result { + brief: Brief, + ) -> Result { let payload = payload.as_ref(); self.req(format!("get_payload/{payload}?brief={brief}")) .await } - pub async fn ping(&self) -> Result { + pub async fn ping(&self) -> Result { self.req("ping").await } } diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 3611ad7..10a8026 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -223,7 +223,7 @@ impl TryFrom> for Job { } }; - raw.meta.payload_id = payload.as_ref().map(|p| p.id); + raw.meta.payload_id = payload.as_ref().map(|p| p.id).or(raw.meta.payload_id); Ok(Job { meta: raw.meta.validate()?, diff --git a/lib/u_lib/src/models/mod.rs b/lib/u_lib/src/models/mod.rs index 44d56a8..61c9a56 100644 --- a/lib/u_lib/src/models/mod.rs +++ b/lib/u_lib/src/models/mod.rs @@ -9,7 +9,7 @@ use serde::Deserialize; use strum::{Display as StrumDisplay, EnumString}; #[derive(Default, Debug, StrumDisplay, EnumString, Deserialize)] -pub enum BriefMode { +pub enum Brief { Yes, #[default] Auto, diff --git a/lib/u_lib/src/models/payload.rs b/lib/u_lib/src/models/payload.rs index 9b982bf..b20155d 100644 --- a/lib/u_lib/src/models/payload.rs +++ b/lib/u_lib/src/models/payload.rs @@ -129,12 +129,12 @@ impl Payload { const BUFFER_LEN: usize = 4096; let mut buffer: [u8; BUFFER_LEN] = [0; BUFFER_LEN]; - let mut payload_src = if let Some(data) = &self.data { - Box::new(data.as_slice()) as Box + let mut payload_src: Box = if let Some(data) = &self.data { + Box::new(data.as_slice()) } else { let payload_path = ufs::read_meta(&self.name).context("prep")?.path; let file = File::open(&payload_path).map_err(|e| ufs::Error::new(e, &payload_path))?; - Box::new(file) as Box + Box::new(file) }; let fd = memfd_create( @@ -167,6 +167,7 @@ impl Payload { } } +#[cfg(unix)] fn get_mime_type(path: impl AsRef) -> Result { let path = path.as_ref(); diff --git a/migrations/2020-10-24-111622_create_all/up.sql b/migrations/2020-10-24-111622_create_all/up.sql index 819c7b6..f4058a8 100644 --- a/migrations/2020-10-24-111622_create_all/up.sql +++ b/migrations/2020-10-24-111622_create_all/up.sql @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS jobs ( payload_id UUID, schedule TEXT, - FOREIGN KEY(payload_id) REFERENCES payloads(id) ON DELETE SET NULL, + FOREIGN KEY(payload_id) REFERENCES payloads(id), PRIMARY KEY(id) ); -- 2.36.2 From e2ba50d947b72fd03831f3277b3f36c7911c8dc7 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sat, 7 Oct 2023 22:05:37 +0300 Subject: [PATCH 10/10] make assigning jobs work again --- Cargo.lock | 16 ++-- bin/u_panel/src/argparse.rs | 40 +++++---- .../assign-job-dialog.component.ts | 14 ++- .../src/gui/fe/src/app/models/result.model.ts | 5 ++ .../gui/fe/src/app/services/api.service.ts | 4 +- bin/u_server/src/db.rs | 85 +++++++++++++------ bin/u_server/src/error.rs | 2 +- bin/u_server/src/handlers.rs | 51 ++++------- bin/u_server/src/u_server.rs | 11 ++- deploy/podman-compose.yml | 4 +- integration-tests/docker-compose.yml | 4 +- .../tests/integration_tests/behaviour.rs | 26 ++++-- .../tests/integration_tests/endpoints.rs | 2 +- lib/u_lib/src/api.rs | 20 ++--- lib/u_lib/src/config.rs | 5 +- lib/u_lib/src/models/jobs/assigned.rs | 1 + lib/u_lib/src/models/payload.rs | 28 +++++- 17 files changed, 189 insertions(+), 129 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40d3168..9719584 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -391,9 +391,9 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" @@ -1287,9 +1287,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.148" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libflate" @@ -1721,9 +1721,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "5b1106fec09662ec6dd98ccac0f81cef56984d0b49f75c92d8cbad76e20c005c" dependencies = [ "unicode-ident", ] @@ -3188,9 +3188,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907" dependencies = [ "memchr", ] diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 1455635..a4e67a6 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -16,7 +16,7 @@ pub struct Args { enum Cmd { Agents(RUD), Jobs(CRUD), - Map(MapCRUD), + Map(AssignedCRUD), Payloads(PayloadCRUD), Ping, Serve, @@ -32,12 +32,9 @@ enum CRUD { } #[derive(StructOpt, Debug)] -enum MapCRUD { +enum AssignedCRUD { Create { - #[structopt(parse(try_from_str = parse_uuid))] - agent_id: Id, - - job_idents: Vec, + item: String, }, #[structopt(flatten)] RUD(RUD), @@ -55,7 +52,7 @@ enum PayloadCRUD { item: String, }, Delete { - #[structopt(parse(try_from_str = parse_uuid))] + #[structopt(parse(try_from_str = parse::uuid))] id: Id, }, } @@ -63,20 +60,24 @@ enum PayloadCRUD { #[derive(StructOpt, Debug)] enum RUD { Read { - #[structopt(parse(try_from_str = parse_uuid))] + #[structopt(parse(try_from_str = parse::uuid))] id: Option, }, Update { item: String, }, Delete { - #[structopt(parse(try_from_str = parse_uuid))] + #[structopt(parse(try_from_str = parse::uuid))] id: Id, }, } -fn parse_uuid(src: &str) -> Result { - Id::parse_str(src).map_err(|e| e.to_string()) +mod parse { + use super::*; + + pub fn uuid(src: &str) -> Result { + Id::parse_str(src).map_err(|e| e.to_string()) + } } pub fn into_value(data: M) -> Value { @@ -125,17 +126,20 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { CRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), }, Cmd::Map(action) => match action { - MapCRUD::Create { - agent_id, - job_idents, - } => into_value(client.set_jobs(agent_id, &job_idents).await?), - MapCRUD::RUD(RUD::Read { id }) => into_value(client.get_assigned_jobs(id).await?), - MapCRUD::RUD(RUD::Update { item }) => { + AssignedCRUD::Create { item } => { + let payload = serde_json::from_str::>(&item) + .map_err(|e| UError::DeserializeError(e.to_string(), item))?; + into_value(client.assign_jobs(&payload).await?) + } + AssignedCRUD::RUD(RUD::Read { id }) => { + into_value(client.get_assigned_jobs(id).await?) + } + AssignedCRUD::RUD(RUD::Update { item }) => { let assigned = from_str::(&item) .map_err(|e| UError::DeserializeError(e.to_string(), item))?; into_value(client.update_result(&assigned).await?) } - MapCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), + AssignedCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), }, Cmd::Payloads(action) => match action { PayloadCRUD::Create { item } => { diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts index 1963201..4cfdfc8 100644 --- a/bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts @@ -1,5 +1,6 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { AssignedJobByIdModel } from 'src/app/models'; import { ApiTableService } from '../../../services'; @Component({ @@ -21,8 +22,15 @@ export class AssignJobDialogComponent { } assignSelectedJobs() { - const job_ids = this.selected_rows.map(row => row.split(' ', 1)[0]).join(' '); - const request = `${this.agent_id} ${job_ids}` - this.dataSource.createResult(request) + const assigned_jobs: AssignedJobByIdModel[] = this.selected_rows.map(row => { + const job_id = row.split(' ', 1)[0]; + return { + job_id: job_id, + agent_id: this.agent_id + } + }); + this.dataSource.createResult(assigned_jobs).subscribe(_ => { + alert("Created") + }); } } \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/models/result.model.ts b/bin/u_panel/src/gui/fe/src/app/models/result.model.ts index 4234b5e..eb474a6 100644 --- a/bin/u_panel/src/gui/fe/src/app/models/result.model.ts +++ b/bin/u_panel/src/gui/fe/src/app/models/result.model.ts @@ -10,4 +10,9 @@ export interface ResultModel { state: "Queued" | "Running" | "Finished", retcode: number | null, updated: UTCDate, +} + +export interface AssignedJobByIdModel { + job_id: string, + agent_id: string } \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/services/api.service.ts b/bin/u_panel/src/gui/fe/src/app/services/api.service.ts index 705b903..b410f05 100644 --- a/bin/u_panel/src/gui/fe/src/app/services/api.service.ts +++ b/bin/u_panel/src/gui/fe/src/app/services/api.service.ts @@ -1,7 +1,7 @@ import { environment } from 'src/environments/environment'; import { HttpClient, HttpErrorResponse } from '@angular/common/http'; import { Observable, map, catchError, throwError } from 'rxjs'; -import { ApiModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, Job, NewPayloadModel } from '../models'; +import { ApiModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, Job, NewPayloadModel, AssignedJobByIdModel } from '../models'; import { Injectable, Inject } from '@angular/core'; import { ErrorService } from './error.service'; @@ -115,7 +115,7 @@ export class ApiTableService { return this.filterErrStatus(this.req(`${area} create '${serialized}'`)) } - createResult(item: string): Observable { + createResult(item: AssignedJobByIdModel[]): Observable { return this.create(item, 'map') } diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index 2b6248c..21b6c61 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -1,12 +1,14 @@ use crate::error::Error; use diesel::{pg::PgConnection, prelude::*, result::Error as DslError, Connection}; +use std::collections::{HashMap, HashSet}; use std::mem::drop; use u_lib::{ db::PgAsyncPool, - models::{schema, Agent, AssignedJob, Job, JobMeta, JobState, Payload}, + models::{schema, Agent, AssignedJob, AssignedJobById, Job, JobMeta, JobState, Payload}, platform::Platform, types::Id, }; +use uuid::Uuid; type Result = std::result::Result; @@ -215,39 +217,72 @@ impl UDB<'_> { Ok(result) } - pub fn set_jobs_for_agent(&mut self, agent_id: Id, job_ids: &[Id]) -> Result<()> { + // todo: move to handlers + pub fn assign_jobs(&mut self, assigned_jobs: &[AssignedJobById]) -> Result<()> { use schema::{jobs, results}; - let agent_platform = match self.get_agent(agent_id)? { - Some(agent) => Platform::new(&agent.platform), - None => { - return Err(Error::ProcessingError(format!( - "Agent {agent_id} not found" - ))) - } - }; + struct JobBriefMeta { + alias: Option, + target_platform: String, + } + + let assigned_job_ids = HashSet::::from_iter(assigned_jobs.iter().map(|a| a.job_id)); + + let jobs_meta = HashMap::::from_iter( + jobs::table + .select((jobs::id, jobs::alias, jobs::target_platforms)) + .filter(jobs::id.eq_any(&assigned_job_ids)) + .load::<(Id, Option, String)>(self.conn) + .map_err(with_err_ctx(format!( + "Can't find jobs {:?}", + assigned_job_ids + )))? + .into_iter() + .map(|(id, alias, target_platform)| { + ( + id, + JobBriefMeta { + alias, + target_platform, + }, + ) + }), + ); + + let existing_job_ids = HashSet::from_iter(jobs_meta.keys().copied()); - let jobs_meta = jobs::table - .select((jobs::id, jobs::alias, jobs::target_platforms)) - .filter(jobs::id.eq_any(job_ids)) - .load::<(Id, Option, String)>(self.conn) - .map_err(with_err_ctx(format!("Can't find jobs {job_ids:?}")))?; + if assigned_job_ids != existing_job_ids { + return Err(Error::ProcessingError(format!( + "Jobs not found: {:?}", + assigned_job_ids.difference(&existing_job_ids), + ))); + } - for meta in &jobs_meta { - if !agent_platform.matches(&meta.2) { + for ajob in assigned_jobs { + let meta = &jobs_meta[&ajob.job_id]; + let agent_platform = match self.get_agent(ajob.agent_id)? { + Some(agent) => Platform::new(&agent.platform), + None => { + return Err(Error::ProcessingError(format!( + "Agent {} not found", + ajob.agent_id + ))) + } + }; + if !agent_platform.matches(&meta.target_platform) { return Err(Error::InsuitablePlatform( agent_platform.into_string(), - meta.2.clone(), + meta.target_platform.clone(), )); } } - let job_requests = jobs_meta + let job_requests = assigned_jobs .into_iter() - .map(|(job_id, alias, _)| AssignedJob { - job_id, - agent_id, - alias, + .map(|a| AssignedJob { + job_id: a.job_id, + agent_id: a.agent_id, + alias: jobs_meta[&a.job_id].alias.clone(), ..Default::default() }) .collect::>(); @@ -256,9 +291,7 @@ impl UDB<'_> { .values(&job_requests) .execute(self.conn) .map(drop) - .map_err(with_err_ctx(format!( - "Can't setup jobs {job_ids:?} for agent {agent_id:?}" - ))) + .map_err(with_err_ctx("Can't assign jobs")) } pub fn del_jobs(&mut self, ids: &[Id]) -> Result<()> { diff --git a/bin/u_server/src/error.rs b/bin/u_server/src/error.rs index b2c4382..5fab3f7 100644 --- a/bin/u_server/src/error.rs +++ b/bin/u_server/src/error.rs @@ -13,7 +13,7 @@ pub enum Error { #[error("Configs error: {0}")] ConfigError(#[from] u_lib::config::Error), - #[error("Error processing {0}")] + #[error("Processing error: {0}")] ProcessingError(String), #[error(transparent)] diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index 36b3454..e1a8ec7 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -115,10 +115,10 @@ impl Endpoints { pub async fn get_personal_jobs( repo: Arc, - id: Id, + agent_id: Id, ) -> EndpResult { repo.transaction(move |mut db| { - let agent = db.get_agent(id)?; + let agent = db.get_agent(agent_id)?; match agent { Some(mut agent) => { agent.touch(); @@ -126,7 +126,7 @@ impl Endpoints { } None => { let mut new_agent = Agent::empty(); - new_agent.id = id; + new_agent.id = agent_id; db.upsert_agent(&new_agent)?; @@ -134,11 +134,16 @@ impl Endpoints { .get_job_by_alias("agent_hello")? .expect("agent_hello job not found"); - db.set_jobs_for_agent(id, &[job.meta.id])?; + let assigned_job = AssignedJobById { + agent_id, + job_id: job.meta.id, + ..Default::default() + }; + db.assign_jobs(&[assigned_job])?; } } - let assigned_jobs = db.get_assigned_jobs(Some(id), true)?; + let assigned_jobs = db.get_assigned_jobs(Some(agent_id), true)?; for job in &assigned_jobs { db.update_job_status(job.id, JobState::Running)?; @@ -159,21 +164,19 @@ impl Endpoints { ) -> EndpResult { let mut checked_jobs = vec![]; for mut job in jobs { - debug!("{job:?}"); if let Some(payload) = &mut job.payload { payload.maybe_split_payload().map_err(Error::from)?; } else if let Some(pld_id) = job.meta.payload_id { - if repo + if !repo .interact(move |mut db| db.payload_exists(pld_id)) .await? { - checked_jobs.push(job) - } else { Err(Error::ProcessingError(format!( "Payload {pld_id} not found" )))? } } + checked_jobs.push(job) } let (jobs, payloads_opt): (Vec<_>, Vec<_>) = checked_jobs @@ -196,7 +199,7 @@ impl Endpoints { .map_err(From::from) } - pub async fn upload_payloads( + pub async fn upload_payload( repo: Arc, raw_payload: RawPayload, ) -> EndpResult { @@ -223,31 +226,13 @@ impl Endpoints { .map_err(From::from) } - pub async fn set_jobs( + pub async fn assign_jobs( repo: Arc, - agent_id: Id, - job_idents: Vec, - ) -> EndpResult { + assigned_jobs: Vec, + ) -> EndpResult<()> { repo.transaction(move |mut db| { - let assigned_job_idents = job_idents - .into_iter() - .map(|ident| { - Id::parse_str(&ident).or_else(|_| { - let job_from_db = db.get_job_by_alias(&ident); - match job_from_db { - Ok(job) => match job { - Some(j) => Ok(j.meta.id), - None => { - Err(Error::ProcessingError(format!("unknown ident {ident}"))) - } - }, - Err(e) => Err(e), - } - }) - }) - .collect::, Error>>()?; - db.set_jobs_for_agent(agent_id, &assigned_job_idents)?; - Ok(assigned_job_idents) + db.assign_jobs(&assigned_jobs)?; + Ok(()) }) .await .map_err(From::from) diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index bae4a2d..0018463 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -92,11 +92,10 @@ pub fn init_endpoints( .and_then(Endpoints::del) .map(ok); - let set_jobs = path("set_jobs") + let assign_jobs = path("assign_jobs") .and(with_db.clone()) - .and(warp::path::param::()) - .and(body::json::>()) - .and_then(Endpoints::set_jobs) + .and(body::json::>()) + .and_then(Endpoints::assign_jobs) .map(into_message); let report = path("report") @@ -141,7 +140,7 @@ pub fn init_endpoints( let upload_payload = path("upload_payload") .and(with_db.clone()) .and(body::json::()) - .and_then(Endpoints::upload_payloads) + .and_then(Endpoints::upload_payload) .map(ok); let update_payload = path("update_payload") @@ -163,7 +162,7 @@ pub fn init_endpoints( .or(upload_jobs) .or(upload_payload) .or(del) - .or(set_jobs) + .or(assign_jobs) .or(get_assigned_jobs) .or(update_agent) .or(update_job) diff --git a/deploy/podman-compose.yml b/deploy/podman-compose.yml index 5e27513..d8edd93 100644 --- a/deploy/podman-compose.yml +++ b/deploy/podman-compose.yml @@ -18,14 +18,14 @@ services: u_db: condition: service_healthy ports: - - 63714:63714 + - 9990:9990 env_file: - ./.env - ./.env.private environment: RUST_LOG: warp=info,u_server_lib=debug healthcheck: - test: ss -tlpn | grep 63714 + test: ss -tlpn | grep 9990 interval: 5s timeout: 2s retries: 2 diff --git a/integration-tests/docker-compose.yml b/integration-tests/docker-compose.yml index bbd3424..093312d 100644 --- a/integration-tests/docker-compose.yml +++ b/integration-tests/docker-compose.yml @@ -23,14 +23,14 @@ services: u_db: condition: service_healthy ports: - - 63714:63714 + - 9990:9990 env_file: - ../.env - ../.env.private environment: RUST_LOG: warp=info,u_server_lib=debug,u_lib=debug healthcheck: - test: ss -tlpn | grep 63714 + test: ss -tlpn | grep 9990 interval: 5s timeout: 2s retries: 2 diff --git a/integration-tests/tests/integration_tests/behaviour.rs b/integration-tests/tests/integration_tests/behaviour.rs index 51726a1..cea4f85 100644 --- a/integration-tests/tests/integration_tests/behaviour.rs +++ b/integration-tests/tests/integration_tests/behaviour.rs @@ -5,7 +5,6 @@ use rstest::rstest; use serde_json::to_string; use u_lib::config::AGENT_ITERATION_INTERVAL; use u_lib::models::*; -use uuid::Uuid; #[rstest] #[tokio::test] @@ -28,16 +27,21 @@ async fn setup_tasks() { .with_target_platforms("*linux*") .try_into_job() .unwrap(); + let job_id = job.meta.id; Panel::check_status(["jobs", "create", &to_string(&RawJob::from(job)).unwrap()]); - let cmd = format!("map create {agent_id} {job_alias}"); - let assigned_ids: Vec = Panel::check_output(cmd); + let assigned = AssignedJobById { + agent_id, + job_id, + ..Default::default() + }; + + Panel::check_status(["map", "create", &to_string(&[assigned]).unwrap()]); retry_with_interval(5, AGENT_ITERATION_INTERVAL, || { let result = - Panel::check_output::>(format!("map read {}", assigned_ids[0])) - .remove(0); + Panel::check_output::>(format!("map read {}", job_id)).remove(0); if result.state == JobState::Finished { eprintln!("{}", result.to_str_result()); assert!(result.to_str_result().contains("root:x:0:0")); @@ -61,16 +65,20 @@ async fn large_payload() { .with_target_platforms(&agent.platform) .try_into_job() .unwrap(); + let job_id = job.meta.id; Panel::check_status(["jobs", "create", &to_string(&RawJob::from(job)).unwrap()]); - let cmd = format!("map create {agent_id} {job_alias}"); - let assigned_ids: Vec = Panel::check_output(cmd); + let assigned = AssignedJobById { + agent_id, + job_id, + ..Default::default() + }; + Panel::check_status(["map", "create", &to_string(&[assigned]).unwrap()]); retry_with_interval(5, AGENT_ITERATION_INTERVAL, || { let result = - Panel::check_output::>(format!("map read {}", assigned_ids[0])) - .remove(0); + Panel::check_output::>(format!("map read {}", job_id)).remove(0); if result.state == JobState::Finished { assert_eq!(result.to_str_result(), "type echo\n"); Ok(()) diff --git a/integration-tests/tests/integration_tests/endpoints.rs b/integration-tests/tests/integration_tests/endpoints.rs index 45c9f39..e575d77 100644 --- a/integration-tests/tests/integration_tests/endpoints.rs +++ b/integration-tests/tests/integration_tests/endpoints.rs @@ -9,7 +9,7 @@ // update_result(&self, result: AssignedJob) // upload_jobs(&self, payload: impl OneOrVec) // del(&self, item: Id) -// set_jobs(&self, agent: Id, job_idents: impl OneOrVec) +// assign_jobs(&self, agent: Id, job_idents: impl OneOrVec) // get_agent_jobs(&self, agent: Option) // ping(&self) diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index c89b57d..e0e9771 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -35,7 +35,7 @@ pub mod api_types { pub type UploadJobs = (); pub type UploadPayloads = (); pub type Del = (); - pub type SetJobs = Vec; + pub type SetJobs = (); pub type GetAgentJobs = Vec; pub type Ping = (); pub type GetPayloads = Vec; @@ -229,24 +229,20 @@ impl HttpClient { } /// set jobs for any agent - pub async fn set_jobs( + pub async fn assign_jobs( &self, - agent: Id, - job_idents: impl IntoIterator>, + assigned: impl IntoIterator, ) -> Result { self.req_with_payload( - format!("set_jobs/{agent}"), - &job_idents - .into_iter() - .map(|i| i.into()) - .collect::>(), + format!("assign_jobs"), + &assigned.into_iter().collect::>(), ) .await } - /// get jobs for any agent - pub async fn get_assigned_jobs(&self, agent: Option) -> Result { - self.req(format!("get_assigned_jobs/{}", opt_to_string(agent))) + /// get jobs for any agent by job_id, agent_id or result_id + pub async fn get_assigned_jobs(&self, id: Option) -> Result { + self.req(format!("get_assigned_jobs/{}", opt_to_string(id))) .await } diff --git a/lib/u_lib/src/config.rs b/lib/u_lib/src/config.rs index 5eb1c7e..5890996 100644 --- a/lib/u_lib/src/config.rs +++ b/lib/u_lib/src/config.rs @@ -1,3 +1,4 @@ +use crate::types::Id; use envy::{from_env, prefixed, Result as EnvResult}; use lazy_static::lazy_static; use serde::Deserialize; @@ -5,9 +6,7 @@ use std::time::Duration; pub use envy::Error; -use crate::types::Id; - -pub const MASTER_PORT: u16 = 63714; +pub const MASTER_PORT: u16 = 9990; pub const AGENT_ITERATION_INTERVAL: Duration = Duration::from_secs(5); diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index fef995f..a7caf20 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -56,6 +56,7 @@ impl Debug for AssignedJob { #[derive(Serialize, Deserialize, Clone, Copy, Debug)] pub struct AssignedJobById { pub agent_id: Id, + #[serde(default)] pub id: Id, pub job_id: Id, } diff --git a/lib/u_lib/src/models/payload.rs b/lib/u_lib/src/models/payload.rs index b20155d..84db3d7 100644 --- a/lib/u_lib/src/models/payload.rs +++ b/lib/u_lib/src/models/payload.rs @@ -1,7 +1,11 @@ -use crate::{conv::bytes_to_string, types::Id, ufs, UError}; +use crate::{ + conv::{bytes_to_string, bytes_to_string_truncated}, + types::Id, + ufs, UError, +}; use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; -use std::{fs::File, path::Path, process::Command}; +use std::{fmt::Debug, fs::File, path::Path, process::Command}; pub const MAX_READABLE_PAYLOAD_SIZE: i64 = 1024 * 32; @@ -28,7 +32,7 @@ impl RawPayload { diesel(table_name = payloads), diesel(treat_none_as_null = true) )] -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct Payload { pub id: Id, pub mime_type: String, @@ -37,6 +41,24 @@ pub struct Payload { pub data: Option>, // when None, payload data is stored in ufs } +impl Debug for Payload { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Payload") + .field("id", &self.id) + .field("mime_type", &self.mime_type) + .field("name", &self.name) + .field("size", &self.size) + .field( + "data", + &self + .data + .as_ref() + .map(|data| bytes_to_string_truncated(data, 256)), + ) + .finish() + } +} + impl Payload { pub fn is_human_readable(&self) -> bool { self.size < MAX_READABLE_PAYLOAD_SIZE && self.mime_type.starts_with("text/") -- 2.36.2