From a21bc40323a1a4ebcc4a8a9b73a85d97131cc224 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Thu, 16 Feb 2023 18:23:01 +0300 Subject: [PATCH] add readable fmt instead of hexlify every debug output --- Cargo.lock | 34 +++++------ bin/u_panel/src/argparse.rs | 14 ++--- bin/u_server/src/db.rs | 32 +++++----- bin/u_server/src/handlers.rs | 26 ++++---- bin/u_server/src/u_server.rs | 26 +++----- images/u_agent.Dockerfile | 6 +- integration/docker-compose.yml | 2 +- integration/integration_tests.py | 2 +- integration/tests/fixtures/agent.rs | 5 +- integration/tests/helpers/panel.rs | 6 +- integration/tests/integration/behaviour.rs | 4 +- lib/u_lib/Cargo.toml | 2 +- lib/u_lib/src/api.rs | 56 ++++++++--------- lib/u_lib/src/cache.rs | 12 ++-- lib/u_lib/src/config.rs | 7 ++- lib/u_lib/src/conv.rs | 17 ++++-- lib/u_lib/src/lib.rs | 2 +- lib/u_lib/src/messaging/mod.rs | 4 +- lib/u_lib/src/models/agent.rs | 37 +++++++++--- lib/u_lib/src/models/jobs/assigned.rs | 70 ++++++++++++++++------ lib/u_lib/src/models/jobs/meta.rs | 44 +++++++++++--- lib/u_lib/src/models/jobs/misc.rs | 4 +- lib/u_lib/src/models/payload.rs | 12 +--- lib/u_lib/src/{datatypes.rs => types.rs} | 2 + lib/u_lib/src/ufs/mod.rs | 5 ++ 25 files changed, 257 insertions(+), 174 deletions(-) rename lib/u_lib/src/{datatypes.rs => types.rs} (95%) diff --git a/Cargo.lock b/Cargo.lock index d019d9b..6bc172a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,7 +182,7 @@ dependencies = [ "serde_urlencoded", "smallvec", "socket2", - "time 0.3.17", + "time 0.3.18", "url", ] @@ -462,7 +462,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding", - "time 0.3.17", + "time 0.3.18", "version_check", ] @@ -555,9 +555,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.90" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90d59d9acd2a682b4e40605a242f6670eaa58c5957471cbf85e8aa6a0b97a5e8" +checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" dependencies = [ "cc", "cxxbridge-flags", @@ -567,9 +567,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.90" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebfa40bda659dd5c864e65f4c9a2b0aff19bea56b017b9b77c73d3766a453a38" +checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" dependencies = [ "cc", "codespan-reporting", @@ -582,15 +582,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.90" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "457ce6757c5c70dc6ecdbda6925b958aae7f959bda7d8fb9bde889e34a09dc03" +checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" [[package]] name = "cxxbridge-macro" -version = "1.0.90" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebf883b7aacd7b2aeb2a7b338648ee19f57c140d4ee8e52c68979c6b2f7f2263" +checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" dependencies = [ "proc-macro2", "quote", @@ -1257,9 +1257,9 @@ checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libflate" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05605ab2bce11bcfc0e9c635ff29ef8b2ea83f29be257ee7d730cac3ee373093" +checksum = "97822bf791bd4d5b403713886a5fbe8bf49520fe78e323b0dc480ca1a03e50b0" dependencies = [ "adler32", "crc32fast", @@ -1268,9 +1268,9 @@ dependencies = [ [[package]] name = "libflate_lz77" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39a734c0493409afcd49deee13c006a04e3586b9761a03543c6272c9c51f2f5a" +checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf" dependencies = [ "rle-decode-fast", ] @@ -2345,9 +2345,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "af0097eaf301d576d0b2aead7a59facab6d53cc636340f0291fab8446a2e8613" dependencies = [ "itoa", "serde", @@ -2509,7 +2509,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" dependencies = [ "crossbeam-channel", - "time 0.3.17", + "time 0.3.18", "tracing-subscriber", ] diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index f3a2dce..bacfc23 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -2,12 +2,12 @@ use serde_json::{from_str, to_value, Value}; use structopt::StructOpt; use u_lib::{ api::ClientHandler, - datatypes::PanelResult, messaging::AsMsg, models::{Agent, AssignedJob, FatJobMeta}, + types::Id, + types::PanelResult, UError, UResult, }; -use uuid::Uuid; #[derive(StructOpt, Debug)] pub struct Args { @@ -43,7 +43,7 @@ enum JobCmd { enum JobMapCRUD { Create { #[structopt(parse(try_from_str = parse_uuid))] - agent_id: Uuid, + agent_id: Id, job_idents: Vec, }, @@ -55,19 +55,19 @@ enum JobMapCRUD { enum RUD { Read { #[structopt(parse(try_from_str = parse_uuid))] - id: Option, + id: Option, }, Update { item: String, }, Delete { #[structopt(parse(try_from_str = parse_uuid))] - id: Uuid, + id: Id, }, } -fn parse_uuid(src: &str) -> Result { - Uuid::parse_str(src).map_err(|e| e.to_string()) +fn parse_uuid(src: &str) -> Result { + Id::parse_str(src).map_err(|e| e.to_string()) } pub fn into_value(data: M) -> Value { diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index dba691e..23b1dc1 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -3,7 +3,7 @@ use diesel::{pg::PgConnection, prelude::*, result::Error as DslError, Connection use u_lib::db::PgAsyncPool; use u_lib::models::{schema, Agent, AssignedJob, JobState, ThinJobMeta}; use u_lib::platform::Platform; -use uuid::Uuid; +use u_lib::types::Id; type Result = std::result::Result; @@ -47,7 +47,7 @@ pub struct UDB<'c> { } impl UDB<'_> { - pub fn insert_jobs(&mut self, job_metas: &[ThinJobMeta]) -> Result> { + pub fn insert_jobs(&mut self, job_metas: &[ThinJobMeta]) -> Result> { use schema::jobs; diesel::insert_into(jobs::table) @@ -57,7 +57,7 @@ impl UDB<'_> { .map_err(with_err_ctx("Can't insert jobs")) } - pub fn get_job(&mut self, id: Uuid) -> Result> { + pub fn get_job(&mut self, id: Id) -> Result> { use schema::jobs; jobs::table @@ -94,7 +94,7 @@ impl UDB<'_> { .do_update() .set(agent) .execute(self.conn) - .map_err(with_err_ctx(format!("Can't insert agent {agent:x?}")))?; + .map_err(with_err_ctx(format!("Can't insert agent {agent:?}")))?; Ok(()) } @@ -104,11 +104,11 @@ impl UDB<'_> { diesel::insert_into(results::table) .values(result) .execute(self.conn) - .map_err(with_err_ctx(format!("Can't insert result {result:x?}")))?; + .map_err(with_err_ctx(format!("Can't insert result {result:?}")))?; Ok(()) } - pub fn get_agent(&mut self, id: Uuid) -> Result> { + pub fn get_agent(&mut self, id: Id) -> Result> { use schema::agents; agents::table @@ -126,7 +126,7 @@ impl UDB<'_> { .map_err(with_err_ctx(format!("Can't get agents"))) } - pub fn update_job_status(&mut self, id: Uuid, status: JobState) -> Result<()> { + pub fn update_job_status(&mut self, id: Id, status: JobState) -> Result<()> { use schema::results; diesel::update(results::table) @@ -138,7 +138,7 @@ impl UDB<'_> { } //TODO: filters possibly could work in a wrong way, check - pub fn get_exact_jobs(&mut self, id: Option, personal: bool) -> Result> { + pub fn get_exact_jobs(&mut self, id: Option, personal: bool) -> Result> { use schema::results; let mut q = results::table.into_boxed(); @@ -163,7 +163,7 @@ impl UDB<'_> { Ok(result) } - pub fn set_jobs_for_agent(&mut self, agent_id: Uuid, job_ids: &[Uuid]) -> Result> { + pub fn set_jobs_for_agent(&mut self, agent_id: Id, job_ids: &[Id]) -> Result> { use schema::{jobs, results}; let agent_platform = match self.get_agent(agent_id)? { @@ -178,7 +178,7 @@ impl UDB<'_> { let jobs_meta = jobs::table .select((jobs::id, jobs::alias, jobs::platform)) .filter(jobs::id.eq_any(job_ids)) - .load::<(Uuid, Option, String)>(self.conn) + .load::<(Id, Option, String)>(self.conn) .map_err(with_err_ctx(format!("Can't find jobs {job_ids:?}")))?; for meta in &jobs_meta { @@ -210,7 +210,7 @@ impl UDB<'_> { Ok(job_requests.iter().map(|aj| aj.id).collect()) } - pub fn del_jobs(&mut self, ids: &[Uuid]) -> Result { + pub fn del_jobs(&mut self, ids: &[Id]) -> Result { use schema::jobs; let mut affected = 0; @@ -224,7 +224,7 @@ impl UDB<'_> { Ok(affected) } - pub fn del_results(&mut self, ids: &[Uuid]) -> Result { + pub fn del_results(&mut self, ids: &[Id]) -> Result { use schema::results; let mut affected = 0; @@ -238,7 +238,7 @@ impl UDB<'_> { Ok(affected) } - pub fn del_agents(&mut self, ids: &[Uuid]) -> Result { + pub fn del_agents(&mut self, ids: &[Id]) -> Result { use schema::agents; let mut affected = 0; @@ -255,13 +255,13 @@ impl UDB<'_> { pub fn update_agent(&mut self, agent: &Agent) -> Result<()> { agent .save_changes::(self.conn) - .map_err(with_err_ctx(format!("Can't update agent {agent:x?}")))?; + .map_err(with_err_ctx(format!("Can't update agent {agent:?}")))?; Ok(()) } pub fn update_job(&mut self, job: &ThinJobMeta) -> Result<()> { job.save_changes::(self.conn) - .map_err(with_err_ctx(format!("Can't update job {job:x?}")))?; + .map_err(with_err_ctx(format!("Can't update job {job:?}")))?; Ok(()) } @@ -272,7 +272,7 @@ impl UDB<'_> { ); result .save_changes::(self.conn) - .map_err(with_err_ctx(format!("Can't update result {result:x?}")))?; + .map_err(with_err_ctx(format!("Can't update result {result:?}")))?; Ok(()) } } diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index 2e0da72..f0fcf32 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -3,13 +3,13 @@ use std::sync::Arc; use crate::db::{PgRepo, UDB}; use crate::error::Error; use crate::ValidJobMeta; -use u_lib::jobs::{fat_meta_to_thin, thin_meta_to_fat}; use u_lib::{ + jobs::{fat_meta_to_thin, thin_meta_to_fat}, messaging::{AsMsg, Reportable}, misc::OneOrVec, models::*, + types::Id }; -use uuid::Uuid; use warp::reject::not_found; use warp::Rejection; @@ -18,7 +18,7 @@ type EndpResult = Result; pub struct Endpoints; impl Endpoints { - pub async fn get_agents(repo: Arc, id: Option) -> EndpResult> { + pub async fn get_agents(repo: Arc, id: Option) -> EndpResult> { repo.interact(move |mut db| { Ok(match id { Some(id) => { @@ -35,7 +35,7 @@ impl Endpoints { .map_err(From::from) } - pub async fn get_job(repo: Arc, id: Uuid) -> EndpResult { + pub async fn get_job(repo: Arc, id: Id) -> EndpResult { let Some(job) = repo.interact(move |mut db| db.get_job(id)).await? else { return Err(not_found()) }; @@ -52,14 +52,14 @@ impl Endpoints { pub async fn get_agent_jobs( repo: Arc, - id: Option, + id: Option, ) -> EndpResult> { repo.interact(move |mut db| db.get_exact_jobs(id, false)) .await .map_err(From::from) } - pub async fn get_personal_jobs(repo: Arc, id: Uuid) -> EndpResult> { + pub async fn get_personal_jobs(repo: Arc, id: Id) -> EndpResult> { repo.transaction(move |mut db| { let agent = db.get_agent(id)?; match agent { @@ -95,7 +95,7 @@ impl Endpoints { pub async fn upload_jobs( repo: Arc, msg: Vec, - ) -> EndpResult> { + ) -> EndpResult> { let jobs = msg .into_iter() .map(|meta| Ok(fat_meta_to_thin(meta)?)) @@ -106,7 +106,7 @@ impl Endpoints { .map_err(From::from) } - pub async fn del(repo: Arc, id: Uuid) -> EndpResult { + pub async fn del(repo: Arc, id: Id) -> EndpResult { repo.transaction(move |mut db| { let del_fns = &[UDB::del_agents, UDB::del_jobs, UDB::del_results]; for del_fn in del_fns { @@ -123,14 +123,14 @@ impl Endpoints { pub async fn set_jobs( repo: Arc, - agent_id: Uuid, + agent_id: Id, job_idents: Vec, - ) -> EndpResult> { + ) -> EndpResult> { repo.transaction(move |mut db| { job_idents .into_iter() .map(|ident| { - Uuid::parse_str(&ident).or_else(|_| { + Id::parse_str(&ident).or_else(|_| { let job_from_db = db.find_job_by_alias(&ident); match job_from_db { Ok(job) => match job { @@ -143,7 +143,7 @@ impl Endpoints { } }) }) - .collect::, Error>>() + .collect::, Error>>() .and_then(|j| db.set_jobs_for_agent(agent_id, &j)) }) .await @@ -153,7 +153,7 @@ impl Endpoints { pub async fn report + AsMsg + Send + Sync + 'static>( repo: Arc, msg: Data, - agent_id: Uuid + agent_id: Id ) -> EndpResult<()> { repo.transaction(move |mut db| { for entry in msg.into_vec() { diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index b0f86d5..34b37d4 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -18,8 +18,8 @@ use u_lib::{ jobs::fat_meta_to_thin, messaging::{AsMsg, Reportable}, models::*, + types::Id, }; -use uuid::Uuid; use warp::{ body, log::{custom, Info}, @@ -40,7 +40,7 @@ pub fn init_endpoints( db: PgRepo, ) -> impl Filter + Clone { let path = |p: &'static str| warp::post().and(warp::path(p)); - let infallible_none = |_| async { Result::<(Option,), Infallible>::Ok((None,)) }; + let infallible_none = |_| async { Result::<(Option,), Infallible>::Ok((None,)) }; let with_db = { let adb = Arc::new(db); @@ -49,11 +49,7 @@ pub fn init_endpoints( let get_agents = path("get_agents") .and(with_db.clone()) - .and( - warp::path::param::() - .map(Some) - .or_else(infallible_none), - ) + .and(warp::path::param::().map(Some).or_else(infallible_none)) .and_then(Endpoints::get_agents) .map(into_message); @@ -65,7 +61,7 @@ pub fn init_endpoints( let get_job = path("get_job") .and(with_db.clone()) - .and(warp::path::param::()) + .and(warp::path::param::()) .and_then(Endpoints::get_job) .map(into_message); @@ -76,29 +72,25 @@ pub fn init_endpoints( let get_agent_jobs = path("get_agent_jobs") .and(with_db.clone()) - .and( - warp::path::param::() - .map(Some) - .or_else(infallible_none), - ) + .and(warp::path::param::().map(Some).or_else(infallible_none)) .and_then(Endpoints::get_agent_jobs) .map(into_message); let get_personal_jobs = path("get_personal_jobs") .and(with_db.clone()) - .and(warp::path::param::()) + .and(warp::path::param::()) .and_then(Endpoints::get_personal_jobs) .map(into_message); let del = path("del") .and(with_db.clone()) - .and(warp::path::param::()) + .and(warp::path::param::()) .and_then(Endpoints::del) .map(ok); let set_jobs = path("set_jobs") .and(with_db.clone()) - .and(warp::path::param::()) + .and(warp::path::param::()) .and(body::json::>()) .and_then(Endpoints::set_jobs) .map(into_message); @@ -206,7 +198,7 @@ pub async fn serve() -> Result<(), ServerError> { async fn handle_rejection(rej: Rejection) -> Result { let resp = if let Some(err) = rej.find::() { - error!("{:x?}", err); + error!("{:?}", err); RejResponse::bad_request(err.to_string()) } else if rej.is_not_found() { RejResponse::not_found("not found placeholder") diff --git a/images/u_agent.Dockerfile b/images/u_agent.Dockerfile index ca455aa..d089c63 100644 --- a/images/u_agent.Dockerfile +++ b/images/u_agent.Dockerfile @@ -1,3 +1,5 @@ -FROM alpine:3.17 +FROM ubuntu:xenial -RUN apk add bash \ No newline at end of file +RUN apt update && apt upgrade -y +#todo: without this, request to 1.1.1.1 fails due to invalid cert (?), research more +RUN apt install curl -y diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml index b75bb88..8d82795 100644 --- a/integration/docker-compose.yml +++ b/integration/docker-compose.yml @@ -66,7 +66,7 @@ services: - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_agent:/unki/u_agent - ../logs:/unki/logs:rw working_dir: /unki - command: bash -c "/unki/u_agent u_server && sleep 3600" + command: bash -c "/unki/u_agent u_server; sleep 3600" env_file: - ../.env environment: diff --git a/integration/integration_tests.py b/integration/integration_tests.py index 512ea98..4b1ab63 100644 --- a/integration/integration_tests.py +++ b/integration/integration_tests.py @@ -64,7 +64,7 @@ def run_tests(): if not only_setup_cluster: CLUSTER.run('cargo test --test integration') except Exception as e: - CLUSTER.print_containers_logs() + #CLUSTER.print_containers_logs() fail(e) finally: _cleanup() diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs index 766f884..af023f3 100644 --- a/integration/tests/fixtures/agent.rs +++ b/integration/tests/fixtures/agent.rs @@ -1,12 +1,11 @@ use crate::helpers::ENV; use u_lib::{ api::ClientHandler, config::get_self_id, jobs::fat_meta_to_thin, messaging::Reportable, - models::*, + models::*, types::Id, }; -use uuid::Uuid; pub struct RegisteredAgent { - pub id: Uuid, + pub id: Id, } impl RegisteredAgent { diff --git a/integration/tests/helpers/panel.rs b/integration/tests/helpers/panel.rs index 1d76cc8..865e16e 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration/tests/helpers/panel.rs @@ -2,7 +2,7 @@ use serde::de::DeserializeOwned; use serde_json::{from_slice, Value}; use std::fmt::{Debug, Display}; use std::process::{Command, Output}; -use u_lib::{conv::bytes_to_string, datatypes::PanelResult, proc_output::ProcOutput}; +use u_lib::{conv::bytes_to_string, proc_output::ProcOutput, types::PanelResult}; const PANEL_BINARY: &str = "/u_panel"; @@ -40,8 +40,8 @@ impl Panel { .as_ref(), ); match &result { - PanelResult::Ok(r) => eprintln!("+<< {r:02x?}"), - PanelResult::Err(e) => eprintln!("!<< {e:02x?}"), + PanelResult::Ok(r) => eprintln!("+<< {r:?}"), + PanelResult::Err(e) => eprintln!("!<< {e:?}"), } result } diff --git a/integration/tests/integration/behaviour.rs b/integration/tests/integration/behaviour.rs index 397b893..8faa9b3 100644 --- a/integration/tests/integration/behaviour.rs +++ b/integration/tests/integration/behaviour.rs @@ -31,7 +31,7 @@ async fn setup_tasks() { let cmd = format!("map create {} {}", agent_id, job_alias); let assigned_ids: Vec = Panel::check_output(cmd); - retry_with_interval(3, AGENT_ITERATION_INTERVAL, || { + retry_with_interval(5, AGENT_ITERATION_INTERVAL, || { let result = Panel::check_output::>(format!("map read {}", assigned_ids[0])) .remove(0); @@ -64,7 +64,7 @@ async fn large_payload() { let cmd = format!("map create {agent_id} {job_alias}"); let assigned_ids: Vec = Panel::check_output(cmd); - retry_with_interval(3, AGENT_ITERATION_INTERVAL, || { + retry_with_interval(5, AGENT_ITERATION_INTERVAL, || { let result = Panel::check_output::>(format!("map read {}", assigned_ids[0])) .remove(0); diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index 6affbe7..f702142 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -10,7 +10,7 @@ edition = "2021" anyhow = { workspace = true } chrono = "0.4.19" diesel = { workspace = true, optional = true } -diesel-derive-enum = { version = "2.0.0-rc.0", features = ["postgres"], optional = true } +diesel-derive-enum = { version = "2.0.0", features = ["postgres"], optional = true } deadpool-diesel = { workspace = true, optional = true } dotenv = "0.15.0" envy = "0.4.2" diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 370ea8d..9d306a9 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -6,14 +6,14 @@ use anyhow::{Context, Result}; use reqwest::{header, header::HeaderMap, Certificate, Client, Identity, Method, Url}; use serde::de::DeserializeOwned; use serde_json::{from_str, Value}; -use uuid::Uuid; use crate::{ config::{get_self_id, MASTER_PORT}, conv::opt_to_string, messaging::{self, AsMsg}, misc::OneOrVec, - models::{self}, + models::*, + types::Id, UError, UResult, }; @@ -36,23 +36,24 @@ impl ClientHandler { default_headers.insert(header::AUTHORIZATION, format!("Bearer {pwd}")); } - let dns_response = Client::new() - .request( - Method::GET, - format!("https://1.1.1.1/dns-query?name={server}&type=A"), - ) - .header(header::ACCEPT, "application/dns-json") - .send() - .await? - .text() - .await?; - + // todo: don't rely only on dns resolve let client = { let client = Client::builder() .identity(identity) .default_headers(HeaderMap::try_from(&default_headers).unwrap()) .add_root_certificate(Certificate::from_pem(ROOT_CA_CERT).unwrap()); + let dns_response = Client::new() + .request( + Method::GET, + format!("https://1.1.1.1/dns-query?name={server}&type=A"), + ) + .header(header::ACCEPT, "application/dns-json") + .send() + .await? + .text() + .await?; + match from_str::(&dns_response).unwrap()["Answer"] .get(0) .and_then(|a| a.get("data")) @@ -115,7 +116,7 @@ impl ClientHandler { } // get jobs for client - pub async fn get_personal_jobs(&self, url_param: Uuid) -> Result> { + pub async fn get_personal_jobs(&self, url_param: Id) -> Result> { self.req(format!("get_personal_jobs/{}", url_param)).await } @@ -130,12 +131,12 @@ impl ClientHandler { } /// get exact job - pub async fn get_job(&self, job: Uuid) -> Result> { + pub async fn get_job(&self, job: Id) -> Result> { self.req(format!("get_job/{job}")).await } /// get all available jobs - pub async fn get_jobs(&self) -> Result> { + pub async fn get_jobs(&self) -> Result> { self.req("get_jobs").await } } @@ -144,52 +145,45 @@ impl ClientHandler { #[cfg(feature = "panel")] impl ClientHandler { /// agent listing - pub async fn get_agents(&self, agent: Option) -> Result> { + pub async fn get_agents(&self, agent: Option) -> Result> { self.req(format!("get_agents/{}", opt_to_string(agent))) .await } /// update agent - pub async fn update_agent(&self, agent: models::Agent) -> Result<()> { + pub async fn update_agent(&self, agent: Agent) -> Result<()> { self.req_with_payload("update_agent", agent).await } /// update job - pub async fn update_job(&self, job: models::FatJobMeta) -> Result<()> { + pub async fn update_job(&self, job: FatJobMeta) -> Result<()> { self.req_with_payload("update_job", job).await } /// update result - pub async fn update_result(&self, result: models::AssignedJob) -> Result<()> { + pub async fn update_result(&self, result: AssignedJob) -> Result<()> { self.req_with_payload("update_result", result).await } /// create and upload job - pub async fn upload_jobs( - &self, - payload: impl OneOrVec>, - ) -> Result> { + pub async fn upload_jobs(&self, payload: impl OneOrVec>) -> Result> { self.req_with_payload("upload_jobs", payload.into_vec()) .await } /// delete something - pub async fn del(&self, item: Uuid) -> Result { + pub async fn del(&self, item: Id) -> Result { self.req(format!("del/{item}")).await } /// set jobs for any agent - pub async fn set_jobs( - &self, - agent: Uuid, - job_idents: impl OneOrVec, - ) -> Result> { + pub async fn set_jobs(&self, agent: Id, job_idents: impl OneOrVec) -> Result> { self.req_with_payload(format!("set_jobs/{agent}"), job_idents.into_vec()) .await } /// get jobs for any agent - pub async fn get_agent_jobs(&self, agent: Option) -> Result> { + pub async fn get_agent_jobs(&self, agent: Option) -> Result> { self.req(format!("get_agent_jobs/{}", opt_to_string(agent))) .await } diff --git a/lib/u_lib/src/cache.rs b/lib/u_lib/src/cache.rs index 0a10b81..17ccfaa 100644 --- a/lib/u_lib/src/cache.rs +++ b/lib/u_lib/src/cache.rs @@ -1,10 +1,10 @@ use crate::models::ThinJobMeta; +use crate::types::Id; use lazy_static::lazy_static; use parking_lot::{RwLock, RwLockReadGuard}; use std::{collections::HashMap, ops::Deref}; -use uuid::Uuid; -type Cache = HashMap; +type Cache = HashMap; lazy_static! { static ref JOB_CACHE: RwLock = RwLock::new(HashMap::new()); @@ -17,11 +17,11 @@ impl JobCache { JOB_CACHE.write().insert(job_meta.id, job_meta); } - pub fn contains(id: Uuid) -> bool { + pub fn contains(id: Id) -> bool { JOB_CACHE.read().contains_key(&id) } - pub fn get<'jh>(id: Uuid) -> Option> { + pub fn get<'jh>(id: Id) -> Option> { if !Self::contains(id) { return None; } @@ -29,12 +29,12 @@ impl JobCache { Some(JobCacheHolder(lock, id)) } - pub fn remove(id: Uuid) { + pub fn remove(id: Id) { JOB_CACHE.write().remove(&id); } } -pub struct JobCacheHolder<'jh>(pub RwLockReadGuard<'jh, Cache>, pub Uuid); +pub struct JobCacheHolder<'jh>(pub RwLockReadGuard<'jh, Cache>, pub Id); impl<'jh> Deref for JobCacheHolder<'jh> { type Target = ThinJobMeta; diff --git a/lib/u_lib/src/config.rs b/lib/u_lib/src/config.rs index b189508..5eb1c7e 100644 --- a/lib/u_lib/src/config.rs +++ b/lib/u_lib/src/config.rs @@ -2,20 +2,21 @@ use envy::{from_env, prefixed, Result as EnvResult}; use lazy_static::lazy_static; use serde::Deserialize; use std::time::Duration; -use uuid::Uuid; pub use envy::Error; +use crate::types::Id; + pub const MASTER_PORT: u16 = 63714; pub const AGENT_ITERATION_INTERVAL: Duration = Duration::from_secs(5); lazy_static! { - static ref ID: Uuid = Uuid::new_v4(); + static ref ID: Id = Id::new_v4(); } #[inline] -pub fn get_self_id() -> Uuid { +pub fn get_self_id() -> Id { *ID } diff --git a/lib/u_lib/src/conv.rs b/lib/u_lib/src/conv.rs index 9cb2166..d19a8b3 100644 --- a/lib/u_lib/src/conv.rs +++ b/lib/u_lib/src/conv.rs @@ -1,8 +1,8 @@ use chrono::{offset::Local, DateTime}; use std::time::SystemTime; -pub fn bytes_to_string(v: &[u8]) -> String { - String::from_utf8_lossy(v).to_string() +pub fn bytes_to_string(data: &[u8]) -> String { + String::from_utf8_lossy(data).to_string() } pub fn opt_to_string(item: Option) -> String { @@ -12,8 +12,17 @@ pub fn opt_to_string(item: Option) -> String { } } -pub fn systime_to_string(time: &SystemTime) -> String { - DateTime::::from(*time) +pub fn systime_to_string(time: SystemTime) -> String { + DateTime::::from(time) .format("%d/%m/%Y %T") .to_string() } + +pub fn bytes_to_string_truncated(data: &[u8], max_len: usize) -> String { + if data.len() > max_len { + let truncated = &data[..max_len]; + String::from_utf8_lossy(truncated).to_string() + " " + } else { + String::from_utf8_lossy(&data).to_string() + } +} diff --git a/lib/u_lib/src/lib.rs b/lib/u_lib/src/lib.rs index 31adf56..52ada1b 100644 --- a/lib/u_lib/src/lib.rs +++ b/lib/u_lib/src/lib.rs @@ -4,7 +4,6 @@ pub mod cache; pub mod combined_result; pub mod config; pub mod conv; -pub mod datatypes; #[cfg(feature = "server")] pub mod db; pub mod error; @@ -16,6 +15,7 @@ pub mod misc; pub mod models; pub mod platform; pub mod proc_output; +pub mod types; pub mod ufs; #[cfg(unix)] pub mod unix; diff --git a/lib/u_lib/src/messaging/mod.rs b/lib/u_lib/src/messaging/mod.rs index 9e089fb..1dc699f 100644 --- a/lib/u_lib/src/messaging/mod.rs +++ b/lib/u_lib/src/messaging/mod.rs @@ -1,11 +1,11 @@ mod files; use crate::models::*; +use crate::types::Id; use crate::UError; pub use files::*; use serde::{Deserialize, Serialize}; use std::fmt::Debug; -use uuid::Uuid; pub trait AsMsg: Clone + Serialize + Debug {} @@ -17,7 +17,7 @@ impl AsMsg for FatJobMeta {} impl AsMsg for Reportable {} impl AsMsg for String {} impl AsMsg for ThinJobMeta {} -impl AsMsg for Uuid {} +impl AsMsg for Id {} impl AsMsg for i32 {} impl AsMsg for u8 {} impl AsMsg for () {} diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index 029b441..5a81866 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use std::fmt; use std::time::SystemTime; use strum::Display; @@ -11,15 +12,16 @@ mod server { #[cfg(feature = "server")] use self::server::*; -use crate::{config::get_self_id, executor::ExecResult, jobs::NamedJobBatch, platform::Platform}; - -use uuid::Uuid; +use crate::{ + config::get_self_id, conv::systime_to_string, executor::ExecResult, jobs::NamedJobBatch, + platform::Platform, types::Id, +}; #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Display)] #[cfg_attr( feature = "server", derive(DbEnum), - DieselTypePath = "sql_types::Agentstate" + ExistingTypePath = "sql_types::Agentstate" )] pub enum AgentState { New, @@ -28,7 +30,7 @@ pub enum AgentState { } //belongs_to -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[derive(Clone, Serialize, Deserialize, PartialEq)] #[cfg_attr( feature = "server", derive(Identifiable, Queryable, Insertable, AsChangeset), @@ -38,7 +40,7 @@ pub struct Agent { pub alias: Option, pub hostname: String, pub host_info: String, - pub id: Uuid, + pub id: Id, pub ip_gray: Option, pub ip_white: Option, pub is_root: bool, @@ -52,7 +54,7 @@ pub struct Agent { } impl Agent { - pub fn with_id(id: Uuid) -> Self { + pub fn with_id(id: Id) -> Self { Self { id, ..Default::default() @@ -119,3 +121,24 @@ impl Default for Agent { } } } + +impl fmt::Debug for Agent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Agent") + .field("alias", &self.alias) + .field("hostname", &self.hostname) + .field("host_info", &self.host_info) + .field("id", &self.id.to_string()) + .field("ip_gray", &self.ip_gray) + .field("ip_white", &self.ip_white) + .field("is_root", &self.is_root) + .field("is_root_allowed", &self.is_root_allowed) + .field("last_active", &systime_to_string(self.last_active)) + .field("platform", &self.platform) + .field("regtime", &systime_to_string(self.regtime)) + .field("state", &self.state) + .field("token", &self.token) + .field("username", &self.username) + .finish() + } +} diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index d19cade..0e302ef 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -1,25 +1,28 @@ use super::{JobState, JobType, ThinJobMeta}; -use crate::config::get_self_id; #[cfg(feature = "server")] use crate::models::schema::*; +use crate::{ + config::get_self_id, + conv::{bytes_to_string_truncated, systime_to_string}, + types::Id, +}; #[cfg(feature = "server")] use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; -use std::{borrow::Cow, time::SystemTime}; -use uuid::Uuid; +use std::{borrow::Cow, fmt::Debug, time::SystemTime}; -#[derive(Serialize, Deserialize, Clone, PartialEq, Debug)] +#[derive(Serialize, Deserialize, Clone, PartialEq)] #[cfg_attr( feature = "server", derive(Queryable, Identifiable, Insertable, AsChangeset), diesel(table_name = results) )] pub struct AssignedJob { - pub agent_id: Uuid, + pub agent_id: Id, pub alias: Option, pub created: SystemTime, - pub id: Uuid, - pub job_id: Uuid, + pub id: Id, + pub job_id: Id, pub result: Option>, pub state: JobState, pub exec_type: JobType, @@ -27,19 +30,48 @@ pub struct AssignedJob { pub updated: SystemTime, } +impl Debug for AssignedJob { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AssignedJob") + .field("agent_id", &self.agent_id.to_string()) + .field("alias", &self.alias) + .field("created", &systime_to_string(self.created)) + .field("id", &self.id.to_string()) + .field("job_id", &self.job_id.to_string()) + .field( + "result", + &self + .result + .as_ref() + .map(|r| bytes_to_string_truncated(&r, 256)), + ) + .field("state", &self.state) + .field("exec_type", &self.exec_type) + .field("retcode", &self.retcode) + .field("updated", &systime_to_string(self.updated)) + .finish() + } +} + #[derive(Serialize, Deserialize, Clone, Copy, Debug)] pub struct AssignedJobById { - pub agent_id: Uuid, - pub id: Uuid, - pub job_id: Uuid, + pub agent_id: Id, + pub id: Id, + pub job_id: Id, } impl From<(&ThinJobMeta, AssignedJobById)> for AssignedJob { - fn from((meta, assigned_job_by_id): (&ThinJobMeta, AssignedJobById)) -> Self { + fn from((meta, ids): (&ThinJobMeta, AssignedJobById)) -> Self { + let AssignedJobById { + agent_id, + id, + job_id, + } = ids; + AssignedJob { - id: assigned_job_by_id.id, - agent_id: assigned_job_by_id.agent_id, - job_id: assigned_job_by_id.job_id, + id, + agent_id, + job_id, alias: meta.alias.clone(), exec_type: meta.exec_type, ..Default::default() @@ -51,8 +83,8 @@ impl Default for AssignedJobById { fn default() -> Self { Self { agent_id: get_self_id(), - id: Uuid::new_v4(), - job_id: Uuid::nil(), + id: Id::new_v4(), + job_id: Id::nil(), } } } @@ -60,11 +92,11 @@ impl Default for AssignedJobById { impl Default for AssignedJob { fn default() -> Self { Self { - agent_id: Uuid::nil(), + agent_id: Id::nil(), alias: None, created: SystemTime::now(), - id: Uuid::new_v4(), - job_id: Uuid::nil(), + id: Id::new_v4(), + job_id: Id::nil(), result: None, state: JobState::Queued, retcode: None, diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 09f55c6..2feca3d 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -1,15 +1,17 @@ +use std::fmt; + use super::JobType; use crate::models::payload::Payload; #[cfg(feature = "server")] use crate::models::schema::*; use crate::platform::Platform; +use crate::types::Id; use crate::{UError, UResult}; #[cfg(feature = "server")] use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; -use uuid::Uuid; -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, Clone)] #[cfg_attr( feature = "server", derive(Queryable, Identifiable, Insertable, AsChangeset), @@ -20,7 +22,7 @@ pub struct ThinJobMeta { /// string like `bash -c {} -a 1 --arg2`, /// where {} is replaced by executable's tmp path pub argv: String, - pub id: Uuid, + pub id: Id, pub exec_type: JobType, /// target triple pub platform: String, @@ -29,7 +31,21 @@ pub struct ThinJobMeta { pub schedule: Option, } -#[derive(Serialize, Deserialize, Clone, Debug)] +impl fmt::Debug for ThinJobMeta { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ThinJobMeta") + .field("alias", &self.alias) + .field("argv", &self.argv) + .field("id", &self.id.to_string()) + .field("exec_type", &self.exec_type) + .field("platform", &self.platform) + .field("payload", &self.payload) + .field("schedule", &self.schedule) + .finish() + } +} + +#[derive(Serialize, Deserialize, Clone)] pub struct FatJobMeta { #[serde(default)] pub alias: Option, @@ -39,8 +55,8 @@ pub struct FatJobMeta { #[serde(default)] pub argv: String, - #[serde(default = "Uuid::new_v4")] - pub id: Uuid, + #[serde(default = "Id::new_v4")] + pub id: Id, #[serde(default)] pub exec_type: JobType, @@ -57,6 +73,20 @@ pub struct FatJobMeta { pub schedule: Option, } +impl fmt::Debug for FatJobMeta { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FatJobMeta") + .field("alias", &self.alias) + .field("argv", &self.argv) + .field("id", &self.id.to_string()) + .field("exec_type", &self.exec_type) + .field("platform", &self.platform) + .field("payload", &self.payload) + .field("schedule", &self.schedule) + .finish() + } +} + impl FatJobMeta { pub fn validated(self) -> UResult> { JobMetaBuilder { inner: self }.build() @@ -74,7 +104,7 @@ impl FatJobMeta { impl Default for FatJobMeta { fn default() -> Self { Self { - id: Uuid::new_v4(), + id: Id::new_v4(), alias: None, argv: String::new(), exec_type: JobType::Shell, diff --git a/lib/u_lib/src/models/jobs/misc.rs b/lib/u_lib/src/models/jobs/misc.rs index 36c06d9..6a99f7c 100644 --- a/lib/u_lib/src/models/jobs/misc.rs +++ b/lib/u_lib/src/models/jobs/misc.rs @@ -13,7 +13,7 @@ use self::server::*; #[cfg_attr( feature = "server", derive(DbEnum), - DieselTypePath = "sql_types::Jobstate" + ExistingTypePath = "sql_types::Jobstate" )] pub enum JobState { /// server created a job, but client didn't get it yet @@ -30,7 +30,7 @@ pub enum JobState { #[cfg_attr( feature = "server", derive(DbEnum), - DieselTypePath = "sql_types::Jobtype" + ExistingTypePath = "sql_types::Jobtype" )] pub enum JobType { Init, diff --git a/lib/u_lib/src/models/payload.rs b/lib/u_lib/src/models/payload.rs index f8acf58..4543ec3 100644 --- a/lib/u_lib/src/models/payload.rs +++ b/lib/u_lib/src/models/payload.rs @@ -1,4 +1,4 @@ -use crate::ufs; +use crate::{conv::bytes_to_string_truncated, ufs}; use serde::{Deserialize, Serialize}; use std::{fmt, path::PathBuf}; @@ -64,16 +64,10 @@ impl fmt::Debug for Payload { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Data(data) => { - const MAX_RESULT_LEN: usize = 256; let mut dbg = &mut f.debug_tuple("Data"); + let data = bytes_to_string_truncated(data, 256); - let readable_data = if data.len() > MAX_RESULT_LEN { - let truncated = &data[..MAX_RESULT_LEN]; - String::from_utf8_lossy(truncated).to_string() + " " - } else { - String::from_utf8_lossy(&data).to_string() - }; - dbg = dbg.field(&readable_data); + dbg = dbg.field(&data); dbg.finish() } Self::Ident(ident) => f.debug_tuple("Ident").field(ident).finish(), diff --git a/lib/u_lib/src/datatypes.rs b/lib/u_lib/src/types.rs similarity index 95% rename from lib/u_lib/src/datatypes.rs rename to lib/u_lib/src/types.rs index 25b954a..e4a011f 100644 --- a/lib/u_lib/src/datatypes.rs +++ b/lib/u_lib/src/types.rs @@ -24,3 +24,5 @@ impl ToString for PanelResult { serde_json::to_string(self).unwrap() } } + +pub type Id = uuid::Uuid; diff --git a/lib/u_lib/src/ufs/mod.rs b/lib/u_lib/src/ufs/mod.rs index 13de156..d8ec748 100644 --- a/lib/u_lib/src/ufs/mod.rs +++ b/lib/u_lib/src/ufs/mod.rs @@ -165,6 +165,11 @@ pub fn prepare_executable(name: impl AsRef) -> Result<(File, String), Error } } +#[cfg(windows)] +pub fn prepare_executable(name: impl AsRef) -> Result<(File, String), Error> { + todo!() +} + pub fn cleanup() { let index = INDEX.read();