From 37ae67bd1c1e4355b6f5c43716becaab41f5df47 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Thu, 19 Aug 2021 15:44:57 +0500 Subject: [PATCH 01/30] moved message structs to its own dir --- .gitignore | 3 +- Makefile | 8 +-- bin/u_agent/src/lib.rs | 2 - certs/gen_certs.sh | 13 ++-- integration/Cargo.toml | 3 +- integration/tests/behaviour.rs | 45 ++++++++++++ integration/tests/fixtures/agent.rs | 36 ++++++++++ integration/tests/fixtures/mod.rs | 1 + integration/tests/helpers/client.rs | 48 ------------- integration/tests/tests.rs | 69 ++----------------- lib/u_lib/src/api.rs | 27 +++++--- lib/u_lib/src/builder.rs | 21 +++--- .../src/{messaging.rs => messaging/base.rs} | 0 lib/u_lib/src/messaging/files.rs | 8 +++ lib/u_lib/src/messaging/mod.rs | 29 ++++++++ lib/u_lib/src/models/jobs/meta.rs | 15 ++-- lib/u_lib/src/models/mod.rs | 26 ++----- lib/u_lib/src/models/result.rs | 9 --- lib/u_lib/src/utils/hexlify.rs | 12 ++++ lib/u_lib/src/utils/mod.rs | 14 ++-- lib/u_lib/src/utils/storage.rs | 39 +++++++++++ 21 files changed, 240 insertions(+), 188 deletions(-) create mode 100644 integration/tests/fixtures/agent.rs create mode 100644 integration/tests/fixtures/mod.rs delete mode 100644 integration/tests/helpers/client.rs rename lib/u_lib/src/{messaging.rs => messaging/base.rs} (100%) create mode 100644 lib/u_lib/src/messaging/files.rs create mode 100644 lib/u_lib/src/messaging/mod.rs delete mode 100644 lib/u_lib/src/models/result.rs create mode 100644 lib/u_lib/src/utils/hexlify.rs create mode 100644 lib/u_lib/src/utils/storage.rs diff --git a/.gitignore b/.gitignore index 0f6b5f5..67f21e2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ data/ certs/* *.log echoer -.env.private \ No newline at end of file +.env.private +*.lock \ No newline at end of file diff --git a/Makefile b/Makefile index 969e896..ea72f7c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: _pre_build debug release run clean unit-tests integration-tests test +.PHONY: _pre_build debug release run clean unit integration test CARGO=./scripts/cargo_musl.sh @@ -17,10 +17,10 @@ release: _pre_build run: build ${CARGO} run -unit-tests: +unit: ${CARGO} test --lib -integration-tests: +integration: cd ./integration && ./integration_tests.sh -test: unit-tests integration-tests \ No newline at end of file +test: unit integration \ No newline at end of file diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 5db6b54..fefe9dc 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -2,8 +2,6 @@ // поддержка питона // резолв адреса управляющего сервера через DoT // кроссплатформенность (реализовать интерфейс для винды и никсов) -// проверка обнов -// самоуничтожение #[macro_use] extern crate log; diff --git a/certs/gen_certs.sh b/certs/gen_certs.sh index a0e9d52..f1e9d6f 100755 --- a/certs/gen_certs.sh +++ b/certs/gen_certs.sh @@ -1,17 +1,18 @@ set -ex DIR=. -V3_CFG=v3.ext +V3_CFG=$DIR/v3.ext -cat > $DIR/$V3_CFG << EOF +cat > $V3_CFG << EOF authorityKeyIdentifier=keyid,issuer basicConstraints=CA:FALSE keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment, keyAgreement, keyCertSign, cRLSign EOF - openssl req -x509 -newkey rsa:4096 -keyout $DIR/ca.key -out $DIR/ca.crt -nodes -days 365 -subj "/CN=root" openssl req -newkey rsa:4096 -keyout $DIR/alice.key -out $DIR/alice.csr -nodes -days 365 -subj "/CN=alice" openssl req -newkey rsa:4096 -keyout $DIR/server.key -out $DIR/server.csr -nodes -days 365 -subj "/CN=u_server" -openssl x509 -req -in $DIR/alice.csr -CA $DIR/ca.crt -CAkey $DIR/ca.key -out $DIR/alice.crt -set_serial 01 -days 365 -extfile $DIR/$V3_CFG -openssl x509 -req -in $DIR/server.csr -CA $DIR/ca.crt -CAkey $DIR/ca.key -out $DIR/server.crt -set_serial 01 -days 365 -extfile $DIR/$V3_CFG -openssl pkcs12 -export -out $DIR/alice.p12 -inkey $DIR/alice.key -in $DIR/alice.crt -passin pass: -passout pass: \ No newline at end of file +openssl x509 -req -in $DIR/alice.csr -CA $DIR/ca.crt -CAkey $DIR/ca.key -out $DIR/alice.crt -set_serial 01 -days 365 -extfile $V3_CFG +openssl x509 -req -in $DIR/server.csr -CA $DIR/ca.crt -CAkey $DIR/ca.key -out $DIR/server.crt -set_serial 01 -days 365 -extfile $V3_CFG +openssl pkcs12 -export -out $DIR/alice.p12 -inkey $DIR/alice.key -in $DIR/alice.crt -passin pass: -passout pass: + +rm $V3_CFG \ No newline at end of file diff --git a/integration/Cargo.toml b/integration/Cargo.toml index 28028a2..5bffbf6 100644 --- a/integration/Cargo.toml +++ b/integration/Cargo.toml @@ -16,6 +16,7 @@ serde_json = "1.0" serde = { version = "1.0.114", features = ["derive"] } futures = "0.3.5" shlex = "1.0.0" +rstest = "0.11" [dependencies.u_lib] path = "../lib/u_lib" @@ -24,4 +25,4 @@ version = "*" [[test]] name = "integration" -path = "tests/tests.rs" \ No newline at end of file +path = "tests/tests.rs" diff --git a/integration/tests/behaviour.rs b/integration/tests/behaviour.rs index 8b13789..6139d5e 100644 --- a/integration/tests/behaviour.rs +++ b/integration/tests/behaviour.rs @@ -1 +1,46 @@ +use crate::fixtures::agent::*; +use crate::helpers::Panel; +use rstest::rstest; +use std::error::Error; +use std::thread::sleep; +use std::time::Duration; +use u_lib::models::*; +use uuid::Uuid; + +type TestResult = Result>; + +#[rstest] +#[tokio::test] +async fn test_registration(#[future] register_agent: RegisteredAgent) -> TestResult { + let agent = register_agent.await; + let agents: Vec = Panel::check_output("agents list"); + let found = agents.iter().find(|v| v.id == agent.uid); + assert!(found.is_some()); + //teardown + Panel::check_status::(&format!("agents delete {}", agent.uid)); + Ok(()) //TODO: ______^^^^^ REMOV +} + +#[tokio::test] +async fn test_setup_tasks() -> TestResult { + //some independent agents should present + let agents: Vec = Panel::check_output("agents list"); + let agent_uid = agents[0].id; + let job_alias = "passwd_contents"; + let cmd = format!("jobs add --alias {} 'cat /etc/passwd'", job_alias); + Panel::check_status::(&cmd); + let cmd = format!("jobmap add {} {}", agent_uid, job_alias); + let assigned_uids: Vec = Panel::check_output(cmd); + for _ in 0..3 { + let result: Vec = + Panel::check_output(format!("jobmap list {}", assigned_uids[0])); + if result[0].state == JobState::Finished { + return Ok(()); + } else { + sleep(Duration::from_secs(5)); + eprintln!("waiting for task"); + } + } + panic!("Job didn't appear in the job map"); +} diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs new file mode 100644 index 0000000..928b661 --- /dev/null +++ b/integration/tests/fixtures/agent.rs @@ -0,0 +1,36 @@ +use u_lib::{api::ClientHandler, models::*}; +use uuid::Uuid; + +pub struct RegisteredAgent { + pub uid: Uuid, +} + +impl RegisteredAgent { + pub async fn unregister(self) { + let cli = ClientHandler::new(None); + cli.del(Some(self.uid)).await.unwrap(); + } +} + +#[fixture] +pub async fn register_agent() -> RegisteredAgent { + let cli = ClientHandler::new(None); + let agent_uid = Uuid::new_v4(); + let resp = cli + .get_personal_jobs(Some(agent_uid)) + .await + .unwrap() + .pop() + .unwrap(); + let job_id = resp.job_id; + let resp = cli.get_jobs(Some(job_id)).await.unwrap().pop().unwrap(); + assert_eq!(resp.alias, Some("agent_hello".to_string())); + let agent_data = Agent { + id: agent_uid, + ..Default::default() + }; + cli.report(&vec![ExecResult::Agent(agent_data)]) + .await + .unwrap(); + RegisteredAgent { uid: agent_uid } +} diff --git a/integration/tests/fixtures/mod.rs b/integration/tests/fixtures/mod.rs new file mode 100644 index 0000000..f17bc55 --- /dev/null +++ b/integration/tests/fixtures/mod.rs @@ -0,0 +1 @@ +pub mod agent; diff --git a/integration/tests/helpers/client.rs b/integration/tests/helpers/client.rs deleted file mode 100644 index 01c5c89..0000000 --- a/integration/tests/helpers/client.rs +++ /dev/null @@ -1,48 +0,0 @@ -use reqwest::{Client, RequestBuilder, Url}; -use serde::Serialize; -use serde_json::{from_str, json, Value}; - -const SERVER: &str = "u_server"; -const PORT: &str = "63714"; - -pub struct AgentClient { - client: Client, - base_url: Url, -} - -impl AgentClient { - pub fn new() -> Self { - Self { - client: Client::new(), - base_url: Url::parse(&format!("http://{}:{}", SERVER, PORT)).unwrap(), - } - } - - async fn process_request(&self, req: RequestBuilder, resp_needed: bool) -> Value { - let resp = req.send().await.unwrap(); - if let Err(e) = resp.error_for_status_ref() { - panic!( - "Server responded with code {}\nError: {}", - e.status() - .map(|s| s.to_string()) - .unwrap_or(String::from("")), - e.to_string() - ); - } - if !resp_needed { - return json!([]); - } - let resp: Value = from_str(&resp.text().await.unwrap()).unwrap(); - resp.get("inner").unwrap().get(0).unwrap().clone() - } - - pub async fn get>(&self, url: S) -> Value { - let req = self.client.get(self.base_url.join(url.as_ref()).unwrap()); - self.process_request(req, true).await - } - - pub async fn post, B: Serialize>(&self, url: S, body: &B) -> Value { - let req = self.client.post(self.base_url.join(url.as_ref()).unwrap()); - self.process_request(req.json(body), false).await - } -} diff --git a/integration/tests/tests.rs b/integration/tests/tests.rs index 636a4be..7ef28c7 100644 --- a/integration/tests/tests.rs +++ b/integration/tests/tests.rs @@ -1,67 +1,6 @@ +mod behaviour; +mod fixtures; mod helpers; -use helpers::Panel; - -use std::error::Error; -use std::thread::sleep; -use std::time::Duration; -use u_lib::{api::ClientHandler, models::*}; -use uuid::Uuid; - -type TestResult = Result>; - -async fn register_agent() -> Uuid { - let cli = ClientHandler::new(None); - let agent_uid = Uuid::new_v4(); - let resp = cli - .get_personal_jobs(Some(agent_uid)) - .await - .unwrap() - .pop() - .unwrap(); - let job_id = resp.job_id; - let resp = cli.get_jobs(Some(job_id)).await.unwrap().pop().unwrap(); - assert_eq!(resp.alias, Some("agent_hello".to_string())); - let agent_data = Agent { - id: agent_uid, - ..Default::default() - }; - cli.report(&vec![ExecResult::Agent(agent_data)]) - .await - .unwrap(); - agent_uid -} - -#[tokio::test] -async fn test_registration() -> TestResult { - let agent_uid = register_agent().await; - let agents: Vec = Panel::check_output("agents list"); - let found = agents.iter().find(|v| v.id == agent_uid); - assert!(found.is_some()); - //teardown - Panel::check_status::(&format!("agents delete {}", agent_uid)); - Ok(()) -} - -#[tokio::test] -async fn test_setup_tasks() -> TestResult { - //some independent agents should present - let agents: Vec = Panel::check_output("agents list"); - let agent_uid = agents[0].id; - let job_alias = "passwd_contents"; - let cmd = format!("jobs add --alias {} 'cat /etc/passwd'", job_alias); - Panel::check_status::(&cmd); - let cmd = format!("jobmap add {} {}", agent_uid, job_alias); - let assigned_uids: Vec = Panel::check_output(cmd); - for _ in 0..3 { - let result: Vec = - Panel::check_output(format!("jobmap list {}", assigned_uids[0])); - if result[0].state == JobState::Finished { - return Ok(()); - } else { - sleep(Duration::from_secs(5)); - eprintln!("waiting for task"); - } - } - panic!() -} +#[macro_use] +extern crate rstest; diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 82ba4bf..763f5f7 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -1,4 +1,5 @@ -#[allow(non_upper_case_globals)] +use crate::messaging; +//#[allow(non_upper_case_globals)] use crate::{ config::{MASTER_PORT, MASTER_SERVER}, messaging::{AsMsg, BaseMessage}, @@ -61,34 +62,42 @@ impl ClientHandler { // // get jobs for client #[api_route("GET")] - fn get_personal_jobs(&self, url_param: Option) -> VecDisplay {} + async fn get_personal_jobs(&self, url_param: Option) -> VecDisplay {} // // send something to server #[api_route("POST")] - fn report(&self, payload: &M) -> models::Empty {} + async fn report(&self, payload: &M) -> messaging::Empty {} + // + // download file + #[api_route("GET")] + async fn dl(&self, url_param: Option) -> Vec {} + // + // request download + #[api_route("POST")] + async fn dlr(&self, url_param: Option) -> messaging::DownloadInfo {} //##########// Admin area //##########// /// client listing #[api_route("GET")] - fn get_agents(&self, url_param: Option) -> VecDisplay {} + async fn get_agents(&self, url_param: Option) -> VecDisplay {} // // get all available jobs #[api_route("GET")] - fn get_jobs(&self, url_param: Option) -> VecDisplay {} + async fn get_jobs(&self, url_param: Option) -> VecDisplay {} // // create and upload job #[api_route("POST")] - fn upload_jobs(&self, payload: &[models::JobMeta]) -> models::Empty {} + async fn upload_jobs(&self, payload: &[models::JobMeta]) -> messaging::Empty {} // // delete something #[api_route("GET")] - fn del(&self, url_param: Option) -> i32 {} + async fn del(&self, url_param: Option) -> i32 {} // // set jobs for any client #[api_route("POST")] - fn set_jobs(&self, url_param: Option, payload: &[String]) -> VecDisplay {} + async fn set_jobs(&self, url_param: Option, payload: &[String]) -> VecDisplay {} // // get jobs for any client #[api_route("GET")] - fn get_agent_jobs(&self, url_param: Option) -> VecDisplay {} + async fn get_agent_jobs(&self, url_param: Option) -> VecDisplay {} } diff --git a/lib/u_lib/src/builder.rs b/lib/u_lib/src/builder.rs index da94225..5657745 100644 --- a/lib/u_lib/src/builder.rs +++ b/lib/u_lib/src/builder.rs @@ -1,15 +1,13 @@ use crate::{ - cache::JobCache, - executor::{Waiter, DynFut}, - models::{Agent, AssignedJob, JobMeta, JobType, ExecResult}, - utils::{CombinedResult, OneOrVec}, - UError, + UError, UResult, cache::JobCache, executor::{Waiter, DynFut}, + models::{Agent, AssignedJob, JobMeta, JobType, ExecResult}, + utils::{CombinedResult, OneOrVec} }; use guess_host_triple::guess_host_triple; use std::collections::HashMap; pub struct JobBuilder { - jobs: Waiter, + waiter: Waiter, } impl JobBuilder { @@ -25,11 +23,12 @@ impl JobBuilder { } let job_meta = job_meta.unwrap(); //waiting for try-blocks stabilization - let built_req = (|| { + let built_req = (|| -> UResult<()> { Ok(match job_meta.exec_type { JobType::Shell => { let meta = JobCache::get(&req.job_id).ok_or(UError::NoJob(req.job_id))?; let curr_platform = guess_host_triple().unwrap_or("unknown").to_string(); + //extend platform checking (partial check) if meta.platform != curr_platform { return Err(UError::InsuitablePlatform( meta.platform.clone(), @@ -48,7 +47,7 @@ impl JobBuilder { } } result.ok(Self { - jobs: Waiter::new(prepared), + waiter: Waiter::new(prepared), }); result } @@ -68,18 +67,18 @@ impl JobBuilder { /// Spawn jobs and pop results later pub async fn spawn(mut self) -> Self { - self.jobs = self.jobs.spawn().await; + self.waiter = self.waiter.spawn().await; self } /// Spawn jobs and wait for result pub async fn wait(self) -> Vec { - self.jobs.spawn().await.wait().await + self.waiter.spawn().await.wait().await } /// Spawn one job and wait for result pub async fn wait_one(self) -> ExecResult { - self.jobs.spawn().await.wait().await.pop().unwrap() + self.waiter.spawn().await.wait().await.pop().unwrap() } } diff --git a/lib/u_lib/src/messaging.rs b/lib/u_lib/src/messaging/base.rs similarity index 100% rename from lib/u_lib/src/messaging.rs rename to lib/u_lib/src/messaging/base.rs diff --git a/lib/u_lib/src/messaging/files.rs b/lib/u_lib/src/messaging/files.rs new file mode 100644 index 0000000..4cdb804 --- /dev/null +++ b/lib/u_lib/src/messaging/files.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +pub struct DownloadInfo { + hashsum: String, + dl_fid: Uuid, +} diff --git a/lib/u_lib/src/messaging/mod.rs b/lib/u_lib/src/messaging/mod.rs new file mode 100644 index 0000000..12898ce --- /dev/null +++ b/lib/u_lib/src/messaging/mod.rs @@ -0,0 +1,29 @@ +mod base; +mod files; + +use crate::models::*; +pub use base::{AsMsg, BaseMessage}; +pub use files::*; +use serde::{Deserialize, Serialize}; +use std::fmt; +use uuid::Uuid; + +impl AsMsg for Agent {} +impl AsMsg for AssignedJob {} +impl AsMsg for DownloadInfo {} +impl AsMsg for ExecResult {} +impl AsMsg for JobMeta {} +impl AsMsg for String {} +impl AsMsg for Uuid {} +impl AsMsg for Empty {} +impl AsMsg for i32 {} +impl AsMsg for u8 {} + +#[derive(Serialize, Deserialize, Clone, Default, Debug)] +pub struct Empty; + +impl fmt::Display for Empty { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "") + } +} diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 50e1b65..a0d9d8a 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -41,12 +41,12 @@ impl fmt::Display for JobMeta { out += &format!("\nPlatform: {}", self.platform); if self.exec_type == JobType::Shell && self.payload.is_some() { let payload = self.payload.as_ref().unwrap(); - let pld_len = { + let (pld_len, large) = { let pl = payload.len(); if pl > 20 { - 20 + (20, true) } else { - pl + (pl, false) } }; let pld_beginning = payload @@ -57,7 +57,7 @@ impl fmt::Display for JobMeta { out += &format!( "\nPayload: {}{}", String::from_utf8_lossy(&pld_beginning), - if pld_len <= 20 { "" } else { " <...>" } + if large { "" } else { " <...>" } ); } write!(f, "{}", out) @@ -69,7 +69,7 @@ impl Default for JobMeta { Self { id: Uuid::new_v4(), alias: None, - argv: String::from("/bin/bash -c {}"), + argv: String::new(), exec_type: JobType::Shell, platform: guess_host_triple().unwrap_or("unknown").to_string(), payload: None, @@ -111,9 +111,12 @@ impl JobMetaBuilder { } pub fn build(self) -> UResult { - let inner = self.inner; + let mut inner = self.inner; match inner.exec_type { JobType::Shell => { + if inner.argv == "" { + inner.argv = String::from("/bin/bash -c {}") + } let argv_parts = shlex::split(&inner.argv).ok_or(UError::JobArgsError("Shlex failed".into()))?; let empty_err = UError::JobArgsError("Empty argv".into()); diff --git a/lib/u_lib/src/models/mod.rs b/lib/u_lib/src/models/mod.rs index 29b14de..d8092f3 100644 --- a/lib/u_lib/src/models/mod.rs +++ b/lib/u_lib/src/models/mod.rs @@ -1,29 +1,13 @@ mod agent; pub mod jobs; -mod result; pub mod schema; -use crate::messaging::AsMsg; -pub use crate::models::result::ExecResult; pub use crate::models::{agent::*, jobs::*}; use serde::{Deserialize, Serialize}; -use std::fmt; -use uuid::Uuid; -impl AsMsg for Agent {} -impl AsMsg for AssignedJob {} -impl AsMsg for ExecResult {} -impl AsMsg for JobMeta {} -impl AsMsg for String {} -impl AsMsg for Uuid {} -impl AsMsg for Empty {} -impl AsMsg for i32 {} - -#[derive(Serialize, Deserialize, Clone, Default, Debug)] -pub struct Empty; - -impl fmt::Display for Empty { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "") - } +#[derive(Serialize, Deserialize, Clone, PartialEq)] +pub enum ExecResult { + Assigned(AssignedJob), + Agent(Agent), + Dummy, } diff --git a/lib/u_lib/src/models/result.rs b/lib/u_lib/src/models/result.rs deleted file mode 100644 index 849405a..0000000 --- a/lib/u_lib/src/models/result.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::models::{Agent, AssignedJob}; -use serde::{Deserialize, Serialize}; - -#[derive(Serialize, Deserialize, Clone, PartialEq)] -pub enum ExecResult { - Assigned(AssignedJob), - Agent(Agent), - Dummy, -} diff --git a/lib/u_lib/src/utils/hexlify.rs b/lib/u_lib/src/utils/hexlify.rs new file mode 100644 index 0000000..d3a0d14 --- /dev/null +++ b/lib/u_lib/src/utils/hexlify.rs @@ -0,0 +1,12 @@ +use std::fmt; + +pub struct Hexlify<'b>(pub &'b [u8]); + +impl<'a> fmt::LowerHex for Hexlify<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for byte in self.0.iter() { + write!(f, "{:02x}", byte)?; + } + Ok(()) + } +} diff --git a/lib/u_lib/src/utils/mod.rs b/lib/u_lib/src/utils/mod.rs index 13f94d3..0082b16 100644 --- a/lib/u_lib/src/utils/mod.rs +++ b/lib/u_lib/src/utils/mod.rs @@ -1,11 +1,15 @@ -pub mod combined_result; -pub mod conv; -pub mod misc; -pub mod tempfile; -pub mod vec_display; +mod combined_result; +mod conv; +mod hexlify; +mod misc; +mod storage; +mod tempfile; +mod vec_display; pub use combined_result::*; pub use conv::*; +pub use hexlify::*; pub use misc::*; +pub use storage::*; pub use tempfile::*; pub use vec_display::*; diff --git a/lib/u_lib/src/utils/storage.rs b/lib/u_lib/src/utils/storage.rs new file mode 100644 index 0000000..d96faba --- /dev/null +++ b/lib/u_lib/src/utils/storage.rs @@ -0,0 +1,39 @@ +use once_cell::sync::Lazy; +use std::cmp::Eq; +use std::collections::HashMap; +use std::hash::Hash; +use std::ops::Deref; +use std::sync::Arc; +use std::sync::{Mutex, MutexGuard}; + +//improve this later, replace job cacher with it +//possibly add different backends (memory, disk) +pub struct SharedStorage(Arc>>); + +impl SharedStorage { + pub fn new() -> Lazy> { + Lazy::new(|| SharedStorage(Arc::new(Mutex::new(HashMap::new())))) + } + + pub fn lock(&self) -> MutexGuard<'_, HashMap> { + self.0.lock().unwrap() + } + + pub fn get<'get, 'slf: 'get>(&'slf self, key: &'get Key) -> Option> { + if !self.lock().contains_key(key) { + return None; + } + let lock = self.lock(); + Some(RefHolder(lock, key)) + } +} + +pub struct RefHolder<'h, Key, Val>(pub MutexGuard<'h, HashMap>, pub &'h Key); + +impl<'h, Key: Eq + Hash, Val> Deref for RefHolder<'h, Key, Val> { + type Target = Val; + + fn deref(&self) -> &Self::Target { + self.0.get(self.1).unwrap() + } +} From 56bdb3bac717e3ba5b795eb939b02b03276f1e86 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Fri, 20 Aug 2021 03:35:48 +0500 Subject: [PATCH 02/30] new makefile, add db table for errors, minor fixes --- Makefile | 26 ---------- Makefile.toml | 48 ++++++++++++++++++ bin/u_agent/src/lib.rs | 50 ++++++++++--------- bin/u_panel/src/main.rs | 16 +++--- bin/u_server/src/db.rs | 10 +++- bin/u_server/src/filters.rs | 4 +- bin/u_server/src/handlers.rs | 27 ++++++---- bin/u_server/src/u_server.rs | 6 +-- integration/tests/behaviour.rs | 6 +-- integration/tests/fixtures/agent.rs | 6 +-- lib/u_lib/src/api.rs | 2 +- lib/u_lib/src/builder.rs | 36 ++++++------- lib/u_lib/src/errors.rs | 3 ++ lib/u_lib/src/executor.rs | 14 +++--- lib/u_lib/src/messaging/mod.rs | 10 +++- lib/u_lib/src/models/agent.rs | 21 ++++---- lib/u_lib/src/models/errors.rs | 35 +++++++++++++ lib/u_lib/src/models/jobs/assigned.rs | 28 +++++------ lib/u_lib/src/models/jobs/meta.rs | 41 +++++++-------- lib/u_lib/src/models/mod.rs | 13 ++--- lib/u_lib/src/models/schema.rs | 13 +++++ lib/u_lib/src/utils/combined_result.rs | 4 ++ .../2020-10-24-111622_create_all/down.sql | 1 + .../2020-10-24-111622_create_all/up.sql | 10 ++++ 24 files changed, 266 insertions(+), 164 deletions(-) delete mode 100644 Makefile create mode 100644 Makefile.toml create mode 100644 lib/u_lib/src/models/errors.rs diff --git a/Makefile b/Makefile deleted file mode 100644 index ea72f7c..0000000 --- a/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -.PHONY: _pre_build debug release run clean unit integration test - -CARGO=./scripts/cargo_musl.sh - -clean: - ${CARGO} clean - -_pre_build: - docker build -t unki/musllibs ./muslrust - -debug: _pre_build - ${CARGO} build - -release: _pre_build - ${CARGO} build --release - -run: build - ${CARGO} run - -unit: - ${CARGO} test --lib - -integration: - cd ./integration && ./integration_tests.sh - -test: unit integration \ No newline at end of file diff --git a/Makefile.toml b/Makefile.toml new file mode 100644 index 0000000..87cffed --- /dev/null +++ b/Makefile.toml @@ -0,0 +1,48 @@ +[config] +default_to_workspace = false + +[env] +CARGO = "./scripts/cargo_musl.sh" + +[tasks.build_cargo_image] +script = "docker build -t unki/musllibs ./muslrust" + +[tasks.clean] +command = "${CARGO}" +args = ["clean"] + +[tasks.debug] +dependencies = ["build_cargo_image"] +command = "${CARGO}" +args = ["build"] + +[tasks.release] +dependencies = ["build_cargo_image"] +command = "${CARGO}" +args = ["build", "--release"] + +[tasks.run] +script = ''' +echo "Only integration tests are supported." +exit 1 +''' + +[tasks.unit] +command = "${CARGO}" +args = ["test", "--lib", "${@}"] + +[tasks.integration] +script = ''' +cd ./integration +bash integration_tests.sh +''' + +[tasks.gen_schema] +script = ''' +cd ./integration +docker-compose up -d u_server +docker-compose down +''' + +[tasks.test] +dependencies = ["unit", "integration"] \ No newline at end of file diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index fefe9dc..eeb1b0d 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -14,32 +14,28 @@ use u_lib::{ builder::JobBuilder, cache::JobCache, executor::pop_completed, - models::{AssignedJob, ExecResult}, + messaging::Reportable, + models::AssignedJob, UID, //daemonize }; -#[macro_export] -macro_rules! retry_until_ok { - ( $body:expr ) => { - loop { - match $body { - Ok(r) => break r, - Err(e) => error!("{:?}", e), - }; - sleep(Duration::from_secs(5)).await; - } - }; -} +const ITERATION_LATENCY: u64 = 5; pub async fn process_request(job_requests: Vec, client: &ClientHandler) { if job_requests.len() > 0 { for jr in &job_requests { if !JobCache::contains(&jr.job_id) { info!("Fetching job: {}", &jr.job_id); - let fetched_job = retry_until_ok!(client.get_jobs(Some(jr.job_id)).await) - .pop() - .unwrap(); + let fetched_job = loop { + match client.get_jobs(Some(jr.job_id)).await { + Ok(mut result) => break result.pop().unwrap(), + Err(err) => { + error!("{:?} \nretrying...", err); + sleep(Duration::from_secs(ITERATION_LATENCY)).await; + } + } + }; JobCache::insert(fetched_job); } } @@ -71,16 +67,24 @@ pub async fn run_forever() { //daemonize(); env_logger::init(); let arg_ip = env::args().nth(1); - let instance = ClientHandler::new(arg_ip.as_deref()); + let client = ClientHandler::new(arg_ip.as_deref()); info!("Connecting to the server"); loop { - let job_requests: Vec = - retry_until_ok!(instance.get_personal_jobs(Some(*UID)).await).into_builtin_vec(); - process_request(job_requests, &instance).await; - let result: Vec = pop_completed().await.into_iter().collect(); + match client.get_personal_jobs(Some(*UID)).await { + Ok(resp) => { + let job_requests = resp.into_builtin_vec(); + process_request(job_requests, &client).await; + } + Err(err) => { + error!("{:?}", err); + } + } + let result: Vec = pop_completed().await.into_iter().collect(); if result.len() > 0 { - retry_until_ok!(instance.report(&result).await); + if let Err(err) = client.report(&result).await { + error!("{:?}", err); + } } - sleep(Duration::from_secs(5)).await; + sleep(Duration::from_secs(ITERATION_LATENCY)).await; } } diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index 4b86905..39f0a7b 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -1,9 +1,10 @@ use std::env; use std::fmt; +use std::process; use structopt::StructOpt; use u_lib::{ api::ClientHandler, datatypes::DataResult, messaging::AsMsg, models::JobMeta, utils::init_env, - UResult, + UError, UResult, }; use uuid::Uuid; @@ -78,7 +79,7 @@ fn parse_uuid(src: &str) -> Result { Uuid::parse_str(src).map_err(|e| e.to_string()) } -async fn process_cmd(args: Args) { +async fn process_cmd(args: Args) -> UResult<()> { struct Printer { json: bool, } @@ -100,7 +101,7 @@ async fn process_cmd(args: Args) { } } - let token = env::var("ADMIN_AUTH_TOKEN").expect("Authentication token is not set"); + let token = env::var("ADMIN_AUTH_TOKEN").map_err(|_| UError::WrongToken)?; let cli_handler = ClientHandler::new(None).password(token); let printer = Printer { json: args.json }; match args.cmd { @@ -117,8 +118,7 @@ async fn process_cmd(args: Args) { let job = JobMeta::builder() .with_shell(cmd.join(" ")) .with_alias(alias) - .build() - .unwrap(); + .build()?; printer.print(cli_handler.upload_jobs(&[job]).await); } JobALD::LD(LD::List { uid }) => printer.print(cli_handler.get_jobs(uid).await), @@ -133,11 +133,15 @@ async fn process_cmd(args: Args) { JobMapALD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await), }, } + Ok(()) } #[tokio::main] async fn main() { init_env(); let args: Args = Args::from_args(); - process_cmd(args).await; + if let Err(e) = process_cmd(args).await { + eprintln!("Error: {}", e); + process::exit(1) + } } diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index fa39ad4..d5db645 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -5,7 +5,7 @@ use std::{ sync::{Arc, Mutex, MutexGuard}, }; use u_lib::{ - models::{schema, Agent, AssignedJob, JobMeta, JobState}, + models::{schema, Agent, AgentError, AssignedJob, JobMeta, JobState}, ULocalError, ULocalResult, }; use uuid::Uuid; @@ -37,6 +37,14 @@ impl UDB { .unwrap() } + pub fn report_error(&self, error: &AgentError) -> ULocalResult<()> { + use schema::errors; + diesel::insert_into(errors::table) + .values(error) + .execute(&self.conn)?; + Ok(()) + } + pub fn insert_jobs(&self, job_metas: &[JobMeta]) -> ULocalResult<()> { use schema::jobs; diesel::insert_into(jobs::table) diff --git a/bin/u_server/src/filters.rs b/bin/u_server/src/filters.rs index 59527a0..126f11b 100644 --- a/bin/u_server/src/filters.rs +++ b/bin/u_server/src/filters.rs @@ -2,7 +2,7 @@ use crate::handlers::Endpoints; use serde::de::DeserializeOwned; use std::env; use u_lib::{ - messaging::{AsMsg, BaseMessage}, + messaging::{AsMsg, BaseMessage, Reportable}, models::*, }; use uuid::Uuid; @@ -68,7 +68,7 @@ pub fn make_filters() -> impl Filter let report = warp::post() .and(warp::path("report")) - .and(get_content::>().and_then(Endpoints::report)); + .and(get_content::>().and_then(Endpoints::report)); let auth_token = format!("Bearer {}", env::var("ADMIN_AUTH_TOKEN").unwrap()).into_boxed_str(); let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index db8ff4c..d769206 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -3,8 +3,9 @@ use diesel::SaveChangesDsl; use hyper::Body; use serde::Serialize; use u_lib::{ - messaging::{AsMsg, BaseMessage}, - models::{Agent, AgentState, AssignedJob, ExecResult, JobMeta, JobState}, + messaging::{AsMsg, BaseMessage, Reportable}, + models::*, + utils::OneOrVec, ULocalError, }; use uuid::Uuid; @@ -13,15 +14,15 @@ use warp::{ Rejection, Reply, }; -pub fn build_response>(code: StatusCode, body: S) -> Response { +pub fn build_response(code: StatusCode, body: impl Into) -> Response { Response::builder().status(code).body(body.into()).unwrap() } -pub fn build_ok>(body: S) -> Response { +pub fn build_ok(body: impl Into) -> Response { build_response(StatusCode::OK, body) } -pub fn build_err(body: S) -> Response { +pub fn build_err(body: impl ToString) -> Response { build_response(StatusCode::BAD_REQUEST, body.to_string()) } @@ -135,15 +136,15 @@ impl Endpoints { } } - pub async fn report( - msg: BaseMessage<'static, Vec>, + pub async fn report + AsMsg + 'static>( + msg: BaseMessage<'static, Data>, ) -> Result, Rejection> { info!("hnd: report"); let id = msg.id; let mut failed = vec![]; - for entry in msg.into_inner() { + for entry in msg.into_inner().into_vec() { match entry { - ExecResult::Assigned(res) => { + Reportable::Assigned(res) => { if id != res.agent_id { continue; } @@ -155,11 +156,15 @@ impl Endpoints { failed.push(e.to_string()) } } - ExecResult::Agent(mut a) => { + Reportable::Agent(mut a) => { a.state = AgentState::Active; Self::add_agent(a).await?; } - ExecResult::Dummy => (), + Reportable::Error(e) => { + let err = AgentError::from_msg(e, id); + UDB::lock_db().report_error(&err).unwrap(); + } + Reportable::Dummy => (), } } if failed.len() > 0 { diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index cbba6b1..e246bca 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -77,7 +77,7 @@ mod tests { use handlers::build_ok; use mockall::predicate::*; use test_case::test_case; - use u_lib::messaging::{AsMsg, BaseMessage}; + use u_lib::messaging::{AsMsg, BaseMessage, Reportable}; use uuid::Uuid; use warp::test::request; @@ -103,12 +103,12 @@ mod tests { async fn test_report_unauth_successful() { let mock = Endpoints::report_context(); mock.expect() - .withf(|msg: &BaseMessage<'_, Vec>| msg.inner_ref()[0] == ExecResult::Dummy) + .withf(|msg: &BaseMessage<'_, Vec>| msg.inner_ref()[0] == Reportable::Dummy) .returning(|_| Ok(build_ok(""))); request() .path("/report/") .method("POST") - .json(&vec![ExecResult::Dummy].as_message()) + .json(&vec![Reportable::Dummy].as_message()) .filter(&make_filters()) .await .unwrap(); diff --git a/integration/tests/behaviour.rs b/integration/tests/behaviour.rs index 6139d5e..616c7a9 100644 --- a/integration/tests/behaviour.rs +++ b/integration/tests/behaviour.rs @@ -3,9 +3,9 @@ use crate::helpers::Panel; use rstest::rstest; use std::error::Error; -use std::thread::sleep; use std::time::Duration; -use u_lib::models::*; +use tokio::time::sleep; +use u_lib::{messaging::Empty, models::*}; use uuid::Uuid; type TestResult = Result>; @@ -38,7 +38,7 @@ async fn test_setup_tasks() -> TestResult { if result[0].state == JobState::Finished { return Ok(()); } else { - sleep(Duration::from_secs(5)); + sleep(Duration::from_secs(5)).await; eprintln!("waiting for task"); } } diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs index 928b661..596f9a8 100644 --- a/integration/tests/fixtures/agent.rs +++ b/integration/tests/fixtures/agent.rs @@ -1,4 +1,4 @@ -use u_lib::{api::ClientHandler, models::*}; +use u_lib::{api::ClientHandler, messaging::Reportable, models::*}; use uuid::Uuid; pub struct RegisteredAgent { @@ -29,8 +29,6 @@ pub async fn register_agent() -> RegisteredAgent { id: agent_uid, ..Default::default() }; - cli.report(&vec![ExecResult::Agent(agent_data)]) - .await - .unwrap(); + cli.report(&[Reportable::Agent(agent_data)]).await.unwrap(); RegisteredAgent { uid: agent_uid } } diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 763f5f7..7dbb774 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -66,7 +66,7 @@ impl ClientHandler { // // send something to server #[api_route("POST")] - async fn report(&self, payload: &M) -> messaging::Empty {} + async fn report(&self, payload: &[messaging::Reportable]) -> messaging::Empty {} // // download file #[api_route("GET")] diff --git a/lib/u_lib/src/builder.rs b/lib/u_lib/src/builder.rs index 5657745..4075a24 100644 --- a/lib/u_lib/src/builder.rs +++ b/lib/u_lib/src/builder.rs @@ -1,6 +1,7 @@ use crate::{ UError, UResult, cache::JobCache, executor::{Waiter, DynFut}, - models::{Agent, AssignedJob, JobMeta, JobType, ExecResult}, + models::{Agent, AssignedJob, JobMeta, JobType}, + messaging::Reportable, utils::{CombinedResult, OneOrVec} }; use guess_host_triple::guess_host_triple; @@ -72,12 +73,12 @@ impl JobBuilder { } /// Spawn jobs and wait for result - pub async fn wait(self) -> Vec { + pub async fn wait(self) -> Vec { self.waiter.spawn().await.wait().await } /// Spawn one job and wait for result - pub async fn wait_one(self) -> ExecResult { + pub async fn wait_one(self) -> Reportable { self.waiter.spawn().await.wait().await.pop().unwrap() } } @@ -86,7 +87,7 @@ impl JobBuilder { pub struct NamedJobBuilder { builder: Option, job_names: Vec<&'static str>, - results: HashMap<&'static str, ExecResult>, + results: HashMap<&'static str, Reportable>, } impl NamedJobBuilder { @@ -136,11 +137,11 @@ impl NamedJobBuilder { self } - pub fn pop_opt(&mut self, name: &'static str) -> Option { + pub fn pop_opt(&mut self, name: &'static str) -> Option { self.results.remove(name) } - pub fn pop(&mut self, name: &'static str) -> ExecResult { + pub fn pop(&mut self, name: &'static str) -> Reportable { self.pop_opt(name).unwrap() } } @@ -148,13 +149,13 @@ impl NamedJobBuilder { #[cfg(test)] mod tests { + use super::*; use test_case::test_case; use std::{time::SystemTime}; use crate::{ errors::UError, models::{ - jobs::{JobMeta}, - ExecResult, + JobMeta, misc::JobType }, builder::{JobBuilder, NamedJobBuilder}, @@ -192,7 +193,8 @@ mod tests { TMPPATH=/tmp/lol mkdir -p $TMPPATH echo test03 > $TMPPATH/t - cat $TMPPATH/t"# + cat $TMPPATH/t + rm -rf $TMPPATH"# ), "test03" ;"sh multiline payload" @@ -211,8 +213,8 @@ mod tests { } let job = job.build().unwrap(); let job_result = JobBuilder::from_meta(job).unwrap_one().wait_one().await; - let result = unwrap_enum!(job_result, ExecResult::Assigned); - let result = result.to_string_result().unwrap(); + let result = unwrap_enum!(job_result, Reportable::Assigned); + let result = result.to_string_result(); assert_eq!(result.trim(), expected_result); Ok(()) } @@ -226,9 +228,9 @@ mod tests { let ls = JobBuilder::from_meta(JobMeta::from_shell("ls")?).unwrap_one() .wait_one() .await; - let ls = unwrap_enum!(ls, ExecResult::Assigned); + let ls = unwrap_enum!(ls, Reportable::Assigned); assert_eq!(ls.retcode.unwrap(), 0); - let folders = ls.to_string_result().unwrap(); + let folders = ls.to_string_result(); let subfolders_jobs: Vec = folders .lines() .map(|f| JobMeta::from_shell(format!("ls {}", f)).unwrap()) @@ -238,7 +240,7 @@ mod tests { .wait() .await; for result in ls_subfolders { - let result = unwrap_enum!(result, ExecResult::Assigned); + let result = unwrap_enum!(result, Reportable::Assigned); assert_eq!(result.retcode.unwrap(), 0); } longest_job.wait().await; @@ -270,8 +272,8 @@ mod tests { .unwrap_one() .wait_one() .await; - let job_result = unwrap_enum!(job_result, ExecResult::Assigned); - let output = job_result.to_string_result().unwrap(); + let job_result = unwrap_enum!(job_result, Reportable::Assigned); + let output = job_result.to_string_result(); assert!(output.contains("No such file")); assert!(job_result.retcode.is_none()); Ok(()) @@ -308,7 +310,7 @@ mod tests { ("gatherer", JobMeta::builder().with_type(JobType::Manage).build()?) ]).wait().await; let gathered = jobs.pop("gatherer"); - assert_eq!(unwrap_enum!(gathered, ExecResult::Agent).alias, None); + assert_eq!(unwrap_enum!(gathered, Reportable::Agent).alias, None); Ok(()) } diff --git a/lib/u_lib/src/errors.rs b/lib/u_lib/src/errors.rs index 206e363..5f46197 100644 --- a/lib/u_lib/src/errors.rs +++ b/lib/u_lib/src/errors.rs @@ -35,6 +35,9 @@ pub enum UError { #[error("Error opening {0}: {1}")] FSError(String, String), + + #[error("Wrong auth token")] + WrongToken, } impl From for UError { diff --git a/lib/u_lib/src/executor.rs b/lib/u_lib/src/executor.rs index 1c128e7..4155cf1 100644 --- a/lib/u_lib/src/executor.rs +++ b/lib/u_lib/src/executor.rs @@ -1,4 +1,4 @@ -use crate::{models::ExecResult, utils::OneOrVec}; +use crate::{messaging::Reportable, utils::OneOrVec}; use futures::{future::BoxFuture, lock::Mutex}; use lazy_static::lazy_static; use std::collections::HashMap; @@ -9,7 +9,7 @@ use tokio::{ }; use uuid::Uuid; -pub type DynFut = BoxFuture<'static, ExecResult>; +pub type DynFut = BoxFuture<'static, Reportable>; lazy_static! { static ref FUT_RESULTS: Mutex> = Mutex::new(HashMap::new()); @@ -21,7 +21,7 @@ lazy_static! { } struct JoinInfo { - handle: JoinHandle, + handle: JoinHandle, completed: bool, collectable: bool, // indicates if future can be popped from pool via pop_task_if_completed } @@ -68,7 +68,7 @@ impl Waiter { /// Wait until a bunch of tasks is finished. /// NOT GUARANTEED that all tasks will be returned due to /// possibility to pop them in other places - pub async fn wait(self) -> Vec { + pub async fn wait(self) -> Vec { let mut result = vec![]; for fid in self.fids { if let Some(task) = pop_task(fid).await { @@ -94,7 +94,7 @@ async fn pop_task(fid: Uuid) -> Option { FUT_RESULTS.lock().await.remove(&fid) } -pub async fn pop_task_if_completed(fid: Uuid) -> Option { +pub async fn pop_task_if_completed(fid: Uuid) -> Option { let &mut JoinInfo { handle: _, collectable, @@ -112,8 +112,8 @@ pub async fn pop_task_if_completed(fid: Uuid) -> Option { } } -pub async fn pop_completed() -> Vec { - let mut completed: Vec = vec![]; +pub async fn pop_completed() -> Vec { + let mut completed: Vec = vec![]; let fids = FUT_RESULTS .lock() .await diff --git a/lib/u_lib/src/messaging/mod.rs b/lib/u_lib/src/messaging/mod.rs index 12898ce..64485ae 100644 --- a/lib/u_lib/src/messaging/mod.rs +++ b/lib/u_lib/src/messaging/mod.rs @@ -11,7 +11,7 @@ use uuid::Uuid; impl AsMsg for Agent {} impl AsMsg for AssignedJob {} impl AsMsg for DownloadInfo {} -impl AsMsg for ExecResult {} +impl AsMsg for Reportable {} impl AsMsg for JobMeta {} impl AsMsg for String {} impl AsMsg for Uuid {} @@ -27,3 +27,11 @@ impl fmt::Display for Empty { write!(f, "") } } + +#[derive(Serialize, Deserialize, Clone, PartialEq)] +pub enum Reportable { + Assigned(AssignedJob), + Agent(Agent), + Dummy, + Error(String), +} diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index 16a78db..bcc46e9 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -5,11 +5,8 @@ use std::{fmt, time::SystemTime}; use strum::Display; use crate::{ - builder::NamedJobBuilder, - models::{schema::*, ExecResult}, - unwrap_enum, - utils::systime_to_string, - UID, + builder::NamedJobBuilder, messaging::Reportable, models::schema::*, unwrap_enum, + utils::systime_to_string, UID, }; use guess_host_triple::guess_host_triple; @@ -54,8 +51,8 @@ pub struct Agent { impl fmt::Display for Agent { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut out = format!("Agent: {}", self.id); - if self.alias.is_some() { - out += &format!(" ({})", self.alias.as_ref().unwrap()) + if let Some(ref alias) = self.alias { + out += &format!(" ({})", alias) } out += &format!("\nUsername: {}", self.username); out += &format!("\nHostname: {}", self.hostname); @@ -85,9 +82,9 @@ impl Agent { .unwrap_one() .wait() .await; - let decoder = |job_result: ExecResult| { - let assoc_job = unwrap_enum!(job_result, ExecResult::Assigned); - assoc_job.to_string_result().unwrap().trim().to_string() + let decoder = |job_result: Reportable| { + let assoc_job = unwrap_enum!(job_result, Reportable::Assigned); + assoc_job.to_string_result().trim().to_string() }; Self { @@ -99,9 +96,9 @@ impl Agent { } } - pub async fn run() -> ExecResult { + pub async fn run() -> Reportable { #[cfg(unix)] - ExecResult::Agent(Agent::gather().await) + Reportable::Agent(Agent::gather().await) } } diff --git a/lib/u_lib/src/models/errors.rs b/lib/u_lib/src/models/errors.rs new file mode 100644 index 0000000..a916dd0 --- /dev/null +++ b/lib/u_lib/src/models/errors.rs @@ -0,0 +1,35 @@ +use crate::models::schema::*; +use diesel::{Insertable, Queryable}; +use serde::{Deserialize, Serialize}; +use std::time::SystemTime; +use uuid::Uuid; + +#[derive(Serialize, Deserialize, Clone, Debug, Queryable, Insertable, PartialEq)] +#[table_name = "errors"] +pub struct AgentError { + pub agent_id: Uuid, + pub created: SystemTime, + pub id: Uuid, + pub msg: String, +} + +impl AgentError { + pub fn from_msg(msg: impl Into, agent_id: Uuid) -> Self { + AgentError { + agent_id, + msg: msg.into(), + ..Default::default() + } + } +} + +impl Default for AgentError { + fn default() -> Self { + Self { + agent_id: Uuid::new_v4(), + created: SystemTime::now(), + id: Uuid::new_v4(), + msg: String::new(), + } + } +} diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index b11c50b..a4a0d35 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -1,13 +1,14 @@ use super::JobState; use crate::{ cache::JobCache, - models::{schema::*, ExecResult, JobOutput}, + messaging::Reportable, + models::{schema::*, JobOutput}, utils::{systime_to_string, TempFile}, UID, }; use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; -use std::{fmt, process::Output, string::FromUtf8Error, time::SystemTime}; +use std::{fmt, process::Output, time::SystemTime}; use tokio::process::Command; use uuid::Uuid; @@ -40,20 +41,17 @@ impl fmt::Display for AssignedJob { let mut out = format!("Result: {}", self.id); out += &format!("\nAgent: {}", self.agent_id); out += &format!("\nJob: {}", self.job_id); - if self.alias.is_some() { - out += &format!("\nAlias: {}", self.alias.as_ref().unwrap()); + if let Some(ref alias) = self.alias { + out += &format!("\nAlias: {}", alias); } out += &format!("\nUpdated: {}", systime_to_string(&self.updated)); out += &format!("\nState: {}", self.state); if self.state == JobState::Finished { - if self.retcode.is_some() { - out += &format!("\nReturn code: {}", self.retcode.unwrap()); + if let Some(ref retcode) = self.retcode { + out += &format!("\nReturn code: {}", retcode); } - if self.result.is_some() { - out += &format!( - "\nResult: {}", - String::from_utf8_lossy(self.result.as_ref().unwrap()) - ); + if let Some(ref result) = self.result { + out += &format!("\nResult: {}", String::from_utf8_lossy(result)); } } write!(f, "{}", out) @@ -77,7 +75,7 @@ impl Default for AssignedJob { } impl AssignedJob { - pub async fn run(mut self) -> ExecResult { + pub async fn run(mut self) -> Reportable { let (argv, _payload) = { let meta = JobCache::get(&self.job_id).unwrap(); let extracted_payload = meta @@ -118,7 +116,7 @@ impl AssignedJob { self.retcode = retcode; self.updated = SystemTime::now(); self.state = JobState::Finished; - ExecResult::Assigned(self) + Reportable::Assigned(self) } pub fn new(job_id: Uuid, other: Option<&Self>) -> Self { @@ -145,7 +143,7 @@ impl AssignedJob { } } - pub fn to_string_result(&self) -> Result { - String::from_utf8(self.to_raw_result()) + pub fn to_string_result(&self) -> String { + String::from_utf8_lossy(&self.to_raw_result()).into_owned() } } diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index a0d9d8a..4cb07cc 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -33,32 +33,29 @@ impl JobMeta { impl fmt::Display for JobMeta { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut out = format!("Job: {}", self.id); - if self.alias.is_some() { - out += &format!(" ({})", self.alias.as_ref().unwrap()); + if let Some(ref alias) = self.alias { + out += &format!(" ({})", alias); } out += &format!("\nArgv: {}", self.argv); out += &format!("\nExecutable type: {}", self.exec_type); out += &format!("\nPlatform: {}", self.platform); - if self.exec_type == JobType::Shell && self.payload.is_some() { - let payload = self.payload.as_ref().unwrap(); - let (pld_len, large) = { - let pl = payload.len(); - if pl > 20 { - (20, true) - } else { - (pl, false) - } - }; - let pld_beginning = payload - .iter() - .take(pld_len) - .map(|u| *u) - .collect::>(); - out += &format!( - "\nPayload: {}{}", - String::from_utf8_lossy(&pld_beginning), - if large { "" } else { " <...>" } - ); + if let Some(ref payload) = self.payload { + if self.exec_type == JobType::Shell { + let (pld_len, large) = { + let pl = payload.len(); + if pl > 20 { + (20, true) + } else { + (pl, false) + } + }; + let pld_beginning = &payload[..pld_len]; + out += &format!( + "\nPayload: {}{}", + String::from_utf8_lossy(pld_beginning), + if large { "" } else { " <...>" } + ); + } } write!(f, "{}", out) } diff --git a/lib/u_lib/src/models/mod.rs b/lib/u_lib/src/models/mod.rs index d8092f3..bd44781 100644 --- a/lib/u_lib/src/models/mod.rs +++ b/lib/u_lib/src/models/mod.rs @@ -1,13 +1,6 @@ mod agent; -pub mod jobs; +mod errors; +mod jobs; pub mod schema; -pub use crate::models::{agent::*, jobs::*}; -use serde::{Deserialize, Serialize}; - -#[derive(Serialize, Deserialize, Clone, PartialEq)] -pub enum ExecResult { - Assigned(AssignedJob), - Agent(Agent), - Dummy, -} +pub use crate::models::{agent::*, errors::*, jobs::*}; diff --git a/lib/u_lib/src/models/schema.rs b/lib/u_lib/src/models/schema.rs index be2edf1..83c1f59 100644 --- a/lib/u_lib/src/models/schema.rs +++ b/lib/u_lib/src/models/schema.rs @@ -26,6 +26,17 @@ table! { } } +table! { + use crate::schema_exports::*; + + errors (id) { + agent_id -> Uuid, + created -> Timestamp, + id -> Uuid, + msg -> Nullable, + } +} + table! { use crate::schema_exports::*; @@ -71,6 +82,7 @@ table! { } joinable!(certificates -> agents (agent_id)); +joinable!(errors -> agents (agent_id)); joinable!(ip_addrs -> agents (agent_id)); joinable!(results -> agents (agent_id)); joinable!(results -> jobs (job_id)); @@ -78,6 +90,7 @@ joinable!(results -> jobs (job_id)); allow_tables_to_appear_in_same_query!( agents, certificates, + errors, ip_addrs, jobs, results, diff --git a/lib/u_lib/src/utils/combined_result.rs b/lib/u_lib/src/utils/combined_result.rs index 4699ae6..2baf35f 100644 --- a/lib/u_lib/src/utils/combined_result.rs +++ b/lib/u_lib/src/utils/combined_result.rs @@ -30,6 +30,10 @@ impl CombinedResult { self.ok } + pub fn has_err(&self) -> bool { + self.err.len() > 0 + } + pub fn unwrap_one(self) -> T { self.unwrap().pop().unwrap() } diff --git a/migrations/2020-10-24-111622_create_all/down.sql b/migrations/2020-10-24-111622_create_all/down.sql index 3ded775..c3bfd8c 100644 --- a/migrations/2020-10-24-111622_create_all/down.sql +++ b/migrations/2020-10-24-111622_create_all/down.sql @@ -3,6 +3,7 @@ DROP TABLE results; DROP TABLE certificates; DROP TABLE jobs; DROP TABLE agents; +DROP TABLE errors; DROP TYPE IF EXISTS JobState; DROP TYPE IF EXISTS JobType; diff --git a/migrations/2020-10-24-111622_create_all/up.sql b/migrations/2020-10-24-111622_create_all/up.sql index ef956ac..ec2949f 100644 --- a/migrations/2020-10-24-111622_create_all/up.sql +++ b/migrations/2020-10-24-111622_create_all/up.sql @@ -66,4 +66,14 @@ CREATE TABLE IF NOT EXISTS certificates ( , is_revoked BOOLEAN NOT NULL DEFAULT FALSE , PRIMARY KEY(id) , FOREIGN KEY(agent_id) REFERENCES agents(id) +); + + +CREATE TABLE IF NOT EXISTS errors ( + agent_id UUID NOT NULL + , created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + , id UUID NOT NULL DEFAULT uuid_generate_v4() + , msg TEXT + , FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE + , PRIMARY KEY(id) ); \ No newline at end of file From 348bf4a90ae52f2fc19e6dcc43a134b4a17d8839 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sun, 22 Aug 2021 03:08:35 +0500 Subject: [PATCH 03/30] error chan --- bin/u_agent/src/lib.rs | 61 +++++++++----- bin/u_server/src/filters.rs | 6 +- bin/u_server/src/handlers.rs | 7 +- bin/u_server/src/u_server.rs | 21 ++--- integration/tests/helpers/panel.rs | 13 ++- lib/u_api_proc_macro/src/lib.rs | 8 +- lib/u_lib/Cargo.toml | 5 +- lib/u_lib/src/builder.rs | 19 ++--- lib/u_lib/src/cache.rs | 1 + lib/u_lib/src/config.rs | 1 + lib/u_lib/src/datatypes.rs | 4 +- lib/u_lib/src/errors/chan.rs | 28 +++++++ lib/u_lib/src/errors/mod.rs | 5 ++ .../src/{errors.rs => errors/variants.rs} | 46 ++++++++++- lib/u_lib/src/executor.rs | 8 +- lib/u_lib/src/lib.rs | 5 +- lib/u_lib/src/models/{errors.rs => error.rs} | 0 lib/u_lib/src/models/jobs/assigned.rs | 25 +++--- lib/u_lib/src/models/jobs/meta.rs | 29 +++---- lib/u_lib/src/models/mod.rs | 4 +- lib/u_lib/src/utils/combined_result.rs | 4 +- lib/u_lib/src/utils/{ => fmt}/hexlify.rs | 12 +++ lib/u_lib/src/utils/fmt/mod.rs | 5 ++ lib/u_lib/src/utils/fmt/stripped.rs | 80 +++++++++++++++++++ lib/u_lib/src/utils/misc.rs | 2 +- lib/u_lib/src/utils/mod.rs | 4 +- lib/u_lib/src/utils/tempfile.rs | 12 +-- scripts/cargo_musl.sh | 1 + 28 files changed, 311 insertions(+), 105 deletions(-) create mode 100644 lib/u_lib/src/errors/chan.rs create mode 100644 lib/u_lib/src/errors/mod.rs rename lib/u_lib/src/{errors.rs => errors/variants.rs} (55%) rename lib/u_lib/src/models/{errors.rs => error.rs} (100%) rename lib/u_lib/src/utils/{ => fmt}/hexlify.rs (52%) create mode 100644 lib/u_lib/src/utils/fmt/mod.rs create mode 100644 lib/u_lib/src/utils/fmt/stripped.rs diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index eeb1b0d..946afc9 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -8,14 +8,18 @@ extern crate log; extern crate env_logger; use std::env; +use std::panic; +use std::sync::Arc; use tokio::time::{sleep, Duration}; use u_lib::{ api::ClientHandler, builder::JobBuilder, cache::JobCache, + errors::ErrChan, executor::pop_completed, messaging::Reportable, models::AssignedJob, + UError, UID, //daemonize }; @@ -26,12 +30,12 @@ pub async fn process_request(job_requests: Vec, client: &ClientHand if job_requests.len() > 0 { for jr in &job_requests { if !JobCache::contains(&jr.job_id) { - info!("Fetching job: {}", &jr.job_id); + debug!("Fetching job: {}", &jr.job_id); let fetched_job = loop { match client.get_jobs(Some(jr.job_id)).await { Ok(mut result) => break result.pop().unwrap(), Err(err) => { - error!("{:?} \nretrying...", err); + debug!("{:?} \nretrying...", err); sleep(Duration::from_secs(ITERATION_LATENCY)).await; } } @@ -39,7 +43,7 @@ pub async fn process_request(job_requests: Vec, client: &ClientHand JobCache::insert(fetched_job); } } - info!( + debug!( "Scheduling jobs: {}", job_requests .iter() @@ -50,41 +54,56 @@ pub async fn process_request(job_requests: Vec, client: &ClientHand let mut builder = JobBuilder::from_request(job_requests); let errors = builder.pop_errors(); if errors.len() > 0 { - error!( - "Some errors encountered: \n{}", - errors - .iter() - .map(|j| j.to_string()) - .collect::>() - .join("\n") - ); + errors.into_iter().for_each(ErrChan::send) } builder.unwrap_one().spawn().await; } } -pub async fn run_forever() { - //daemonize(); - env_logger::init(); - let arg_ip = env::args().nth(1); - let client = ClientHandler::new(arg_ip.as_deref()); - info!("Connecting to the server"); +async fn error_reporting(client: Arc) { + loop { + let err = ErrChan::recv(); + debug!("Error encountered: {:?}", err); + 'retry: for _ in 0..3 { + match client.report(&[Reportable::Error(err.to_string())]).await { + Ok(_) => break 'retry, + Err(e) => { + debug!("Reporting error: {:?}", e); + sleep(Duration::from_secs(10)).await; + } + } + } + } +} + +async fn do_stuff(client: Arc) -> ! { loop { match client.get_personal_jobs(Some(*UID)).await { Ok(resp) => { let job_requests = resp.into_builtin_vec(); process_request(job_requests, &client).await; } - Err(err) => { - error!("{:?}", err); - } + Err(err) => ErrChan::send(err), } let result: Vec = pop_completed().await.into_iter().collect(); if result.len() > 0 { if let Err(err) = client.report(&result).await { - error!("{:?}", err); + ErrChan::send(err) } } sleep(Duration::from_secs(ITERATION_LATENCY)).await; } } + +pub async fn run_forever() { + //daemonize(); + env_logger::init(); + let arg_ip = env::args().nth(1); + let client = Arc::new(ClientHandler::new(arg_ip.as_deref())); + let _cli = client.clone(); + panic::set_hook(Box::new(|panic_info| { + ErrChan::send(UError::Panic(panic_info.to_string())) + })); + tokio::spawn(error_reporting(_cli)); + do_stuff(client).await; +} diff --git a/bin/u_server/src/filters.rs b/bin/u_server/src/filters.rs index 126f11b..20ba803 100644 --- a/bin/u_server/src/filters.rs +++ b/bin/u_server/src/filters.rs @@ -70,7 +70,11 @@ pub fn make_filters() -> impl Filter .and(warp::path("report")) .and(get_content::>().and_then(Endpoints::report)); - let auth_token = format!("Bearer {}", env::var("ADMIN_AUTH_TOKEN").unwrap()).into_boxed_str(); + let auth_token = format!( + "Bearer {}", + env::var("ADMIN_AUTH_TOKEN").expect("No auth token provided") + ) + .into_boxed_str(); let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); let auth_zone = (get_agents diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index d769206..cdf9e47 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -5,7 +5,7 @@ use serde::Serialize; use u_lib::{ messaging::{AsMsg, BaseMessage, Reportable}, models::*, - utils::OneOrVec, + utils::{OneOrVec, Stripped}, ULocalError, }; use uuid::Uuid; @@ -162,6 +162,11 @@ impl Endpoints { } Reportable::Error(e) => { let err = AgentError::from_msg(e, id); + warn!( + "{} reported an error: {}", + err.agent_id, + Stripped(&err.msg.as_str()) + ); UDB::lock_db().report_error(&err).unwrap(); } Reportable::Dummy => (), diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index e246bca..71d8185 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -1,22 +1,24 @@ #[macro_use] extern crate log; -#[macro_use] +#[cfg_attr(test, macro_use)] extern crate mockall; -#[macro_use] +#[cfg_attr(test, macro_use)] extern crate mockall_double; -// because of linking errors +// due to linking errors extern crate openssl; -#[macro_use] +// don't touch anything extern crate diesel; -// +// in this block + mod db; mod filters; mod handlers; use db::UDB; use filters::make_filters; +use std::path::PathBuf; use u_lib::{config::MASTER_PORT, models::*, utils::init_env}; use warp::Filter; @@ -28,7 +30,7 @@ fn prefill_jobs() { .with_alias("agent_hello") .build() .unwrap(); - UDB::lock_db().insert_jobs(&[agent_hello]).ok(); + UDB::lock_db().insert_jobs(&[agent_hello]).unwrap(); } fn init_logger() { @@ -60,11 +62,12 @@ fn init_all() { pub async fn serve() { init_all(); let routes = make_filters(); + let certs_dir = PathBuf::from("certs"); warp::serve(routes.with(warp::log("warp"))) .tls() - .cert_path("./certs/server.crt") - .key_path("./certs/server.key") - .client_auth_required_path("./certs/ca.crt") + .cert_path(certs_dir.join("server.crt")) + .key_path(certs_dir.join("server.key")) + .client_auth_required_path(certs_dir.join("ca.crt")) .run(([0, 0, 0, 0], MASTER_PORT)) .await; } diff --git a/integration/tests/helpers/panel.rs b/integration/tests/helpers/panel.rs index 41008f9..2c61245 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration/tests/helpers/panel.rs @@ -2,7 +2,7 @@ use serde::de::DeserializeOwned; use serde_json::from_slice; use shlex::split; use std::process::{Command, Output}; -use u_lib::{datatypes::DataResult, messaging::AsMsg}; +use u_lib::{datatypes::DataResult, messaging::AsMsg, utils::VecDisplay}; const PANEL_BINARY: &str = "/u_panel"; @@ -21,8 +21,13 @@ impl Panel { pub fn output_argv(args: &[&str]) -> PanelResult { let result = Self::run(args); - assert!(result.status.success()); - from_slice(&result.stdout).map_err(|e| e.to_string()) + from_slice(&result.stdout).map_err(|e| { + eprintln!( + "Failed to decode panel response: '{}'", + VecDisplay(result.stdout) + ); + e.to_string() + }) } pub fn output(args: impl Into) -> PanelResult { @@ -39,7 +44,7 @@ impl Panel { fn status_is_ok(data: PanelResult) -> T { match data.unwrap() { DataResult::Ok(r) => r, - DataResult::Err(e) => panic!("Panel failed with erroneous status: {}", e), + DataResult::Err(e) => panic!("Panel failed: {}", e), } } diff --git a/lib/u_api_proc_macro/src/lib.rs b/lib/u_api_proc_macro/src/lib.rs index 46f1efa..f3d3a94 100644 --- a/lib/u_api_proc_macro/src/lib.rs +++ b/lib/u_api_proc_macro/src/lib.rs @@ -69,7 +69,7 @@ pub fn api_route(args: TokenStream, item: TokenStream) -> TokenStream { Ok(_) => Ok(()), Err(e) => Err(UError::from(e)) }; - match is_success { + let result = match is_success { Ok(_) => response.json::>() .await .map(|msg| msg.into_inner()) @@ -82,14 +82,14 @@ pub fn api_route(args: TokenStream, item: TokenStream) -> TokenStream { Err(UError::NetError(err_src, _)) => Err( UError::NetError( err_src, - response.text().await.unwrap() + response.text().await? ) ), _ => unreachable!() - } + }; + Ok(result?) } }; - //eprintln!("#!#! RESULT:\n{}", q); q.into() } diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index 5d56eff..6d66582 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -28,10 +28,13 @@ strum = { version = "0.20", features = ["derive"] } once_cell = "1.7.2" shlex = "1.0.0" u_api_proc_macro = { version = "*", path = "../u_api_proc_macro" } +crossbeam = "0.8.1" +backtrace = "0.3.61" [dependencies.diesel] version = "1.4.5" features = ["postgres", "uuid"] [dev-dependencies] -test-case = "1.1.0" \ No newline at end of file +test-case = "1.1.0" +rstest = "0.11" diff --git a/lib/u_lib/src/builder.rs b/lib/u_lib/src/builder.rs index 4075a24..726d74a 100644 --- a/lib/u_lib/src/builder.rs +++ b/lib/u_lib/src/builder.rs @@ -1,9 +1,4 @@ -use crate::{ - UError, UResult, cache::JobCache, executor::{Waiter, DynFut}, - models::{Agent, AssignedJob, JobMeta, JobType}, - messaging::Reportable, - utils::{CombinedResult, OneOrVec} -}; +use crate::{UError, UErrorBt, UResult, cache::JobCache, executor::{Waiter, DynFut}, messaging::Reportable, models::{Agent, AssignedJob, JobMeta, JobType}, utils::{CombinedResult, OneOrVec}}; use guess_host_triple::guess_host_triple; use std::collections::HashMap; @@ -15,11 +10,11 @@ impl JobBuilder { pub fn from_request(job_requests: impl OneOrVec) -> CombinedResult { let job_requests = job_requests.into_vec(); let mut prepared: Vec = vec![]; - let mut result = CombinedResult::new(); + let mut result = CombinedResult::::new(); for req in job_requests { let job_meta = JobCache::get(&req.job_id); if job_meta.is_none() { - result.err(UError::NoJob(req.job_id)); + result.err(UError::NoJob(req.job_id).into_bt()); continue; } let job_meta = job_meta.unwrap(); @@ -29,12 +24,12 @@ impl JobBuilder { JobType::Shell => { let meta = JobCache::get(&req.job_id).ok_or(UError::NoJob(req.job_id))?; let curr_platform = guess_host_triple().unwrap_or("unknown").to_string(); - //extend platform checking (partial check) + //TODO: extend platform checking (partial check) if meta.platform != curr_platform { return Err(UError::InsuitablePlatform( meta.platform.clone(), curr_platform, - )); + ).into()); } let job = AssignedJob::new(req.job_id, Some(&req)); prepared.push(Box::pin(job.run())) @@ -211,7 +206,7 @@ mod tests { if let Some(p) = payload { job = job.with_payload(p); } - let job = job.build().unwrap(); + let job = job.build().unwrap(); let job_result = JobBuilder::from_meta(job).unwrap_one().wait_one().await; let result = unwrap_enum!(job_result, Reportable::Assigned); let result = result.to_string_result(); @@ -298,7 +293,7 @@ mod tests { job = job.with_payload(p); } let err = job.build().unwrap_err(); - let err_msg = unwrap_enum!(err, UError::JobArgsError); + let err_msg = unwrap_enum!(err.err, UError::JobArgsError); assert!(err_msg.contains(err_str)); Ok(()) } diff --git a/lib/u_lib/src/cache.rs b/lib/u_lib/src/cache.rs index a486906..3af1724 100644 --- a/lib/u_lib/src/cache.rs +++ b/lib/u_lib/src/cache.rs @@ -1,4 +1,5 @@ use crate::models::JobMeta; +use lazy_static::lazy_static; use std::{ collections::HashMap, ops::Deref, diff --git a/lib/u_lib/src/config.rs b/lib/u_lib/src/config.rs index 88fc795..1fd815c 100644 --- a/lib/u_lib/src/config.rs +++ b/lib/u_lib/src/config.rs @@ -1,3 +1,4 @@ +use lazy_static::lazy_static; use uuid::Uuid; pub const MASTER_SERVER: &str = "127.0.0.1"; //Ipv4Addr::new(3,9,16,40) diff --git a/lib/u_lib/src/datatypes.rs b/lib/u_lib/src/datatypes.rs index d980734..983747f 100644 --- a/lib/u_lib/src/datatypes.rs +++ b/lib/u_lib/src/datatypes.rs @@ -1,4 +1,4 @@ -use crate::UError; +use crate::UErrorBt; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] @@ -6,5 +6,5 @@ use serde::{Deserialize, Serialize}; #[serde(tag = "status", content = "data")] pub enum DataResult { Ok(M), - Err(UError), + Err(UErrorBt), } diff --git a/lib/u_lib/src/errors/chan.rs b/lib/u_lib/src/errors/chan.rs new file mode 100644 index 0000000..cf78fd0 --- /dev/null +++ b/lib/u_lib/src/errors/chan.rs @@ -0,0 +1,28 @@ +use crate::UErrorBt; +use crossbeam::channel::{self, Receiver, Sender}; +use once_cell::sync::OnceCell; + +type ChanError = UErrorBt; +static ERR_CHAN: OnceCell = OnceCell::new(); + +pub struct ErrChan { + tx: Sender, + rx: Receiver, +} + +impl ErrChan { + fn get() -> &'static Self { + ERR_CHAN.get_or_init(|| { + let (tx, rx) = channel::bounded(20); + Self { tx, rx } + }) + } + + pub fn send(msg: impl Into) { + Self::get().tx.send(msg.into()).unwrap() + } + + pub fn recv() -> ChanError { + Self::get().rx.recv().unwrap() + } +} diff --git a/lib/u_lib/src/errors/mod.rs b/lib/u_lib/src/errors/mod.rs new file mode 100644 index 0000000..c910a5a --- /dev/null +++ b/lib/u_lib/src/errors/mod.rs @@ -0,0 +1,5 @@ +mod chan; +mod variants; + +pub use chan::*; +pub use variants::*; diff --git a/lib/u_lib/src/errors.rs b/lib/u_lib/src/errors/variants.rs similarity index 55% rename from lib/u_lib/src/errors.rs rename to lib/u_lib/src/errors/variants.rs index 5f46197..2b2570d 100644 --- a/lib/u_lib/src/errors.rs +++ b/lib/u_lib/src/errors/variants.rs @@ -1,16 +1,39 @@ +use backtrace::Backtrace as CrateBacktrace; use diesel::result::Error as DslError; use reqwest::Error as ReqError; use serde::{Deserialize, Serialize}; +use std::fmt; use thiserror::Error; use uuid::Uuid; -pub type UResult = std::result::Result; +pub type UResult = std::result::Result; pub type ULocalResult = std::result::Result; +#[derive(Error, Debug, Serialize, Deserialize, Clone)] +pub struct UErrorBt { + pub err: UError, + pub backtrace: String, +} + +impl fmt::Display for UErrorBt { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}\nBACKTRACE: \n{:?}", self.err, self.backtrace) + } +} + +impl From for UErrorBt { + fn from(err: UError) -> UErrorBt { + UErrorBt { + err, + backtrace: format!("{:?}", CrateBacktrace::new()), + } + } +} + #[derive(Error, Debug, Serialize, Deserialize, Clone)] pub enum UError { - #[error("Error: {0}")] - Raw(String), + #[error("Runtime error: {0}")] + Runtime(String), #[error("Connection error: {0}. Body: {1}")] NetError(String, String), @@ -33,11 +56,26 @@ pub enum UError { #[error("Job {0} doesn't exist")] NoJob(Uuid), - #[error("Error opening {0}: {1}")] + #[error("Error while processing {0}: {1}")] FSError(String, String), #[error("Wrong auth token")] WrongToken, + + #[error("Panicked: {0}")] + Panic(String), +} + +impl UError { + pub fn into_bt(self) -> UErrorBt { + UErrorBt::from(self) + } +} + +impl From for UErrorBt { + fn from(e: ReqError) -> Self { + UError::from(e).into_bt() + } } impl From for UError { diff --git a/lib/u_lib/src/executor.rs b/lib/u_lib/src/executor.rs index 4155cf1..dc4bc79 100644 --- a/lib/u_lib/src/executor.rs +++ b/lib/u_lib/src/executor.rs @@ -55,12 +55,12 @@ impl Waiter { tx.send(fid).await.unwrap(); result }; - let handle = JoinInfo { + let handler = JoinInfo { handle: spawn(task_wrapper), completed: false, collectable, }; - FUT_RESULTS.lock().await.insert(fid, handle); + FUT_RESULTS.lock().await.insert(fid, handler); } self } @@ -147,9 +147,9 @@ mod tests { }) }; assert_eq!(0, *val.lock().await); - spawn(async {}).await.ok(); + spawn(async {}).await.unwrap(); assert_eq!(5, *val.lock().await); - t.await.ok(); + t.await.unwrap(); assert_eq!(5, *val.lock().await); } } diff --git a/lib/u_lib/src/lib.rs b/lib/u_lib/src/lib.rs index 55f4e1e..c614b47 100644 --- a/lib/u_lib/src/lib.rs +++ b/lib/u_lib/src/lib.rs @@ -11,16 +11,13 @@ pub mod models; pub mod utils; pub use config::UID; -pub use errors::{UError, ULocalError, ULocalResult, UResult}; +pub use errors::{UError, UErrorBt, ULocalError, ULocalResult, UResult}; pub mod schema_exports { pub use crate::models::{Agentstate, Jobstate, Jobtype}; pub use diesel::sql_types::*; } -#[macro_use] -extern crate lazy_static; - #[macro_use] extern crate diesel; diff --git a/lib/u_lib/src/models/errors.rs b/lib/u_lib/src/models/error.rs similarity index 100% rename from lib/u_lib/src/models/errors.rs rename to lib/u_lib/src/models/error.rs diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index a4a0d35..a75d88c 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -1,6 +1,7 @@ use super::JobState; use crate::{ cache::JobCache, + errors::UError, messaging::Reportable, models::{schema::*, JobOutput}, utils::{systime_to_string, TempFile}, @@ -78,16 +79,22 @@ impl AssignedJob { pub async fn run(mut self) -> Reportable { let (argv, _payload) = { let meta = JobCache::get(&self.job_id).unwrap(); - let extracted_payload = meta - .payload - .as_ref() - .and_then(|p| TempFile::write_exec(p).ok()); - let argv = if let Some(ref p) = &extracted_payload { - meta.argv.replace("{}", &p.get_path()) + if let Some(ref payload) = meta.payload { + let extracted_payload = match TempFile::write_exec(payload) { + Ok(p) => p, + Err(e) => { + return Reportable::Error( + UError::Runtime(e.to_string()).into_bt().to_string(), + ) + } + }; + ( + meta.argv.replace("{}", &extracted_payload.get_path()), + Some(extracted_payload), + ) } else { - meta.argv.clone() - }; - (argv, extracted_payload) + (meta.argv.clone(), None) + } }; let mut split_cmd = shlex::split(&argv).unwrap().into_iter(); let cmd = split_cmd.nth(0).unwrap(); diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 4cb07cc..79bce50 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -1,9 +1,10 @@ use super::JobType; -use crate::{models::schema::*, UError, UResult}; +use crate::{models::schema::*, utils::Stripped, UError, UResult}; use diesel::{Identifiable, Insertable, Queryable}; use guess_host_triple::guess_host_triple; use serde::{Deserialize, Serialize}; use std::fmt; +use std::str::from_utf8; use uuid::Uuid; #[derive(Serialize, Deserialize, Clone, Debug, Queryable, Identifiable, Insertable)] @@ -41,20 +42,12 @@ impl fmt::Display for JobMeta { out += &format!("\nPlatform: {}", self.platform); if let Some(ref payload) = self.payload { if self.exec_type == JobType::Shell { - let (pld_len, large) = { - let pl = payload.len(); - if pl > 20 { - (20, true) - } else { - (pl, false) - } + let payload = if let Ok(str_payload) = from_utf8(payload) { + Stripped(&str_payload).to_string() + } else { + Stripped(&payload).to_string() }; - let pld_beginning = &payload[..pld_len]; - out += &format!( - "\nPayload: {}{}", - String::from_utf8_lossy(pld_beginning), - if large { "" } else { " <...>" } - ); + out += &format!("\nPayload: {}", payload); } } write!(f, "{}", out) @@ -118,14 +111,15 @@ impl JobMetaBuilder { shlex::split(&inner.argv).ok_or(UError::JobArgsError("Shlex failed".into()))?; let empty_err = UError::JobArgsError("Empty argv".into()); if argv_parts.get(0).ok_or(empty_err.clone())?.len() == 0 { - return Err(empty_err); + return Err(empty_err.into()); } match inner.payload.as_ref() { Some(_) => { if !inner.argv.contains("{}") { return Err(UError::JobArgsError( "Argv contains no executable placeholder".into(), - )); + ) + .into()); } else { () } @@ -135,7 +129,8 @@ impl JobMetaBuilder { return Err(UError::JobArgsError( "No payload provided, but argv contains executable placeholder" .into(), - )); + ) + .into()); } else { () } diff --git a/lib/u_lib/src/models/mod.rs b/lib/u_lib/src/models/mod.rs index bd44781..3178567 100644 --- a/lib/u_lib/src/models/mod.rs +++ b/lib/u_lib/src/models/mod.rs @@ -1,6 +1,6 @@ mod agent; -mod errors; +mod error; mod jobs; pub mod schema; -pub use crate::models::{agent::*, errors::*, jobs::*}; +pub use crate::models::{agent::*, error::*, jobs::*}; diff --git a/lib/u_lib/src/utils/combined_result.rs b/lib/u_lib/src/utils/combined_result.rs index 2baf35f..aa0968d 100644 --- a/lib/u_lib/src/utils/combined_result.rs +++ b/lib/u_lib/src/utils/combined_result.rs @@ -1,7 +1,7 @@ use crate::utils::OneOrVec; -use crate::UError; +use crate::UErrorBt; -pub struct CombinedResult { +pub struct CombinedResult { ok: Vec, err: Vec, } diff --git a/lib/u_lib/src/utils/hexlify.rs b/lib/u_lib/src/utils/fmt/hexlify.rs similarity index 52% rename from lib/u_lib/src/utils/hexlify.rs rename to lib/u_lib/src/utils/fmt/hexlify.rs index d3a0d14..69bda69 100644 --- a/lib/u_lib/src/utils/hexlify.rs +++ b/lib/u_lib/src/utils/fmt/hexlify.rs @@ -10,3 +10,15 @@ impl<'a> fmt::LowerHex for Hexlify<'a> { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hexlify() { + let data = b"\x5a\x6b\x23\x4f\xa3\x7f\x9e"; + let result = "5a6b234fa37f9e"; + assert_eq!(format!("{:x}", Hexlify(data)), result); + } +} diff --git a/lib/u_lib/src/utils/fmt/mod.rs b/lib/u_lib/src/utils/fmt/mod.rs new file mode 100644 index 0000000..aa14890 --- /dev/null +++ b/lib/u_lib/src/utils/fmt/mod.rs @@ -0,0 +1,5 @@ +mod hexlify; +mod stripped; + +pub use hexlify::*; +pub use stripped::*; diff --git a/lib/u_lib/src/utils/fmt/stripped.rs b/lib/u_lib/src/utils/fmt/stripped.rs new file mode 100644 index 0000000..8681a97 --- /dev/null +++ b/lib/u_lib/src/utils/fmt/stripped.rs @@ -0,0 +1,80 @@ +use std::fmt; +use std::iter::Iterator; +use std::slice::Iter as SliceIter; +use std::str::Chars; + +const MAX_DATA_LEN: usize = 200; + +pub trait Strippable { + //TODO: waiting for stabilizing GATs + type Item: fmt::Display; + type TypeIter: Iterator; + + fn length(&self) -> usize; + fn iterator(&self) -> Self::TypeIter; +} + +impl<'a> Strippable for &'a str { + type Item = char; + type TypeIter = Chars<'a>; + + fn length(&self) -> usize { + self.len() + } + + fn iterator(&self) -> Self::TypeIter { + self.chars() + } +} + +impl<'a> Strippable for &'a Vec { + type Item = &'a u8; + type TypeIter = SliceIter<'a, u8>; + + fn length(&self) -> usize { + self.len() + } + + fn iterator(&self) -> Self::TypeIter { + self.iter() + } +} +pub struct Stripped<'inner, Inner: Strippable + 'inner>(pub &'inner Inner); + +impl<'inner, Inner: Strippable + 'inner> Stripped<'inner, Inner> { + fn iter(&self) -> Inner::TypeIter { + self.0.iterator() + } + + fn placeholder(&self) -> &'static str { + if self.0.length() >= MAX_DATA_LEN { + " <...>" + } else { + "" + } + } +} + +impl<'inner, Inner: Strippable + 'inner> fmt::Display for Stripped<'inner, Inner> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let placeholder = self.placeholder(); + for c in self.iter().take(MAX_DATA_LEN - placeholder.len()) { + write!(f, "{}", c)?; + } + write!(f, "{}", placeholder) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::*; + + #[rstest] + #[case("abc", 3)] + #[case("abcde".repeat(50), MAX_DATA_LEN)] + fn test_strip(#[case] input: impl Into, #[case] result_len: usize) { + let s = input.into(); + assert_eq!(Stripped(&s.as_str()).to_string().len(), result_len); + } +} diff --git a/lib/u_lib/src/utils/misc.rs b/lib/u_lib/src/utils/misc.rs index 375a6ad..3025f24 100644 --- a/lib/u_lib/src/utils/misc.rs +++ b/lib/u_lib/src/utils/misc.rs @@ -22,7 +22,7 @@ impl OneOrVec for Vec { #[macro_export] macro_rules! unwrap_enum { - ($src:ident, $t:path) => { + ($src:expr, $t:path) => { if let $t(result) = $src { result } else { diff --git a/lib/u_lib/src/utils/mod.rs b/lib/u_lib/src/utils/mod.rs index 0082b16..98853c2 100644 --- a/lib/u_lib/src/utils/mod.rs +++ b/lib/u_lib/src/utils/mod.rs @@ -1,6 +1,6 @@ mod combined_result; mod conv; -mod hexlify; +mod fmt; mod misc; mod storage; mod tempfile; @@ -8,7 +8,7 @@ mod vec_display; pub use combined_result::*; pub use conv::*; -pub use hexlify::*; +pub use fmt::*; pub use misc::*; pub use storage::*; pub use tempfile::*; diff --git a/lib/u_lib/src/utils/tempfile.rs b/lib/u_lib/src/utils/tempfile.rs index e76baa8..209ea46 100644 --- a/lib/u_lib/src/utils/tempfile.rs +++ b/lib/u_lib/src/utils/tempfile.rs @@ -1,3 +1,4 @@ +use crate::{UError, UResult}; use std::{env::temp_dir, fs, ops::Drop, os::unix::fs::PermissionsExt, path::PathBuf}; use uuid::Uuid; @@ -17,16 +18,17 @@ impl TempFile { Self { path } } - pub fn write_all(&self, data: &[u8]) -> Result<(), String> { - fs::write(&self.path, data).map_err(|e| e.to_string()) + pub fn write_all(&self, data: &[u8]) -> UResult<()> { + fs::write(&self.path, data).map_err(|e| UError::FSError(self.get_path(), e.to_string()))?; + Ok(()) } - pub fn write_exec(data: &[u8]) -> Result { + pub fn write_exec(data: &[u8]) -> UResult { let this = Self::new(); let path = this.get_path(); - this.write_all(data).map_err(|e| (path.clone(), e))?; + this.write_all(data)?; let perms = fs::Permissions::from_mode(0o555); - fs::set_permissions(&path, perms).map_err(|e| (path, e.to_string()))?; + fs::set_permissions(&path, perms).map_err(|e| UError::FSError(path, e.to_string()))?; Ok(this) } } diff --git a/scripts/cargo_musl.sh b/scripts/cargo_musl.sh index 0a8b3aa..e4149ef 100755 --- a/scripts/cargo_musl.sh +++ b/scripts/cargo_musl.sh @@ -4,6 +4,7 @@ source $(dirname $0)/rootdir.sh #set ROOTDIR ARGS=$@ docker run \ --env-file $ROOTDIR/.env \ + --env-file $ROOTDIR/.env.private \ -v $ROOTDIR:/volume \ -v cargo-cache:/root/.cargo/registry \ -w /volume \ From 745dcb7ff8725c5abbc01130f3bdd2f922aff8dd Mon Sep 17 00:00:00 2001 From: plazmoid Date: Mon, 23 Aug 2021 22:27:41 +0500 Subject: [PATCH 04/30] get rid of in-docker cargo --- .cargo/config.toml | 3 + .gitignore | 3 +- Makefile.toml | 19 ++-- .../tests_runner.Dockerfile | 0 .../integration-tests}/u_agent.Dockerfile | 0 .../integration-tests}/u_db.Dockerfile | 0 .../integration-tests}/u_server.Dockerfile | 0 images/musl-libs.Dockerfile | 96 +++++++++++++++++++ integration/docker.py | 2 +- scripts/build_musl_libs.sh | 15 +++ scripts/cargo_musl.sh | 13 --- 11 files changed, 129 insertions(+), 22 deletions(-) create mode 100644 .cargo/config.toml rename {integration/images => images/integration-tests}/tests_runner.Dockerfile (100%) rename {integration/images => images/integration-tests}/u_agent.Dockerfile (100%) rename {integration/images => images/integration-tests}/u_db.Dockerfile (100%) rename {integration/images => images/integration-tests}/u_server.Dockerfile (100%) create mode 100644 images/musl-libs.Dockerfile create mode 100755 scripts/build_musl_libs.sh delete mode 100755 scripts/cargo_musl.sh diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..bf72e8c --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,3 @@ +[build] +rustflags = ["-L", "/home/ortem/src/rust/unki/static/lib"] +target = "x86_64-unknown-linux-musl" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 67f21e2..19d820c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ certs/* *.log echoer .env.private -*.lock \ No newline at end of file +*.lock +static/ \ No newline at end of file diff --git a/Makefile.toml b/Makefile.toml index 87cffed..7e2d27f 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -2,24 +2,29 @@ default_to_workspace = false [env] -CARGO = "./scripts/cargo_musl.sh" +CARGO = "cargo" +PREFIX = "${CARGO_MAKE_WORKING_DIRECTORY}/static" +PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true" +PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = "${PREFIX}/bin/pg_config" +OPENSSL_STATIC = "true" +OPENSSL_DIR = "${PREFIX}" -[tasks.build_cargo_image] -script = "docker build -t unki/musllibs ./muslrust" +[tasks.build_static_libs] +script = "./scripts/build_musl_libs.sh" [tasks.clean] command = "${CARGO}" args = ["clean"] [tasks.debug] -dependencies = ["build_cargo_image"] +dependencies = ["build_static_libs"] command = "${CARGO}" -args = ["build"] +args = ["build", "${@}"] [tasks.release] -dependencies = ["build_cargo_image"] +dependencies = ["build_static_libs"] command = "${CARGO}" -args = ["build", "--release"] +args = ["build", "--release", "${@}"] [tasks.run] script = ''' diff --git a/integration/images/tests_runner.Dockerfile b/images/integration-tests/tests_runner.Dockerfile similarity index 100% rename from integration/images/tests_runner.Dockerfile rename to images/integration-tests/tests_runner.Dockerfile diff --git a/integration/images/u_agent.Dockerfile b/images/integration-tests/u_agent.Dockerfile similarity index 100% rename from integration/images/u_agent.Dockerfile rename to images/integration-tests/u_agent.Dockerfile diff --git a/integration/images/u_db.Dockerfile b/images/integration-tests/u_db.Dockerfile similarity index 100% rename from integration/images/u_db.Dockerfile rename to images/integration-tests/u_db.Dockerfile diff --git a/integration/images/u_server.Dockerfile b/images/integration-tests/u_server.Dockerfile similarity index 100% rename from integration/images/u_server.Dockerfile rename to images/integration-tests/u_server.Dockerfile diff --git a/images/musl-libs.Dockerfile b/images/musl-libs.Dockerfile new file mode 100644 index 0000000..2441b80 --- /dev/null +++ b/images/musl-libs.Dockerfile @@ -0,0 +1,96 @@ +FROM ubuntu:xenial +LABEL maintainer="Eirik Albrigtsen " + +# Required packages: +# - musl-dev, musl-tools - the musl toolchain +# - curl, g++, make, pkgconf, cmake - for fetching and building third party libs +# - ca-certificates - openssl + curl + peer verification of downloads +# - xutils-dev - for openssl makedepend +# - libssl-dev and libpq-dev - for dynamic linking during diesel_codegen build process +# - git - cargo builds in user projects +# - linux-headers-amd64 - needed for building openssl 1.1 (stretch only) +# - file - needed by rustup.sh install +# - automake autoconf libtool - support crates building C deps as part cargo build +# recently removed: +# cmake (not used), nano, zlib1g-dev +RUN apt-get update && apt-get install -y \ + musl-dev \ + musl-tools \ + git \ + file \ + openssh-client \ + make \ + g++ \ + curl \ + pkgconf \ + ca-certificates \ + xutils-dev \ + libssl-dev \ + libpq-dev \ + automake \ + autoconf \ + libtool \ + python3 \ + --no-install-recommends && \ + rm -rf /var/lib/apt/lists/* + +# Convenience list of versions and variables for compilation later on +# This helps continuing manually if anything breaks. +ENV SSL_VER="1.0.2u" \ + CURL_VER="7.77.0" \ + ZLIB_VER="1.2.11" \ + PQ_VER="11.12" \ + SQLITE_VER="3350500" \ + CC=musl-gcc \ + PREFIX=/musl \ + PATH=/usr/local/bin:/root/.cargo/bin:$PATH \ + PKG_CONFIG_PATH=/usr/local/lib/pkgconfig \ + LD_LIBRARY_PATH=$PREFIX + +# Set up a prefix for musl build libraries, make the linker's job of finding them easier +# Primarily for the benefit of postgres. +# Lastly, link some linux-headers for openssl 1.1 (not used herein) +RUN mkdir $PREFIX && \ + echo "$PREFIX/lib" >> /etc/ld-musl-x86_64.path && \ + ln -s /usr/include/x86_64-linux-gnu/asm /usr/include/x86_64-linux-musl/asm && \ + ln -s /usr/include/asm-generic /usr/include/x86_64-linux-musl/asm-generic && \ + ln -s /usr/include/linux /usr/include/x86_64-linux-musl/linux + +# Build zlib (used in openssl and pq) +RUN curl -sSL https://zlib.net/zlib-$ZLIB_VER.tar.gz | tar xz && \ + cd zlib-$ZLIB_VER && \ + CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure --static --prefix=$PREFIX && \ + make -j$(nproc) && make install && \ + cd .. && rm -rf zlib-$ZLIB_VER + +# Build openssl (used in curl and pq) +# Would like to use zlib here, but can't seem to get it to work properly +# TODO: fix so that it works +RUN curl -sSL https://www.openssl.org/source/old/1.0.2/openssl-$SSL_VER.tar.gz | tar xz && \ + cd openssl-$SSL_VER && \ + ./Configure no-zlib no-shared -fPIC --prefix=$PREFIX --openssldir=$PREFIX/ssl linux-x86_64 && \ + env C_INCLUDE_PATH=$PREFIX/include make depend 2> /dev/null && \ + make -j$(nproc) && make install && \ + cd .. && rm -rf openssl-$SSL_VER + +# Build curl (needs with-zlib and all this stuff to allow https) +# curl_LDFLAGS needed on stretch to avoid fPIC errors - though not sure from what +RUN curl -sSL https://curl.se/download/curl-$CURL_VER.tar.gz | tar xz && \ + cd curl-$CURL_VER && \ + CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \ + --enable-shared=no --with-zlib --enable-static=ssl --enable-optimize --prefix=$PREFIX \ + --with-ca-path=/etc/ssl/certs/ --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt --without-ca-fallback \ + --with-openssl && \ + make -j$(nproc) curl_LDFLAGS="-all-static" && make install && \ + cd .. && rm -rf curl-$CURL_VER + +# Build libpq +RUN curl -sSL https://ftp.postgresql.org/pub/source/v$PQ_VER/postgresql-$PQ_VER.tar.gz | tar xz && \ + cd postgresql-$PQ_VER && \ + CC="musl-gcc -fPIE -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \ + --without-readline \ + --with-openssl \ + --prefix=$PREFIX --host=x86_64-unknown-linux-musl && \ + cd src/interfaces/libpq make -s -j$(nproc) all-static-lib && make -s install install-lib-static && \ + cd ../../bin/pg_config && make -j $(nproc) && make install && \ + cd .. && rm -rf postgresql-$PQ_VER diff --git a/integration/docker.py b/integration/docker.py index 68abd69..0e1c163 100644 --- a/integration/docker.py +++ b/integration/docker.py @@ -1,7 +1,7 @@ import subprocess from utils import * -BASE_IMAGE_DIR = 'images' +BASE_IMAGE_DIR = '../images/integration-tests' DOCKERFILES = { 'u_agent': { diff --git a/scripts/build_musl_libs.sh b/scripts/build_musl_libs.sh new file mode 100755 index 0000000..1cf612c --- /dev/null +++ b/scripts/build_musl_libs.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -ex +source $(dirname $0)/rootdir.sh #set ROOTDIR +ARGS=$@ +STATIC_LIBS=static +DOCKER_EXCHG=/musl-share +IMAGE=unki/musllibs +mkdir -p $STATIC_LIBS +cd $ROOTDIR/images && docker build -t $IMAGE . -f musl-libs.Dockerfile +docker run \ + -v $ROOTDIR/$STATIC_LIBS:$DOCKER_EXCHG \ + -w /volume \ + -it \ + $IMAGE \ + bash -c "[[ \$(ls -A $DOCKER_EXCHG) ]] || cp /musl/. $DOCKER_EXCHG -r" diff --git a/scripts/cargo_musl.sh b/scripts/cargo_musl.sh deleted file mode 100755 index e4149ef..0000000 --- a/scripts/cargo_musl.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -ex -source $(dirname $0)/rootdir.sh #set ROOTDIR -ARGS=$@ -docker run \ - --env-file $ROOTDIR/.env \ - --env-file $ROOTDIR/.env.private \ - -v $ROOTDIR:/volume \ - -v cargo-cache:/root/.cargo/registry \ - -w /volume \ - -it \ - unki/musllibs \ - bash -c "umask 0000; cargo $ARGS" From ac20f1b34397c91e86197a310071f17f4da40bc9 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Tue, 24 Aug 2021 23:59:07 +0500 Subject: [PATCH 05/30] fixed rust-analyzer fail on cargo check, added & improved tests --- .cargo/config.toml | 3 +-- Makefile.toml | 7 +++--- integration/tests/behaviour.rs | 9 ++++---- integration/tests/helpers/panel.rs | 18 ++++++++-------- lib/u_lib/src/utils/tempfile.rs | 34 +++++++++++++++++++++++++++++- 5 files changed, 51 insertions(+), 20 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index bf72e8c..21748f1 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,2 @@ [build] -rustflags = ["-L", "/home/ortem/src/rust/unki/static/lib"] -target = "x86_64-unknown-linux-musl" \ No newline at end of file +rustflags = ["-L", "/home/ortem/src/rust/unki/static/lib"] \ No newline at end of file diff --git a/Makefile.toml b/Makefile.toml index 7e2d27f..256e798 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -2,6 +2,7 @@ default_to_workspace = false [env] +TARGET = "x86_64-unknown-linux-musl" CARGO = "cargo" PREFIX = "${CARGO_MAKE_WORKING_DIRECTORY}/static" PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true" @@ -19,12 +20,12 @@ args = ["clean"] [tasks.debug] dependencies = ["build_static_libs"] command = "${CARGO}" -args = ["build", "${@}"] +args = ["build", "--target", "${TARGET}","${@}"] [tasks.release] dependencies = ["build_static_libs"] command = "${CARGO}" -args = ["build", "--release", "${@}"] +args = ["build", "--target", "${TARGET}", "--release", "${@}"] [tasks.run] script = ''' @@ -34,7 +35,7 @@ exit 1 [tasks.unit] command = "${CARGO}" -args = ["test", "--lib", "${@}"] +args = ["test", "--target", "${TARGET}", "--lib", "--", "${@}"] [tasks.integration] script = ''' diff --git a/integration/tests/behaviour.rs b/integration/tests/behaviour.rs index 616c7a9..38e88f2 100644 --- a/integration/tests/behaviour.rs +++ b/integration/tests/behaviour.rs @@ -5,7 +5,7 @@ use rstest::rstest; use std::error::Error; use std::time::Duration; use tokio::time::sleep; -use u_lib::{messaging::Empty, models::*}; +use u_lib::models::*; use uuid::Uuid; type TestResult = Result>; @@ -17,9 +17,8 @@ async fn test_registration(#[future] register_agent: RegisteredAgent) -> TestRes let agents: Vec = Panel::check_output("agents list"); let found = agents.iter().find(|v| v.id == agent.uid); assert!(found.is_some()); - //teardown - Panel::check_status::(&format!("agents delete {}", agent.uid)); - Ok(()) //TODO: ______^^^^^ REMOV + Panel::check_status(format!("agents delete {}", agent.uid)); + Ok(()) } #[tokio::test] @@ -29,7 +28,7 @@ async fn test_setup_tasks() -> TestResult { let agent_uid = agents[0].id; let job_alias = "passwd_contents"; let cmd = format!("jobs add --alias {} 'cat /etc/passwd'", job_alias); - Panel::check_status::(&cmd); + Panel::check_status(cmd); let cmd = format!("jobmap add {} {}", agent_uid, job_alias); let assigned_uids: Vec = Panel::check_output(cmd); for _ in 0..3 { diff --git a/integration/tests/helpers/panel.rs b/integration/tests/helpers/panel.rs index 2c61245..713c60a 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration/tests/helpers/panel.rs @@ -1,8 +1,8 @@ use serde::de::DeserializeOwned; -use serde_json::from_slice; +use serde_json::{from_slice, Value}; use shlex::split; use std::process::{Command, Output}; -use u_lib::{datatypes::DataResult, messaging::AsMsg, utils::VecDisplay}; +use u_lib::{datatypes::DataResult, utils::VecDisplay}; const PANEL_BINARY: &str = "/u_panel"; @@ -19,8 +19,8 @@ impl Panel { .unwrap() } - pub fn output_argv(args: &[&str]) -> PanelResult { - let result = Self::run(args); + pub fn output_argv(argv: &[&str]) -> PanelResult { + let result = Self::run(argv); from_slice(&result.stdout).map_err(|e| { eprintln!( "Failed to decode panel response: '{}'", @@ -30,7 +30,7 @@ impl Panel { }) } - pub fn output(args: impl Into) -> PanelResult { + pub fn output(args: impl Into) -> PanelResult { let splitted = split(args.into().as_ref()).unwrap(); Self::output_argv( splitted @@ -41,19 +41,19 @@ impl Panel { ) } - fn status_is_ok(data: PanelResult) -> T { + fn status_is_ok(data: PanelResult) -> T { match data.unwrap() { DataResult::Ok(r) => r, DataResult::Err(e) => panic!("Panel failed: {}", e), } } - pub fn check_status<'s, T: AsMsg + DeserializeOwned>(args: &'s str) { - let result: PanelResult = Self::output(args); + pub fn check_status(args: impl Into) { + let result: PanelResult = Self::output(args); Self::status_is_ok(result); } - pub fn check_output(args: impl Into) -> T { + pub fn check_output(args: impl Into) -> T { let result = Self::output(args); Self::status_is_ok(result) } diff --git a/lib/u_lib/src/utils/tempfile.rs b/lib/u_lib/src/utils/tempfile.rs index 209ea46..a5a510e 100644 --- a/lib/u_lib/src/utils/tempfile.rs +++ b/lib/u_lib/src/utils/tempfile.rs @@ -26,6 +26,7 @@ impl TempFile { pub fn write_exec(data: &[u8]) -> UResult { let this = Self::new(); let path = this.get_path(); + dbg!(&path); this.write_all(data)?; let perms = fs::Permissions::from_mode(0o555); fs::set_permissions(&path, perms).map_err(|e| UError::FSError(path, e.to_string()))?; @@ -35,6 +36,37 @@ impl TempFile { impl Drop for TempFile { fn drop(&mut self) { - fs::remove_file(&self.path).ok(); + fs::remove_file(&self.path).unwrap(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::utils::bytes_to_string; + use std::path::Path; + use std::process::Command; + + #[test] + fn test_file_is_not_busy() { + let binary = include_bytes!("../../tests/fixtures/echoer"); + for _ in 0..100 { + let executable = TempFile::write_exec(binary).unwrap(); + let path = executable.get_path(); + let result = Command::new(path).arg("qwe").output().unwrap(); + assert_eq!(bytes_to_string(result.stdout.as_ref()).trim(), "qwe"); + } + } + + #[test] + fn test_file_removed_after_dropping() { + let path; + { + let file = TempFile::new(); + file.write_all(b"asdqwe").unwrap(); + path = file.get_path(); + assert!(Path::new(&path).exists()) + } + assert!(!Path::new(&path).exists()) } } From 700154e311e32689c95e43554fc61a7d36199763 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sun, 3 Oct 2021 17:43:09 +0500 Subject: [PATCH 06/30] clippy refactorings --- Makefile.toml | 9 +- bin/u_agent/src/lib.rs | 6 +- bin/u_agent/src/main.rs | 1 - bin/u_panel/Cargo.toml | 3 +- bin/u_panel/src/argparse.rs | 137 +++++++++++++++++++++ bin/u_panel/src/main.rs | 140 +-------------------- bin/u_panel/src/server/app.rs | 23 ++++ bin/u_panel/src/server/mod.rs | 3 + bin/u_server/src/db.rs | 8 +- bin/u_server/src/filters.rs | 4 +- bin/u_server/src/handlers.rs | 14 +-- bin/u_server/src/u_server.rs | 1 + integration/get_docker_ip.sh | 4 + integration/tests/helpers/panel.rs | 14 ++- integration/tests/tests.rs | 16 +++ lib/u_api_proc_macro/src/lib.rs | 2 +- lib/u_lib/src/executor.rs | 2 +- lib/u_lib/src/messaging/base.rs | 5 +- lib/u_lib/src/models/agent.rs | 2 +- lib/u_lib/src/models/jobs/assigned.rs | 27 ++-- lib/u_lib/src/models/jobs/meta.rs | 16 +-- lib/u_lib/src/models/jobs/misc.rs | 149 ---------------------- lib/u_lib/src/utils/combined_result.rs | 2 +- lib/u_lib/src/utils/fmt/stripped.rs | 1 - lib/u_lib/src/utils/mod.rs | 2 + lib/u_lib/src/utils/proc_output.rs | 163 +++++++++++++++++++++++++ scripts/build_musl_libs.sh | 2 +- 27 files changed, 406 insertions(+), 350 deletions(-) create mode 100644 bin/u_panel/src/argparse.rs create mode 100644 bin/u_panel/src/server/app.rs create mode 100644 bin/u_panel/src/server/mod.rs create mode 100755 integration/get_docker_ip.sh create mode 100644 lib/u_lib/src/utils/proc_output.rs diff --git a/Makefile.toml b/Makefile.toml index 256e798..44d8f2d 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -17,15 +17,10 @@ script = "./scripts/build_musl_libs.sh" command = "${CARGO}" args = ["clean"] -[tasks.debug] +[tasks.build] dependencies = ["build_static_libs"] command = "${CARGO}" -args = ["build", "--target", "${TARGET}","${@}"] - -[tasks.release] -dependencies = ["build_static_libs"] -command = "${CARGO}" -args = ["build", "--target", "${TARGET}", "--release", "${@}"] +args = ["build", "--target", "${TARGET}", "${@}"] [tasks.run] script = ''' diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 946afc9..ad83999 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -27,7 +27,7 @@ use u_lib::{ const ITERATION_LATENCY: u64 = 5; pub async fn process_request(job_requests: Vec, client: &ClientHandler) { - if job_requests.len() > 0 { + if !job_requests.is_empty() { for jr in &job_requests { if !JobCache::contains(&jr.job_id) { debug!("Fetching job: {}", &jr.job_id); @@ -53,7 +53,7 @@ pub async fn process_request(job_requests: Vec, client: &ClientHand ); let mut builder = JobBuilder::from_request(job_requests); let errors = builder.pop_errors(); - if errors.len() > 0 { + if !errors.is_empty() { errors.into_iter().for_each(ErrChan::send) } builder.unwrap_one().spawn().await; @@ -86,7 +86,7 @@ async fn do_stuff(client: Arc) -> ! { Err(err) => ErrChan::send(err), } let result: Vec = pop_completed().await.into_iter().collect(); - if result.len() > 0 { + if !result.is_empty() { if let Err(err) = client.report(&result).await { ErrChan::send(err) } diff --git a/bin/u_agent/src/main.rs b/bin/u_agent/src/main.rs index 43166a8..1f3d50a 100644 --- a/bin/u_agent/src/main.rs +++ b/bin/u_agent/src/main.rs @@ -1,4 +1,3 @@ -use tokio; use u_agent::run_forever; #[tokio::main] diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 1f15786..1746585 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -7,7 +7,6 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process"] } structopt = "0.3.21" log = "^0.4" env_logger = "0.7.1" @@ -17,3 +16,5 @@ openssl = "*" u_lib = { version = "*", path = "../../lib/u_lib" } serde_json = "1.0.4" serde = { version = "1.0.114", features = ["derive"] } +actix-web = "3.3.2" +tokio = "1.11.0" diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs new file mode 100644 index 0000000..9267de4 --- /dev/null +++ b/bin/u_panel/src/argparse.rs @@ -0,0 +1,137 @@ +use std::env; +use std::fmt; +use structopt::StructOpt; +use u_lib::{ + api::ClientHandler, datatypes::DataResult, messaging::AsMsg, models::JobMeta, UError, UResult, +}; +use uuid::Uuid; + +#[derive(StructOpt, Debug)] +pub struct Args { + #[structopt(subcommand)] + cmd: Cmd, + #[structopt(long)] + json: bool, +} + +#[derive(StructOpt, Debug)] +enum Cmd { + Agents(LD), + Jobs(JobALD), + Jobmap(JobMapALD), + Server, +} + +#[derive(StructOpt, Debug)] +enum JobALD { + Add { + #[structopt(long, parse(try_from_str = parse_uuid))] + agent: Option, + + #[structopt(long)] + alias: String, + + #[structopt(subcommand)] + cmd: JobCmd, + }, + #[structopt(flatten)] + LD(LD), +} + +#[derive(StructOpt, Debug)] +enum JobCmd { + #[structopt(external_subcommand)] + Cmd(Vec), +} + +#[derive(StructOpt, Debug)] +enum JobMapALD { + Add { + #[structopt(parse(try_from_str = parse_uuid))] + agent_uid: Uuid, + + job_idents: Vec, + }, + List { + #[structopt(parse(try_from_str = parse_uuid))] + uid: Option, + }, + Delete { + #[structopt(parse(try_from_str = parse_uuid))] + uid: Uuid, + }, +} + +#[derive(StructOpt, Debug)] +enum LD { + List { + #[structopt(parse(try_from_str = parse_uuid))] + uid: Option, + }, + Delete { + #[structopt(parse(try_from_str = parse_uuid))] + uid: Uuid, + }, +} + +fn parse_uuid(src: &str) -> Result { + Uuid::parse_str(src).map_err(|e| e.to_string()) +} + +pub async fn process_cmd(args: Args) -> UResult<()> { + struct Printer { + json: bool, + } + + impl Printer { + pub fn print(&self, data: UResult) { + if self.json { + let data = match data { + Ok(r) => DataResult::Ok(r), + Err(e) => DataResult::Err(e), + }; + println!("{}", serde_json::to_string_pretty(&data).unwrap()); + } else { + match data { + Ok(r) => println!("{}", r), + Err(e) => eprintln!("Error: {}", e), + } + } + } + } + + let token = env::var("ADMIN_AUTH_TOKEN").map_err(|_| UError::WrongToken)?; + let cli_handler = ClientHandler::new(None).password(token); + let printer = Printer { json: args.json }; + match args.cmd { + Cmd::Agents(action) => match action { + LD::List { uid } => printer.print(cli_handler.get_agents(uid).await), + LD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await), + }, + Cmd::Jobs(action) => match action { + JobALD::Add { + cmd: JobCmd::Cmd(cmd), + alias, + .. + } => { + let job = JobMeta::builder() + .with_shell(cmd.join(" ")) + .with_alias(alias) + .build()?; + printer.print(cli_handler.upload_jobs(&[job]).await); + } + JobALD::LD(LD::List { uid }) => printer.print(cli_handler.get_jobs(uid).await), + JobALD::LD(LD::Delete { uid }) => printer.print(cli_handler.del(Some(uid)).await), + }, + Cmd::Jobmap(action) => match action { + JobMapALD::Add { + agent_uid, + job_idents, + } => printer.print(cli_handler.set_jobs(Some(agent_uid), &job_idents).await), + JobMapALD::List { uid } => printer.print(cli_handler.get_agent_jobs(uid).await), + JobMapALD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await), + }, + Cmd::Server => crate::server::serve().unwrap(), + } + Ok(()) +} diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index 39f0a7b..828a67f 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -1,140 +1,10 @@ -use std::env; -use std::fmt; +mod argparse; +mod server; + +use argparse::{process_cmd, Args}; use std::process; use structopt::StructOpt; -use u_lib::{ - api::ClientHandler, datatypes::DataResult, messaging::AsMsg, models::JobMeta, utils::init_env, - UError, UResult, -}; -use uuid::Uuid; - -#[derive(StructOpt, Debug)] -struct Args { - #[structopt(subcommand)] - cmd: Cmd, - #[structopt(long)] - json: bool, -} - -#[derive(StructOpt, Debug)] -enum Cmd { - Agents(LD), - Jobs(JobALD), - Jobmap(JobMapALD), -} - -#[derive(StructOpt, Debug)] -enum JobALD { - Add { - #[structopt(long, parse(try_from_str = parse_uuid))] - agent: Option, - - #[structopt(long)] - alias: String, - - #[structopt(subcommand)] - cmd: JobCmd, - }, - #[structopt(flatten)] - LD(LD), -} - -#[derive(StructOpt, Debug)] -enum JobCmd { - #[structopt(external_subcommand)] - Cmd(Vec), -} - -#[derive(StructOpt, Debug)] -enum JobMapALD { - Add { - #[structopt(parse(try_from_str = parse_uuid))] - agent_uid: Uuid, - - job_idents: Vec, - }, - List { - #[structopt(parse(try_from_str = parse_uuid))] - uid: Option, - }, - Delete { - #[structopt(parse(try_from_str = parse_uuid))] - uid: Uuid, - }, -} - -#[derive(StructOpt, Debug)] -enum LD { - List { - #[structopt(parse(try_from_str = parse_uuid))] - uid: Option, - }, - Delete { - #[structopt(parse(try_from_str = parse_uuid))] - uid: Uuid, - }, -} - -fn parse_uuid(src: &str) -> Result { - Uuid::parse_str(src).map_err(|e| e.to_string()) -} - -async fn process_cmd(args: Args) -> UResult<()> { - struct Printer { - json: bool, - } - - impl Printer { - pub fn print(&self, data: UResult) { - if self.json { - let data = match data { - Ok(r) => DataResult::Ok(r), - Err(e) => DataResult::Err(e), - }; - println!("{}", serde_json::to_string_pretty(&data).unwrap()); - } else { - match data { - Ok(r) => println!("{}", r), - Err(e) => eprintln!("Error: {}", e), - } - } - } - } - - let token = env::var("ADMIN_AUTH_TOKEN").map_err(|_| UError::WrongToken)?; - let cli_handler = ClientHandler::new(None).password(token); - let printer = Printer { json: args.json }; - match args.cmd { - Cmd::Agents(action) => match action { - LD::List { uid } => printer.print(cli_handler.get_agents(uid).await), - LD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await), - }, - Cmd::Jobs(action) => match action { - JobALD::Add { - cmd: JobCmd::Cmd(cmd), - alias, - agent: _agent, - } => { - let job = JobMeta::builder() - .with_shell(cmd.join(" ")) - .with_alias(alias) - .build()?; - printer.print(cli_handler.upload_jobs(&[job]).await); - } - JobALD::LD(LD::List { uid }) => printer.print(cli_handler.get_jobs(uid).await), - JobALD::LD(LD::Delete { uid }) => printer.print(cli_handler.del(Some(uid)).await), - }, - Cmd::Jobmap(action) => match action { - JobMapALD::Add { - agent_uid, - job_idents, - } => printer.print(cli_handler.set_jobs(Some(agent_uid), &job_idents).await), - JobMapALD::List { uid } => printer.print(cli_handler.get_agent_jobs(uid).await), - JobMapALD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await), - }, - } - Ok(()) -} +use u_lib::utils::init_env; #[tokio::main] async fn main() { diff --git a/bin/u_panel/src/server/app.rs b/bin/u_panel/src/server/app.rs new file mode 100644 index 0000000..d7c5117 --- /dev/null +++ b/bin/u_panel/src/server/app.rs @@ -0,0 +1,23 @@ +/* +Tabs: Agents, Tasks, Summary +every tab has list page and item page with more info/actions + +Agents: + +| id | alias | ..see struct | tasks done +| stripped | alias | ... | clickable number of assigned jobs + +almost all fields are editable, rows are deletable + + +*/ + +use actix_web::{web, App, HttpResponse, HttpServer}; + +#[actix_web::main] +pub async fn serve() -> std::io::Result<()> { + let addr = "127.0.0.1:8080"; + let app = || App::new().route("/", web::get().to(|| HttpResponse::Ok().body("ok"))); + println!("Serving at http://{}", addr); + HttpServer::new(app).bind(addr)?.run().await +} diff --git a/bin/u_panel/src/server/mod.rs b/bin/u_panel/src/server/mod.rs new file mode 100644 index 0000000..a4fad6b --- /dev/null +++ b/bin/u_panel/src/server/mod.rs @@ -0,0 +1,3 @@ +mod app; + +pub use app::serve; diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index d5db645..c2a2af1 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -151,7 +151,7 @@ impl UDB { } }) .collect::>(); - if not_found_jobs.len() > 0 { + if !not_found_jobs.is_empty() { return Err(ULocalError::NotFound(not_found_jobs.join(", "))); } let job_requests = job_uids @@ -172,7 +172,7 @@ impl UDB { Ok(assigned_uids) } - pub fn del_jobs(&self, uids: &Vec) -> ULocalResult { + pub fn del_jobs(&self, uids: &[Uuid]) -> ULocalResult { use schema::jobs; let mut affected = 0; for &uid in uids { @@ -184,7 +184,7 @@ impl UDB { Ok(affected) } - pub fn del_results(&self, uids: &Vec) -> ULocalResult { + pub fn del_results(&self, uids: &[Uuid]) -> ULocalResult { use schema::results; let mut affected = 0; for &uid in uids { @@ -196,7 +196,7 @@ impl UDB { Ok(affected) } - pub fn del_agents(&self, uids: &Vec) -> ULocalResult { + pub fn del_agents(&self, uids: &[Uuid]) -> ULocalResult { use schema::agents; let mut affected = 0; for &uid in uids { diff --git a/bin/u_server/src/filters.rs b/bin/u_server/src/filters.rs index 20ba803..44b40b8 100644 --- a/bin/u_server/src/filters.rs +++ b/bin/u_server/src/filters.rs @@ -48,12 +48,12 @@ pub fn make_filters() -> impl Filter .map(Some) .or_else(infallible_none), ) - .and_then(|uid| Endpoints::get_agent_jobs(uid)); + .and_then(Endpoints::get_agent_jobs); let get_personal_jobs = warp::get() .and(warp::path("get_personal_jobs")) .and(warp::path::param::().map(Some)) - .and_then(|uid| Endpoints::get_personal_jobs(uid)); + .and_then(Endpoints::get_personal_jobs); let del = warp::get() .and(warp::path("del")) diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index cdf9e47..3158bcf 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -46,7 +46,7 @@ impl Endpoints { info!("hnd: get_agents"); UDB::lock_db() .get_agents(uid) - .map(|m| build_message(m)) + .map(build_message) .or_else(|e| Ok(build_err(e))) } @@ -54,7 +54,7 @@ impl Endpoints { info!("hnd: get_jobs"); UDB::lock_db() .get_jobs(uid) - .map(|m| build_message(m)) + .map(build_message) .or_else(|e| Ok(build_err(e))) } @@ -62,14 +62,14 @@ impl Endpoints { info!("hnd: get_agent_jobs"); UDB::lock_db() .get_exact_jobs(uid, false) - .map(|m| build_message(m)) + .map(build_message) .or_else(|e| Ok(build_err(e))) } pub async fn get_personal_jobs(uid: Option) -> Result, Rejection> { info!("hnd: get_personal_jobs"); let agents = UDB::lock_db().get_agents(uid).unwrap(); - if agents.len() == 0 { + if agents.is_empty() { let db = UDB::lock_db(); db.insert_agent(&Agent::with_id(uid.unwrap())).unwrap(); let job = db.find_job_by_alias("agent_hello").unwrap(); @@ -105,7 +105,7 @@ impl Endpoints { let db = UDB::lock_db(); let del_fns = &[UDB::del_agents, UDB::del_jobs, UDB::del_results]; for del_fn in del_fns { - let affected = del_fn(&db, &vec![uid]).unwrap(); + let affected = del_fn(&db, &[uid]).unwrap(); if affected > 0 { return Ok(build_message(affected as i32)); } @@ -130,7 +130,7 @@ impl Endpoints { match jobs { Ok(j) => UDB::lock_db() .set_jobs_for_agent(&agent_uid, &j) - .map(|assigned_uids| build_message(assigned_uids)) + .map(build_message) .or_else(|e| Ok(build_err(e))), Err(e) => Ok(build_err(e)), } @@ -172,7 +172,7 @@ impl Endpoints { Reportable::Dummy => (), } } - if failed.len() > 0 { + if !failed.is_empty() { let err_msg = ULocalError::ProcessingError(failed.join(", ")); return Ok(build_err(err_msg)); } diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 71d8185..2317631 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -59,6 +59,7 @@ fn init_all() { prefill_jobs(); } +//TODO: tracing-subscriber pub async fn serve() { init_all(); let routes = make_filters(); diff --git a/integration/get_docker_ip.sh b/integration/get_docker_ip.sh new file mode 100755 index 0000000..9406e5e --- /dev/null +++ b/integration/get_docker_ip.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +PAT=".+IPAddress\W+\"([0-9\.]+).+" +docker ps | grep unki/$1 | cut -d' ' -f1 | xargs docker inspect | grep -P $PAT | sed -r "s/$PAT/\1/" diff --git a/integration/tests/helpers/panel.rs b/integration/tests/helpers/panel.rs index 713c60a..929715a 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration/tests/helpers/panel.rs @@ -1,8 +1,12 @@ use serde::de::DeserializeOwned; use serde_json::{from_slice, Value}; use shlex::split; +use std::fmt::Display; use std::process::{Command, Output}; -use u_lib::{datatypes::DataResult, utils::VecDisplay}; +use u_lib::{ + datatypes::DataResult, + utils::{bytes_to_string, ProcOutput}, +}; const PANEL_BINARY: &str = "/u_panel"; @@ -21,10 +25,11 @@ impl Panel { pub fn output_argv(argv: &[&str]) -> PanelResult { let result = Self::run(argv); - from_slice(&result.stdout).map_err(|e| { + let output = ProcOutput::from_output(&result).to_appropriate(); + from_slice(&output).map_err(|e| { eprintln!( "Failed to decode panel response: '{}'", - VecDisplay(result.stdout) + bytes_to_string(&output) ); e.to_string() }) @@ -48,7 +53,8 @@ impl Panel { } } - pub fn check_status(args: impl Into) { + pub fn check_status(args: impl Into + Display) { + println!("Panel: executing '{}'", &args); let result: PanelResult = Self::output(args); Self::status_is_ok(result); } diff --git a/integration/tests/tests.rs b/integration/tests/tests.rs index 7ef28c7..35a4789 100644 --- a/integration/tests/tests.rs +++ b/integration/tests/tests.rs @@ -2,5 +2,21 @@ mod behaviour; mod fixtures; mod helpers; +use std::env; + #[macro_use] extern crate rstest; + +// TODO: huita +#[ignore] +#[tokio::test] +async fn test_non_auth_connection_dropped() { + let env_server = env::var("U_SERVER").unwrap(); + match reqwest::get(format!("https://{}", env_server)).await { + Err(e) => { + dbg!(e.to_string()); + assert!(e.is_request()) + } + _ => panic!("no error occured on foreign client connection"), + } +} diff --git a/lib/u_api_proc_macro/src/lib.rs b/lib/u_api_proc_macro/src/lib.rs index f3d3a94..1a400ab 100644 --- a/lib/u_api_proc_macro/src/lib.rs +++ b/lib/u_api_proc_macro/src/lib.rs @@ -108,7 +108,7 @@ fn parse_fn_args(raw: Punctuated) -> FnArgs { if &arg_name != "url_param" && &arg_name != "payload" { panic!("Wrong arg name: {}", &arg_name) } - let arg_type = *argt.ty.clone(); + let arg_type = *argt.ty; Some((arg_name, arg_type)) } else { None diff --git a/lib/u_lib/src/executor.rs b/lib/u_lib/src/executor.rs index dc4bc79..c34e08b 100644 --- a/lib/u_lib/src/executor.rs +++ b/lib/u_lib/src/executor.rs @@ -118,7 +118,7 @@ pub async fn pop_completed() -> Vec { .lock() .await .keys() - .map(|k| *k) + .copied() .collect::>(); for fid in fids { if let Some(r) = pop_task_if_completed(fid).await { diff --git a/lib/u_lib/src/messaging/base.rs b/lib/u_lib/src/messaging/base.rs index 5330927..8ac1158 100644 --- a/lib/u_lib/src/messaging/base.rs +++ b/lib/u_lib/src/messaging/base.rs @@ -46,10 +46,7 @@ impl<'cow, I: AsMsg> BaseMessage<'cow, I> { C: Into>, { let Moo(inner) = inner.into(); - Self { - id: UID.clone(), - inner, - } + Self { id: *UID, inner } } pub fn into_inner(self) -> I { diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index bcc46e9..daa49a0 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -106,7 +106,7 @@ impl Default for Agent { fn default() -> Self { Self { alias: None, - id: UID.clone(), + id: *UID, hostname: String::new(), is_root: false, is_root_allowed: false, diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index a75d88c..1b0b3a9 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -3,13 +3,13 @@ use crate::{ cache::JobCache, errors::UError, messaging::Reportable, - models::{schema::*, JobOutput}, - utils::{systime_to_string, TempFile}, + models::schema::*, + utils::{systime_to_string, ProcOutput, TempFile}, UID, }; use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; -use std::{fmt, process::Output, time::SystemTime}; +use std::{fmt, time::SystemTime}; use tokio::process::Command; use uuid::Uuid; @@ -101,19 +101,12 @@ impl AssignedJob { let args = split_cmd.collect::>(); let cmd_result = Command::new(cmd).args(args).output().await; let (data, retcode) = match cmd_result { - Ok(Output { - status, - stdout, - stderr, - }) => ( - JobOutput::new() - .stdout(stdout) - .stderr(stderr) - .into_combined(), - status.code(), + Ok(output) => ( + ProcOutput::from_output(&output).into_combined(), + output.status.code(), ), Err(e) => ( - JobOutput::new() + ProcOutput::new() .stderr(e.to_string().into_bytes()) .into_combined(), None, @@ -134,15 +127,15 @@ impl AssignedJob { } } - pub fn as_job_output(&self) -> Option { + pub fn as_job_output(&self) -> Option { self.result .as_ref() - .and_then(|r| JobOutput::from_combined(r)) + .and_then(|r| ProcOutput::from_combined(r)) } pub fn to_raw_result(&self) -> Vec { match self.result.as_ref() { - Some(r) => match JobOutput::from_combined(r) { + Some(r) => match ProcOutput::from_combined(r) { Some(o) => o.to_appropriate(), None => r.clone(), }, diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 79bce50..2a12e06 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -26,7 +26,7 @@ impl JobMeta { JobMetaBuilder::default() } - pub fn from_shell>(cmd: S) -> UResult { + pub fn from_shell(cmd: impl Into) -> UResult { Self::builder().with_shell(cmd).build() } } @@ -80,17 +80,17 @@ impl Default for JobMetaBuilder { } impl JobMetaBuilder { - pub fn with_shell>(mut self, shell_cmd: S) -> Self { + pub fn with_shell(mut self, shell_cmd: impl Into) -> Self { self.inner.argv = shell_cmd.into(); self } - pub fn with_payload>>(mut self, payload: C) -> Self { + pub fn with_payload(mut self, payload: impl Into>) -> Self { self.inner.payload = Some(payload.into()); self } - pub fn with_alias>(mut self, alias: S) -> Self { + pub fn with_alias(mut self, alias: impl Into) -> Self { self.inner.alias = Some(alias.into()); self } @@ -104,13 +104,13 @@ impl JobMetaBuilder { let mut inner = self.inner; match inner.exec_type { JobType::Shell => { - if inner.argv == "" { + if inner.argv.is_empty() { inner.argv = String::from("/bin/bash -c {}") } let argv_parts = shlex::split(&inner.argv).ok_or(UError::JobArgsError("Shlex failed".into()))?; let empty_err = UError::JobArgsError("Empty argv".into()); - if argv_parts.get(0).ok_or(empty_err.clone())?.len() == 0 { + if argv_parts.get(0).ok_or(empty_err.clone())?.is_empty() { return Err(empty_err.into()); } match inner.payload.as_ref() { @@ -120,8 +120,6 @@ impl JobMetaBuilder { "Argv contains no executable placeholder".into(), ) .into()); - } else { - () } } None => { @@ -131,8 +129,6 @@ impl JobMetaBuilder { .into(), ) .into()); - } else { - () } } }; diff --git a/lib/u_lib/src/models/jobs/misc.rs b/lib/u_lib/src/models/jobs/misc.rs index 7451548..ebcd38d 100644 --- a/lib/u_lib/src/models/jobs/misc.rs +++ b/lib/u_lib/src/models/jobs/misc.rs @@ -35,152 +35,3 @@ pub enum JobType { Shell, Python, } - -#[derive(Clone, Debug)] -pub struct JobOutput { - pub stdout: Vec, - pub stderr: Vec, -} - -impl JobOutput { - const STREAM_BORDER: &'static str = "***"; - const STDOUT: &'static str = "STDOUT"; - const STDERR: &'static str = "STDERR"; - - #[inline] - fn create_delim(header: &'static str) -> Vec { - format!( - "<{border}{head}{border}>", - border = Self::STREAM_BORDER, - head = header - ) - .into_bytes() - } - - pub fn new() -> Self { - Self { - stdout: vec![], - stderr: vec![], - } - } - - pub fn stdout(mut self, data: Vec) -> Self { - self.stdout = data; - self - } - - pub fn stderr(mut self, data: Vec) -> Self { - self.stderr = data; - self - } - - pub fn into_combined(self) -> Vec { - let mut result: Vec = vec![]; - if self.stdout.len() > 0 { - result.extend(Self::create_delim(Self::STDOUT)); - result.extend(self.stdout); - } - - if self.stderr.len() > 0 { - result.extend(Self::create_delim(Self::STDERR)); - result.extend(self.stderr); - } - result - } - - pub fn from_combined(raw: &[u8]) -> Option { - enum ParseFirst { - Stdout, - Stderr, - } - fn split_by_subslice<'s>(slice: &'s [u8], subslice: &[u8]) -> Option<(&'s [u8], &'s [u8])> { - slice - .windows(subslice.len()) - .position(|w| w == subslice) - .map(|split_pos| { - let splitted = slice.split_at(split_pos); - (&splitted.0[..split_pos], &splitted.1[subslice.len()..]) - }) - } - let splitter = |p: ParseFirst| { - let (first_hdr, second_hdr) = match p { - ParseFirst::Stdout => (Self::STDOUT, Self::STDERR), - ParseFirst::Stderr => (Self::STDERR, Self::STDOUT), - }; - let first_hdr = Self::create_delim(first_hdr); - let second_hdr = Self::create_delim(second_hdr); - split_by_subslice(raw, &first_hdr).map(|(_, p2)| { - match split_by_subslice(p2, &second_hdr) { - Some((p2_1, p2_2)) => Self::new().stdout(p2_1.to_vec()).stderr(p2_2.to_vec()), - None => Self::new().stdout(p2.to_vec()), - } - }) - }; - splitter(ParseFirst::Stdout).or(splitter(ParseFirst::Stderr)) - } - - pub fn to_appropriate(&self) -> Vec { - let mut result: Vec = vec![]; - if self.stdout.len() > 0 { - result.extend(&self.stdout); - } - if self.stderr.len() > 0 { - if result.len() > 0 { - result.push(b'\n'); - } - result.extend(&self.stderr); - } - if result.len() == 0 { - result.extend(b"No data"); - } - result - } -} - -#[cfg(test)] -mod tests { - use crate::{models::JobOutput, utils::bytes_to_string}; - use test_case::test_case; - - const STDOUT: &str = "<***STDOUT***>"; - const STDERR: &str = "<***STDERR***>"; - - #[test_case( - "lol", - "kek", - &format!("{}lol{}kek", STDOUT, STDERR) - ;"stdout stderr" - )] - #[test_case( - "", - "kek", - &format!("{}kek", STDERR) - ;"stderr" - )] - fn test_to_combined(stdout: &str, stderr: &str, result: &str) { - let output = JobOutput::new() - .stdout(stdout.as_bytes().to_vec()) - .stderr(stderr.as_bytes().to_vec()); - assert_eq!(&bytes_to_string(&output.into_combined()), result) - } - - #[test_case( - &format!("{}lal{}kik", STDOUT, STDERR), - "lal\nkik" - ;"stdout stderr" - )] - #[test_case( - &format!("{}qeq", STDOUT), - "qeq" - ;"stdout" - )] - #[test_case( - &format!("{}vev", STDERR), - "vev" - ;"stderr" - )] - fn test_from_combined(src: &str, result: &str) { - let output = JobOutput::from_combined(src.as_bytes()).unwrap(); - assert_eq!(bytes_to_string(&output.to_appropriate()).trim(), result); - } -} diff --git a/lib/u_lib/src/utils/combined_result.rs b/lib/u_lib/src/utils/combined_result.rs index aa0968d..4464c39 100644 --- a/lib/u_lib/src/utils/combined_result.rs +++ b/lib/u_lib/src/utils/combined_result.rs @@ -31,7 +31,7 @@ impl CombinedResult { } pub fn has_err(&self) -> bool { - self.err.len() > 0 + !self.err.is_empty() } pub fn unwrap_one(self) -> T { diff --git a/lib/u_lib/src/utils/fmt/stripped.rs b/lib/u_lib/src/utils/fmt/stripped.rs index 8681a97..b1ed949 100644 --- a/lib/u_lib/src/utils/fmt/stripped.rs +++ b/lib/u_lib/src/utils/fmt/stripped.rs @@ -6,7 +6,6 @@ use std::str::Chars; const MAX_DATA_LEN: usize = 200; pub trait Strippable { - //TODO: waiting for stabilizing GATs type Item: fmt::Display; type TypeIter: Iterator; diff --git a/lib/u_lib/src/utils/mod.rs b/lib/u_lib/src/utils/mod.rs index 98853c2..41cf1a2 100644 --- a/lib/u_lib/src/utils/mod.rs +++ b/lib/u_lib/src/utils/mod.rs @@ -2,6 +2,7 @@ mod combined_result; mod conv; mod fmt; mod misc; +mod proc_output; mod storage; mod tempfile; mod vec_display; @@ -10,6 +11,7 @@ pub use combined_result::*; pub use conv::*; pub use fmt::*; pub use misc::*; +pub use proc_output::*; pub use storage::*; pub use tempfile::*; pub use vec_display::*; diff --git a/lib/u_lib/src/utils/proc_output.rs b/lib/u_lib/src/utils/proc_output.rs new file mode 100644 index 0000000..5236e6b --- /dev/null +++ b/lib/u_lib/src/utils/proc_output.rs @@ -0,0 +1,163 @@ +use std::process::Output; + +#[derive(Clone, Debug)] +pub struct ProcOutput { + pub stdout: Vec, + pub stderr: Vec, +} + +impl ProcOutput { + const STREAM_BORDER: &'static str = "***"; + const STDOUT: &'static str = "STDOUT"; + const STDERR: &'static str = "STDERR"; + + #[inline] + fn create_delim(header: &'static str) -> Vec { + format!( + "<{border}{head}{border}>", + border = Self::STREAM_BORDER, + head = header + ) + .into_bytes() + } + + pub fn from_output(output: &Output) -> Self { + let mut this = Self::new().stdout(output.stdout.to_vec()); + if !output.status.success() && output.stderr.len() > 0 { + this.stderr = output.stderr.to_vec(); + } + this + } + + pub fn new() -> Self { + Self { + stdout: vec![], + stderr: vec![], + } + } + + pub fn stdout(mut self, data: Vec) -> Self { + self.stdout = data; + self + } + + pub fn stderr(mut self, data: Vec) -> Self { + self.stderr = data; + self + } + + /// Make bytestring like '<***STDOUT***>...<***STDERR***>...' + pub fn into_combined(self) -> Vec { + let mut result: Vec = vec![]; + if !self.stdout.is_empty() { + result.extend(Self::create_delim(Self::STDOUT)); + result.extend(self.stdout); + } + + if !self.stderr.is_empty() { + result.extend(Self::create_delim(Self::STDERR)); + result.extend(self.stderr); + } + result + } + + pub fn from_combined(raw: &[u8]) -> Option { + enum ParseFirst { + Stdout, + Stderr, + } + fn split_by_subslice<'s>(slice: &'s [u8], subslice: &[u8]) -> Option<(&'s [u8], &'s [u8])> { + slice + .windows(subslice.len()) + .position(|w| w == subslice) + .map(|split_pos| { + let splitted = slice.split_at(split_pos); + (&splitted.0[..split_pos], &splitted.1[subslice.len()..]) + }) + } + let splitter = |p: ParseFirst| { + let (first_hdr, second_hdr) = match p { + ParseFirst::Stdout => (Self::STDOUT, Self::STDERR), + ParseFirst::Stderr => (Self::STDERR, Self::STDOUT), + }; + let first_hdr = Self::create_delim(first_hdr); + let second_hdr = Self::create_delim(second_hdr); + split_by_subslice(raw, &first_hdr).map(|(_, p2)| { + match split_by_subslice(p2, &second_hdr) { + Some((p2_1, p2_2)) => Self::new().stdout(p2_1.to_vec()).stderr(p2_2.to_vec()), + None => Self::new().stdout(p2.to_vec()), + } + }) + }; + splitter(ParseFirst::Stdout).or_else(|| splitter(ParseFirst::Stderr)) + } + + /// Chooses between stdout and stderr or both wisely + pub fn to_appropriate(&self) -> Vec { + let mut result: Vec = vec![]; + let mut altered = false; + if !self.stdout.is_empty() { + result.extend(&self.stdout); + altered = true; + } + if !self.stderr.is_empty() { + if altered { + result.push(b'\n'); + } + result.extend(&self.stderr); + altered = true; + } + if !altered { + result.extend(b"No data"); + } + result + } +} + +#[cfg(test)] +mod tests { + use crate::utils::{bytes_to_string, ProcOutput}; + use test_case::test_case; + + const STDOUT: &str = "<***STDOUT***>"; + const STDERR: &str = "<***STDERR***>"; + + #[test_case( + "lol", + "kek", + &format!("{}lol{}kek", STDOUT, STDERR) + ;"stdout stderr" + )] + #[test_case( + "", + "kek", + &format!("{}kek", STDERR) + ;"stderr" + )] + fn test_to_combined(stdout: &str, stderr: &str, result: &str) { + let output = ProcOutput::new() + .stdout(stdout.as_bytes().to_vec()) + .stderr(stderr.as_bytes().to_vec()); + assert_eq!(&bytes_to_string(&output.into_combined()), result) + } + + #[test_case( + &format!("{}lal{}kik", STDOUT, STDERR), + "lal\nkik" + ;"stdout stderr" + )] + #[test_case( + &format!("{}qeq", STDOUT), + "qeq" + ;"stdout" + )] + #[test_case( + &format!("{}vev", STDERR), + "vev" + ;"stderr" + )] + fn test_from_combined(src: &str, result: &str) { + let output = ProcOutput::from_combined(src.as_bytes()).unwrap(); + assert_eq!(bytes_to_string(&output.to_appropriate()).trim(), result); + } +} diff --git a/scripts/build_musl_libs.sh b/scripts/build_musl_libs.sh index 1cf612c..b63211d 100755 --- a/scripts/build_musl_libs.sh +++ b/scripts/build_musl_libs.sh @@ -2,7 +2,7 @@ set -ex source $(dirname $0)/rootdir.sh #set ROOTDIR ARGS=$@ -STATIC_LIBS=static +STATIC_LIBS=./static DOCKER_EXCHG=/musl-share IMAGE=unki/musllibs mkdir -p $STATIC_LIBS From ee514ecb20b8da8af1b954812f20afd735ebc48e Mon Sep 17 00:00:00 2001 From: plazmoid Date: Thu, 14 Oct 2021 04:28:07 +0500 Subject: [PATCH 07/30] improved dockerizing --- .gitignore | 3 ++- .../integration-tests/tests_runner.Dockerfile | 2 +- images/integration-tests/u_db.Dockerfile | 10 +++++++++- images/integration-tests/u_db_entrypoint.sh | 5 +++++ images/integration-tests/u_server.Dockerfile | 4 ++-- integration/docker-compose.yml | 20 +++++++++++-------- 6 files changed, 31 insertions(+), 13 deletions(-) create mode 100755 images/integration-tests/u_db_entrypoint.sh diff --git a/.gitignore b/.gitignore index 19d820c..8946455 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ certs/* echoer .env.private *.lock -static/ \ No newline at end of file +static/ +.vscode/ \ No newline at end of file diff --git a/images/integration-tests/tests_runner.Dockerfile b/images/integration-tests/tests_runner.Dockerfile index 2411bd6..4bf8287 100644 --- a/images/integration-tests/tests_runner.Dockerfile +++ b/images/integration-tests/tests_runner.Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.53 +FROM rust:1.55 RUN rustup target add x86_64-unknown-linux-musl CMD ["sleep", "3600"] \ No newline at end of file diff --git a/images/integration-tests/u_db.Dockerfile b/images/integration-tests/u_db.Dockerfile index 8577c8d..783d887 100644 --- a/images/integration-tests/u_db.Dockerfile +++ b/images/integration-tests/u_db.Dockerfile @@ -1,3 +1,11 @@ FROM postgres:13.3 -RUN apt update && apt -y upgrade && apt install -y iproute2 \ No newline at end of file +RUN apt update && apt upgrade -y +RUN apt install -y curl build-essential libpq-dev iproute2 +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal +ENV PATH /root/.cargo/bin:$PATH +RUN rustup target add x86_64-unknown-linux-musl +RUN cargo install diesel_cli --no-default-features --features postgres + +RUN mkdir -p /unki +COPY u_db_entrypoint.sh /unki/ \ No newline at end of file diff --git a/images/integration-tests/u_db_entrypoint.sh b/images/integration-tests/u_db_entrypoint.sh new file mode 100755 index 0000000..e67acc1 --- /dev/null +++ b/images/integration-tests/u_db_entrypoint.sh @@ -0,0 +1,5 @@ +set -m + +export DATABASE_URL=postgres://${DB_USER}:${DB_PASSWORD}@127.0.0.1/${DB_NAME} +/usr/local/bin/docker-entrypoint.sh postgres & +sleep 10 && diesel setup && fg %1 \ No newline at end of file diff --git a/images/integration-tests/u_server.Dockerfile b/images/integration-tests/u_server.Dockerfile index ea82ccd..691a618 100644 --- a/images/integration-tests/u_server.Dockerfile +++ b/images/integration-tests/u_server.Dockerfile @@ -1,3 +1,3 @@ -FROM rust:1.53 +FROM alpine:latest -RUN cargo install diesel_cli --no-default-features --features postgres \ No newline at end of file +RUN apk add iproute2 bash \ No newline at end of file diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml index c0381e3..848d792 100644 --- a/integration/docker-compose.yml +++ b/integration/docker-compose.yml @@ -11,11 +11,9 @@ services: - u_net volumes: - ../target/x86_64-unknown-linux-musl/release/u_server:/u_server - - ../:/unki/ + - ../certs:/unki/certs working_dir: /unki - command: bash -c " - export DATABASE_URL=postgres://$${DB_USER}:$${DB_PASSWORD}@$${DB_HOST}/$${DB_NAME} && - diesel setup && diesel migration run && /u_server" + command: /u_server depends_on: u_db: condition: service_healthy @@ -27,7 +25,7 @@ services: environment: RUST_LOG: trace healthcheck: - test: /bin/ss -tlpn | grep 63714 + test: ss -tlpn | grep 63714 interval: 5s timeout: 2s retries: 2 @@ -41,11 +39,17 @@ services: env_file: - ../.env - ../.env.private + working_dir: /unki + volumes: + - ../migrations:/unki/migrations + - ../Cargo.toml:/unki/Cargo.toml + command: /unki/u_db_entrypoint.sh healthcheck: - test: /bin/ss -tlpn | grep 5432 + # test if db's port is open and db is created + test: ss -tlpn | grep 5432 && psql -lqt -U $${DB_USER} | grep -qw $${DB_NAME} interval: 5s - timeout: 2s - retries: 2 + timeout: 5s + retries: 3 u_agent_1: image: unki/u_agent From a7b4b333a5dbca3b96b1ddb36d3d4c3ae2d877bc Mon Sep 17 00:00:00 2001 From: plazmoid Date: Thu, 14 Oct 2021 05:14:04 +0500 Subject: [PATCH 08/30] gen_schema restored --- Makefile.toml | 4 +++- images/integration-tests/u_db_entrypoint.sh | 3 ++- integration/docker-compose.yml | 11 ++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index 44d8f2d..de6a2f0 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -41,7 +41,9 @@ bash integration_tests.sh [tasks.gen_schema] script = ''' cd ./integration -docker-compose up -d u_server +docker-compose up -d u_db_gen_schema +echo "Waiting 10 sec..." +sleep 10 docker-compose down ''' diff --git a/images/integration-tests/u_db_entrypoint.sh b/images/integration-tests/u_db_entrypoint.sh index e67acc1..eb96742 100755 --- a/images/integration-tests/u_db_entrypoint.sh +++ b/images/integration-tests/u_db_entrypoint.sh @@ -1,5 +1,6 @@ set -m export DATABASE_URL=postgres://${DB_USER}:${DB_PASSWORD}@127.0.0.1/${DB_NAME} +touch /unki/Cargo.toml /usr/local/bin/docker-entrypoint.sh postgres & -sleep 10 && diesel setup && fg %1 \ No newline at end of file +sleep 10 && diesel setup && diesel migration run && fg %1 \ No newline at end of file diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml index 848d792..84255c3 100644 --- a/integration/docker-compose.yml +++ b/integration/docker-compose.yml @@ -42,7 +42,6 @@ services: working_dir: /unki volumes: - ../migrations:/unki/migrations - - ../Cargo.toml:/unki/Cargo.toml command: /unki/u_db_entrypoint.sh healthcheck: # test if db's port is open and db is created @@ -51,6 +50,16 @@ services: timeout: 5s retries: 3 + u_db_gen_schema: + image: unki/u_db + env_file: + - ../.env + - ../.env.private + working_dir: /unki + volumes: + - ../:/unki/ + command: /unki/images/integration-tests/u_db_entrypoint.sh + u_agent_1: image: unki/u_agent networks: From b247c8640d486104fd006f6f9720f091cc566133 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Fri, 15 Oct 2021 04:50:01 +0500 Subject: [PATCH 09/30] deployable --- Makefile.toml | 31 ++++++++++---- bin/u_agent/build.rs | 2 +- bin/u_server/src/u_server.rs | 2 +- images/integration-tests/u_db_entrypoint.sh | 3 +- integration/docker-compose.yml | 46 +++++---------------- integration/docker_compose.py | 33 ++++++++------- .env.private.sample => sample.env.private | 0 scripts/deploy.sh | 19 +++++++++ {certs => scripts}/gen_certs.sh | 11 ++++- scripts/start_server.sh | 5 +++ 10 files changed, 90 insertions(+), 62 deletions(-) rename .env.private.sample => sample.env.private (100%) create mode 100755 scripts/deploy.sh rename {certs => scripts}/gen_certs.sh (80%) create mode 100755 scripts/start_server.sh diff --git a/Makefile.toml b/Makefile.toml index de6a2f0..bddcb4d 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -17,11 +17,26 @@ script = "./scripts/build_musl_libs.sh" command = "${CARGO}" args = ["clean"] -[tasks.build] +[tasks.cargo_build] dependencies = ["build_static_libs"] command = "${CARGO}" args = ["build", "--target", "${TARGET}", "${@}"] +[tasks.release_tasks] +script = ''' +if [[ "${@}" =~ "release" ]]; then + echo "Stripping binaries..." + strip $(ls ./target/${TARGET}/release/u_* -1 | grep -v ".d") + echo "Creating symlink to release dir..." + ln -s ./target/${TARGET}/release ./release || true +fi +''' + +[tasks.build] +dependencies = ["cargo_build", "release_tasks"] +command = "true" +args = [] + [tasks.run] script = ''' echo "Only integration tests are supported." @@ -40,12 +55,14 @@ bash integration_tests.sh [tasks.gen_schema] script = ''' -cd ./integration -docker-compose up -d u_db_gen_schema -echo "Waiting 10 sec..." -sleep 10 -docker-compose down +docker run --rm \ + --env-file=$PWD/.env \ + --env-file=$PWD/.env.private \ + -v $PWD:/unki \ + -w /unki \ + unki/u_db \ + /unki/images/integration-tests/u_db_entrypoint.sh || true ''' [tasks.test] -dependencies = ["unit", "integration"] \ No newline at end of file +dependencies = ["unit", "integration"] diff --git a/bin/u_agent/build.rs b/bin/u_agent/build.rs index a9a9abd..0588b1c 100644 --- a/bin/u_agent/build.rs +++ b/bin/u_agent/build.rs @@ -3,6 +3,6 @@ use std::path::PathBuf; fn main() { let server_cert = PathBuf::from("../../certs/ca.crt"); if !server_cert.exists() { - panic!("CA certificate doesn't exist. Create it first with certs/gen_certs.sh"); + panic!("CA certificate doesn't exist. Create it first with scripts/gen_certs.sh"); } } diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 2317631..adfeb1c 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -43,7 +43,7 @@ fn init_logger() { let logfile = OpenOptions::new() .append(true) .create(true) - .open(LOGFILE) + .open(PathBuf::from("logs").join(LOGFILE)) .unwrap(); let level = LevelFilter::Info; let loggers = vec![ diff --git a/images/integration-tests/u_db_entrypoint.sh b/images/integration-tests/u_db_entrypoint.sh index eb96742..f6f560c 100755 --- a/images/integration-tests/u_db_entrypoint.sh +++ b/images/integration-tests/u_db_entrypoint.sh @@ -3,4 +3,5 @@ set -m export DATABASE_URL=postgres://${DB_USER}:${DB_PASSWORD}@127.0.0.1/${DB_NAME} touch /unki/Cargo.toml /usr/local/bin/docker-entrypoint.sh postgres & -sleep 10 && diesel setup && diesel migration run && fg %1 \ No newline at end of file +sleep 10 && diesel setup && diesel migration run +[[ $1 == "svc" ]] && fg %1 \ No newline at end of file diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml index 84255c3..003460d 100644 --- a/integration/docker-compose.yml +++ b/integration/docker-compose.yml @@ -10,15 +10,16 @@ services: networks: - u_net volumes: - - ../target/x86_64-unknown-linux-musl/release/u_server:/u_server + - ../release/u_server:/unki/u_server - ../certs:/unki/certs + - ../logs:/unki/logs working_dir: /unki - command: /u_server + command: /unki/u_server depends_on: u_db: condition: service_healthy - expose: - - '63714' + ports: + - 63714:63714 env_file: - ../.env - ../.env.private @@ -42,7 +43,7 @@ services: working_dir: /unki volumes: - ../migrations:/unki/migrations - command: /unki/u_db_entrypoint.sh + command: /unki/u_db_entrypoint.sh svc healthcheck: # test if db's port is open and db is created test: ss -tlpn | grep 5432 && psql -lqt -U $${DB_USER} | grep -qw $${DB_NAME} @@ -50,37 +51,12 @@ services: timeout: 5s retries: 3 - u_db_gen_schema: - image: unki/u_db - env_file: - - ../.env - - ../.env.private - working_dir: /unki - volumes: - - ../:/unki/ - command: /unki/images/integration-tests/u_db_entrypoint.sh - - u_agent_1: - image: unki/u_agent - networks: - - u_net - volumes: - - ../target/x86_64-unknown-linux-musl/release/u_agent:/u_agent - command: /u_agent u_server - env_file: - - ../.env - environment: - RUST_LOG: u_agent=debug - depends_on: - u_server: - condition: service_healthy - - u_agent_2: + u_agent: image: unki/u_agent networks: - u_net volumes: - - ../target/x86_64-unknown-linux-musl/release/u_agent:/u_agent + - ../release/u_agent:/u_agent command: /u_agent u_server env_file: - ../.env @@ -98,15 +74,13 @@ services: - ~/.cargo/registry:/root/.cargo/registry - ./:/tests/ - ../certs:/certs - - ../target/x86_64-unknown-linux-musl/release/u_panel:/u_panel + - ../release/u_panel:/u_panel - ../lib/u_lib:/lib/u_lib - ../lib/u_api_proc_macro:/lib/u_api_proc_macro working_dir: /tests/ depends_on: - u_agent_1: - condition: service_started - u_agent_2: + u_agent: condition: service_started u_server: condition: service_healthy diff --git a/integration/docker_compose.py b/integration/docker_compose.py index 47e3916..69ef302 100644 --- a/integration/docker_compose.py +++ b/integration/docker_compose.py @@ -6,58 +6,63 @@ from docker import docker, check_state, print_errors class Compose: ALL_CONTAINERS = [ - 'u_agent_1', - 'u_agent_2', + 'u_agent', 'u_server', 'u_db', 'tests_runner', ] def __init__(self): - self.container_tpl = 'integration_%s_1' - self.cmd_container = self.container_tpl % 'tests_runner' - self.ALL_CONTAINERS = [self.container_tpl % c for c in self.ALL_CONTAINERS] + self.container_tpl = 'integration_%s_%d' + self.cmd_container = self.container_tpl % ('tests_runner', 1) + self.ALL_CONTAINERS = [self.container_tpl % (c, 1) for c in self.ALL_CONTAINERS] + self.scaled_svc = {} + self.scale("u_agent", 2) + + def scale(self, svc, count): + for c in range(1, count): + new_container = self.container_tpl % (svc, c + 1) + self.ALL_CONTAINERS.append(new_container) + self.scaled_svc[svc] = count def _call(self, *args): - subprocess.check_call([ + cmd = [ 'docker-compose', '--no-ansi', ] + list(args) - ) + log(f'Running docker-compose command: {cmd}') + subprocess.check_call(cmd) def up(self): log('Instanciating cluster') - self._call('up', '-d') - log('Ok') + scaled = [f"{k}={v}" for k, v in self.scaled_svc.items()] + if len(scaled) > 0: + scaled.insert(0, '--scale') + self._call('up', '-d', *scaled) def down(self): log('Shutting down cluster') self._call('down') - log('Ok') def stop(self): log('Stopping cluster') self._call('stop') - log('Ok') def run(self, cmd): container = self.cmd_container if isinstance(cmd, str): cmd = shlex.split(cmd) - log(f'Running command "{cmd}" in container {container}') result = docker([ 'exec', '-ti', container ] + cmd) - log('Ok') return result def is_alive(self): log('Check if all containers are alive') errors = check_state(self.ALL_CONTAINERS) - log('Check done') if errors: print_errors(errors) diff --git a/.env.private.sample b/sample.env.private similarity index 100% rename from .env.private.sample rename to sample.env.private diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..aa04c86 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -xe +source $(dirname $0)/rootdir.sh #set ROOTDIR + +SERVER="ortem" +REMOTE_DIR=/srv/usrv +REMOTE_PATH=$SERVER:$REMOTE_DIR +RSYNC="rsync -arzh --progress" + +ssh $SERVER mkdir -p $REMOTE_DIR/{release,deploy} +$RSYNC $ROOTDIR/release/u_server $REMOTE_PATH/release/u_server +$RSYNC --exclude="*.sh" $ROOTDIR/certs/ $REMOTE_PATH/certs +$RSYNC $ROOTDIR/migrations/ $REMOTE_PATH/migrations +$RSYNC $ROOTDIR/.env* $REMOTE_PATH/ +$RSYNC $ROOTDIR/integration/docker-compose.yml $REMOTE_PATH/deploy/ +$RSYNC $ROOTDIR/images/integration-tests/u_db* $REMOTE_PATH/deploy/ +$RSYNC $ROOTDIR/images/integration-tests/u_server.Dockerfile $REMOTE_PATH/deploy/ +$RSYNC $ROOTDIR/scripts/start_server.sh $REMOTE_PATH/start_server.sh +ssh $SERVER "cd $REMOTE_DIR/deploy && ./start_server.sh" \ No newline at end of file diff --git a/certs/gen_certs.sh b/scripts/gen_certs.sh similarity index 80% rename from certs/gen_certs.sh rename to scripts/gen_certs.sh index f1e9d6f..3be58ea 100755 --- a/certs/gen_certs.sh +++ b/scripts/gen_certs.sh @@ -1,16 +1,23 @@ set -ex -DIR=. +source $(dirname $0)/rootdir.sh #set ROOTDIR +DIR=$ROOTDIR/certs V3_CFG=$DIR/v3.ext +mkdir -p $DIR cat > $V3_CFG << EOF authorityKeyIdentifier=keyid,issuer basicConstraints=CA:FALSE keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment, keyAgreement, keyCertSign, cRLSign +subjectAltName = @alt_names + +[alt_names] +DNS.1 = ortem.xyz +DNS.2 = u_server EOF openssl req -x509 -newkey rsa:4096 -keyout $DIR/ca.key -out $DIR/ca.crt -nodes -days 365 -subj "/CN=root" openssl req -newkey rsa:4096 -keyout $DIR/alice.key -out $DIR/alice.csr -nodes -days 365 -subj "/CN=alice" -openssl req -newkey rsa:4096 -keyout $DIR/server.key -out $DIR/server.csr -nodes -days 365 -subj "/CN=u_server" +openssl req -newkey rsa:4096 -keyout $DIR/server.key -out $DIR/server.csr -nodes -days 365 -subj "/CN=ortem.xyz" openssl x509 -req -in $DIR/alice.csr -CA $DIR/ca.crt -CAkey $DIR/ca.key -out $DIR/alice.crt -set_serial 01 -days 365 -extfile $V3_CFG openssl x509 -req -in $DIR/server.csr -CA $DIR/ca.crt -CAkey $DIR/ca.key -out $DIR/server.crt -set_serial 01 -days 365 -extfile $V3_CFG openssl pkcs12 -export -out $DIR/alice.p12 -inkey $DIR/alice.key -in $DIR/alice.crt -passin pass: -passout pass: diff --git a/scripts/start_server.sh b/scripts/start_server.sh new file mode 100755 index 0000000..07a6d6a --- /dev/null +++ b/scripts/start_server.sh @@ -0,0 +1,5 @@ +#!/bin/bash +docker build -t unki/u_db -f u_db.Dockerfile . +docker build -t unki/u_server -f u_server.Dockerfile . +docker-compose down +docker-compose up -d u_server \ No newline at end of file From f8408655979b8f8af9c91f7c22f594d220e1b115 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sun, 17 Oct 2021 04:40:18 +0500 Subject: [PATCH 10/30] wasm initial --- Cargo.toml | 2 + Makefile.toml | 3 ++ bin/u_agent/src/lib.rs | 5 +- bin/u_panel/Cargo.toml | 6 +-- bin/u_panel/be/Cargo.toml | 10 ++++ .../{src/server/app.rs => be/src/lib.rs} | 0 bin/u_panel/fe/Cargo.toml | 12 +++++ bin/u_panel/fe/index.html | 9 ++++ bin/u_panel/fe/src/main.rs | 51 +++++++++++++++++++ bin/u_panel/src/argparse.rs | 2 +- bin/u_panel/src/main.rs | 1 - bin/u_panel/src/server/mod.rs | 3 -- integration/tests/tests.rs | 10 ++-- lib/u_lib/Cargo.toml | 15 +++--- lib/u_lib/src/config.rs | 2 +- lib/u_lib/src/errors/variants.rs | 3 ++ lib/u_lib/src/lib.rs | 35 +++++++++---- lib/u_lib/src/models/agent.rs | 15 +++--- lib/u_lib/src/models/jobs/assigned.rs | 8 +-- lib/u_lib/src/models/jobs/meta.rs | 8 ++- lib/u_lib/src/utils/misc.rs | 38 -------------- lib/u_lib/src/utils/mod.rs | 7 +++ lib/u_lib/src/utils/unix.rs | 37 ++++++++++++++ scripts/deploy.sh | 3 +- 24 files changed, 198 insertions(+), 87 deletions(-) create mode 100644 bin/u_panel/be/Cargo.toml rename bin/u_panel/{src/server/app.rs => be/src/lib.rs} (100%) create mode 100644 bin/u_panel/fe/Cargo.toml create mode 100644 bin/u_panel/fe/index.html create mode 100644 bin/u_panel/fe/src/main.rs delete mode 100644 bin/u_panel/src/server/mod.rs create mode 100644 lib/u_lib/src/utils/unix.rs diff --git a/Cargo.toml b/Cargo.toml index ede90c2..9cb212d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,8 @@ members = [ "bin/u_agent", "bin/u_panel", + "bin/u_panel/be", + "bin/u_panel/fe", "bin/u_run", "bin/u_server", "lib/u_lib", diff --git a/Makefile.toml b/Makefile.toml index bddcb4d..45c2842 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -66,3 +66,6 @@ docker run --rm \ [tasks.test] dependencies = ["unit", "integration"] + +[tasks.deploy] +script = './scripts/deploy.sh' diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index ad83999..a0b56f1 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -60,7 +60,7 @@ pub async fn process_request(job_requests: Vec, client: &ClientHand } } -async fn error_reporting(client: Arc) { +async fn error_reporting(client: Arc) -> ! { loop { let err = ErrChan::recv(); debug!("Error encountered: {:?}", err); @@ -100,10 +100,9 @@ pub async fn run_forever() { env_logger::init(); let arg_ip = env::args().nth(1); let client = Arc::new(ClientHandler::new(arg_ip.as_deref())); - let _cli = client.clone(); panic::set_hook(Box::new(|panic_info| { ErrChan::send(UError::Panic(panic_info.to_string())) })); - tokio::spawn(error_reporting(_cli)); + tokio::spawn(error_reporting(client.clone())); do_stuff(client).await; } diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 1746585..511cdbf 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -11,10 +11,8 @@ structopt = "0.3.21" log = "^0.4" env_logger = "0.7.1" uuid = "0.6.5" -reqwest = { version = "0.11", features = ["json"] } -openssl = "*" -u_lib = { version = "*", path = "../../lib/u_lib" } serde_json = "1.0.4" serde = { version = "1.0.114", features = ["derive"] } -actix-web = "3.3.2" tokio = "1.11.0" +be = { version = "*", path = "./be" } +u_lib = { version = "*", path = "../../lib/u_lib" } \ No newline at end of file diff --git a/bin/u_panel/be/Cargo.toml b/bin/u_panel/be/Cargo.toml new file mode 100644 index 0000000..8831379 --- /dev/null +++ b/bin/u_panel/be/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "be" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +actix-web = "3.3.2" +u_lib = { version = "*", path = "../../../lib/u_lib" } \ No newline at end of file diff --git a/bin/u_panel/src/server/app.rs b/bin/u_panel/be/src/lib.rs similarity index 100% rename from bin/u_panel/src/server/app.rs rename to bin/u_panel/be/src/lib.rs diff --git a/bin/u_panel/fe/Cargo.toml b/bin/u_panel/fe/Cargo.toml new file mode 100644 index 0000000..f84fa64 --- /dev/null +++ b/bin/u_panel/fe/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "fe" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +u_lib = { version = "*", path = "../../../lib/u_lib" } +wasm-bindgen = "0.2.78" +yew = "0.18.0" +yew-router = "0.15.0" diff --git a/bin/u_panel/fe/index.html b/bin/u_panel/fe/index.html new file mode 100644 index 0000000..de24c98 --- /dev/null +++ b/bin/u_panel/fe/index.html @@ -0,0 +1,9 @@ + + + + + + Yew App + + + \ No newline at end of file diff --git a/bin/u_panel/fe/src/main.rs b/bin/u_panel/fe/src/main.rs new file mode 100644 index 0000000..c7e8f03 --- /dev/null +++ b/bin/u_panel/fe/src/main.rs @@ -0,0 +1,51 @@ +use wasm_bindgen::prelude::*; +use yew::prelude::*; +enum Msg { + AddOne, +} +struct Model { + // `ComponentLink` is like a reference to a component. + // It can be used to send messages to the component + link: ComponentLink, + value: i64, +} + +impl Component for Model { + type Message = Msg; + type Properties = (); + + fn create(_props: Self::Properties, link: ComponentLink) -> Self { + Self { link, value: 0 } + } + + fn update(&mut self, msg: Self::Message) -> ShouldRender { + match msg { + Msg::AddOne => { + self.value += 1; + // the value has changed so we need to + // re-render for it to appear on the page + true + } + } + } + + fn change(&mut self, _props: Self::Properties) -> ShouldRender { + // Should only return "true" if new properties are different to + // previously received properties. + // This component has no properties so we will always return "false". + false + } + + fn view(&self) -> Html { + html! { +
+ +

{ self.value }

+
+ } + } +} + +pub fn main() { + yew::start_app::(); +} diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 9267de4..03cfb2b 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -131,7 +131,7 @@ pub async fn process_cmd(args: Args) -> UResult<()> { JobMapALD::List { uid } => printer.print(cli_handler.get_agent_jobs(uid).await), JobMapALD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await), }, - Cmd::Server => crate::server::serve().unwrap(), + Cmd::Server => be::serve().unwrap(), } Ok(()) } diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index 828a67f..0ab0a09 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -1,5 +1,4 @@ mod argparse; -mod server; use argparse::{process_cmd, Args}; use std::process; diff --git a/bin/u_panel/src/server/mod.rs b/bin/u_panel/src/server/mod.rs deleted file mode 100644 index a4fad6b..0000000 --- a/bin/u_panel/src/server/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod app; - -pub use app::serve; diff --git a/integration/tests/tests.rs b/integration/tests/tests.rs index 35a4789..32fcc9f 100644 --- a/integration/tests/tests.rs +++ b/integration/tests/tests.rs @@ -3,19 +3,19 @@ mod fixtures; mod helpers; use std::env; +use u_lib::config::MASTER_PORT; #[macro_use] extern crate rstest; -// TODO: huita -#[ignore] #[tokio::test] async fn test_non_auth_connection_dropped() { let env_server = env::var("U_SERVER").unwrap(); - match reqwest::get(format!("https://{}", env_server)).await { + match reqwest::get(format!("https://{}:{}", env_server, MASTER_PORT)).await { Err(e) => { - dbg!(e.to_string()); - assert!(e.is_request()) + assert!(e + .to_string() + .contains("unable to get local issuer certificate")) } _ => panic!("no error occured on foreign client connection"), } diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index 6d66582..a2afaf6 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -13,11 +13,7 @@ uuid = { version = "0.6.5", features = ["serde", "v4"] } nix = "0.17" libc = "^0.2" lazy_static = "1.4.0" -tokio = { version = "1.2.0", features = ["rt-multi-thread", "sync", "macros", "process", "time"] } -reqwest = { version = "0.11", features = ["json", "native-tls"] } -openssl = "*" futures = "0.3.5" -guess_host_triple = "0.1.2" thiserror = "*" log = "*" mockall = "0.9.1" @@ -29,11 +25,14 @@ once_cell = "1.7.2" shlex = "1.0.0" u_api_proc_macro = { version = "*", path = "../u_api_proc_macro" } crossbeam = "0.8.1" -backtrace = "0.3.61" +backtrace = "0.3.61" +diesel = { version = "1.4.5", features = ["postgres", "uuid"] } -[dependencies.diesel] -version = "1.4.5" -features = ["postgres", "uuid"] +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +reqwest = { version = "0.11", features = ["json", "native-tls"] } +tokio = { version = "1.2.0", features = ["rt-multi-thread", "sync", "macros", "process", "time"] } +guess_host_triple = "0.1.2" +openssl = "*" [dev-dependencies] test-case = "1.1.0" diff --git a/lib/u_lib/src/config.rs b/lib/u_lib/src/config.rs index 1fd815c..ed9052d 100644 --- a/lib/u_lib/src/config.rs +++ b/lib/u_lib/src/config.rs @@ -1,7 +1,7 @@ use lazy_static::lazy_static; use uuid::Uuid; -pub const MASTER_SERVER: &str = "127.0.0.1"; //Ipv4Addr::new(3,9,16,40) +pub const MASTER_SERVER: &str = "ortem.xyz"; //Ipv4Addr::new(3,9,16,40) pub const MASTER_PORT: u16 = 63714; lazy_static! { diff --git a/lib/u_lib/src/errors/variants.rs b/lib/u_lib/src/errors/variants.rs index 2b2570d..0819a96 100644 --- a/lib/u_lib/src/errors/variants.rs +++ b/lib/u_lib/src/errors/variants.rs @@ -1,5 +1,6 @@ use backtrace::Backtrace as CrateBacktrace; use diesel::result::Error as DslError; +#[cfg(not(target_arch = "wasm32"))] use reqwest::Error as ReqError; use serde::{Deserialize, Serialize}; use std::fmt; @@ -72,12 +73,14 @@ impl UError { } } +#[cfg(not(target_arch = "wasm32"))] impl From for UErrorBt { fn from(e: ReqError) -> Self { UError::from(e).into_bt() } } +#[cfg(not(target_arch = "wasm32"))] impl From for UError { fn from(e: ReqError) -> Self { UError::NetError(e.to_string(), String::new()) diff --git a/lib/u_lib/src/lib.rs b/lib/u_lib/src/lib.rs index c614b47..1f37843 100644 --- a/lib/u_lib/src/lib.rs +++ b/lib/u_lib/src/lib.rs @@ -1,17 +1,32 @@ #![allow(non_upper_case_globals)] -pub mod api; -pub mod builder; -pub mod cache; -pub mod config; -pub mod datatypes; -pub mod errors; -pub mod executor; -pub mod messaging; -pub mod models; -pub mod utils; +#[cfg(not(target_arch = "wasm32"))] +#[path = "."] +pub mod exports { + pub mod api; + pub mod builder; + pub mod cache; + pub mod config; + pub mod datatypes; + pub mod errors; + pub mod executor; + pub mod messaging; + pub mod models; + pub mod utils; +} + +#[cfg(target_arch = "wasm32")] +#[path = "."] +pub mod exports { + pub mod config; + pub mod errors; + pub mod messaging; + pub mod models; + pub mod utils; +} pub use config::UID; pub use errors::{UError, UErrorBt, ULocalError, ULocalResult, UResult}; +pub use exports::*; pub mod schema_exports { pub use crate::models::{Agentstate, Jobstate, Jobtype}; diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index daa49a0..7c238a8 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -4,12 +4,10 @@ use serde::{Deserialize, Serialize}; use std::{fmt, time::SystemTime}; use strum::Display; -use crate::{ - builder::NamedJobBuilder, messaging::Reportable, models::schema::*, unwrap_enum, - utils::systime_to_string, UID, -}; +#[cfg(not(target_arch = "wasm32"))] +use crate::builder::NamedJobBuilder; +use crate::{messaging::Reportable, models::schema::*, unwrap_enum, utils::systime_to_string, UID}; -use guess_host_triple::guess_host_triple; use uuid::Uuid; #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, DbEnum, Display)] @@ -65,6 +63,7 @@ impl fmt::Display for Agent { } } +#[cfg(not(target_arch = "wasm32"))] impl Agent { pub fn with_id(uid: Uuid) -> Self { Self { @@ -73,6 +72,7 @@ impl Agent { } } + #[cfg(unix)] pub async fn gather() -> Self { let mut builder = NamedJobBuilder::from_shell(vec![ ("hostname", "hostname"), @@ -91,13 +91,14 @@ impl Agent { hostname: decoder(builder.pop("hostname")), is_root: &decoder(builder.pop("is_root")) == "0", username: decoder(builder.pop("username")), - platform: guess_host_triple().unwrap_or("unknown").to_string(), + platform: guess_host_triple::guess_host_triple() + .unwrap_or("unknown") + .to_string(), ..Default::default() } } pub async fn run() -> Reportable { - #[cfg(unix)] Reportable::Agent(Agent::gather().await) } } diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index 1b0b3a9..3182796 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -1,16 +1,16 @@ use super::JobState; +#[cfg(not(target_arch = "wasm32"))] +use crate::{cache::JobCache, utils::TempFile}; use crate::{ - cache::JobCache, errors::UError, messaging::Reportable, models::schema::*, - utils::{systime_to_string, ProcOutput, TempFile}, + utils::{systime_to_string, ProcOutput}, UID, }; use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; use std::{fmt, time::SystemTime}; -use tokio::process::Command; use uuid::Uuid; #[derive( @@ -75,8 +75,10 @@ impl Default for AssignedJob { } } +#[cfg(not(target_arch = "wasm32"))] impl AssignedJob { pub async fn run(mut self) -> Reportable { + use tokio::process::Command; let (argv, _payload) = { let meta = JobCache::get(&self.job_id).unwrap(); if let Some(ref payload) = meta.payload { diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 2a12e06..939d256 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -1,7 +1,6 @@ use super::JobType; use crate::{models::schema::*, utils::Stripped, UError, UResult}; use diesel::{Identifiable, Insertable, Queryable}; -use guess_host_triple::guess_host_triple; use serde::{Deserialize, Serialize}; use std::fmt; use std::str::from_utf8; @@ -61,7 +60,12 @@ impl Default for JobMeta { alias: None, argv: String::new(), exec_type: JobType::Shell, - platform: guess_host_triple().unwrap_or("unknown").to_string(), + #[cfg(not(target_arch = "wasm32"))] + platform: guess_host_triple::guess_host_triple() + .unwrap_or("unknown") + .to_string(), + #[cfg(target_arch = "wasm32")] + platform: "unknown".to_string(), payload: None, } } diff --git a/lib/u_lib/src/utils/misc.rs b/lib/u_lib/src/utils/misc.rs index 3025f24..8981bd3 100644 --- a/lib/u_lib/src/utils/misc.rs +++ b/lib/u_lib/src/utils/misc.rs @@ -1,9 +1,3 @@ -use nix::{ - sys::signal::{signal, SigHandler, Signal}, - unistd::{chdir, close as fdclose, fork, getppid, setsid, ForkResult}, -}; -use std::process::exit; - pub trait OneOrVec { fn into_vec(self) -> Vec; } @@ -31,38 +25,6 @@ macro_rules! unwrap_enum { }; } -pub fn daemonize() { - if getppid().as_raw() != 1 { - setsig(Signal::SIGTTOU, SigHandler::SigIgn); - setsig(Signal::SIGTTIN, SigHandler::SigIgn); - setsig(Signal::SIGTSTP, SigHandler::SigIgn); - } - for fd in 0..=2 { - match fdclose(fd) { - _ => (), - } - } - match chdir("/") { - _ => (), - }; - - match fork() { - Ok(ForkResult::Parent { .. }) => { - exit(0); - } - Ok(ForkResult::Child) => match setsid() { - _ => (), - }, - Err(_) => exit(255), - } -} - -pub fn setsig(sig: Signal, hnd: SigHandler) { - unsafe { - signal(sig, hnd).unwrap(); - } -} - pub fn init_env() { let envs = [".env", ".env.private"]; for envfile in &envs { diff --git a/lib/u_lib/src/utils/mod.rs b/lib/u_lib/src/utils/mod.rs index 41cf1a2..139fd5a 100644 --- a/lib/u_lib/src/utils/mod.rs +++ b/lib/u_lib/src/utils/mod.rs @@ -4,7 +4,10 @@ mod fmt; mod misc; mod proc_output; mod storage; +#[cfg(not(target_arch = "wasm32"))] mod tempfile; +#[cfg(unix)] +mod unix; mod vec_display; pub use combined_result::*; @@ -13,5 +16,9 @@ pub use fmt::*; pub use misc::*; pub use proc_output::*; pub use storage::*; +#[cfg(not(target_arch = "wasm32"))] pub use tempfile::*; pub use vec_display::*; + +#[cfg(unix)] +pub use unix::*; diff --git a/lib/u_lib/src/utils/unix.rs b/lib/u_lib/src/utils/unix.rs new file mode 100644 index 0000000..a4e4133 --- /dev/null +++ b/lib/u_lib/src/utils/unix.rs @@ -0,0 +1,37 @@ +use nix::{ + sys::signal::{signal, SigHandler, Signal}, + unistd::{chdir, close as fdclose, fork, getppid, setsid, ForkResult}, +}; +use std::process::exit; + +pub fn daemonize() { + if getppid().as_raw() != 1 { + setsig(Signal::SIGTTOU, SigHandler::SigIgn); + setsig(Signal::SIGTTIN, SigHandler::SigIgn); + setsig(Signal::SIGTSTP, SigHandler::SigIgn); + } + for fd in 0..=2 { + match fdclose(fd) { + _ => (), + } + } + match chdir("/") { + _ => (), + }; + + match fork() { + Ok(ForkResult::Parent { .. }) => { + exit(0); + } + Ok(ForkResult::Child) => match setsid() { + _ => (), + }, + Err(_) => exit(255), + } +} + +pub fn setsig(sig: Signal, hnd: SigHandler) { + unsafe { + signal(sig, hnd).unwrap(); + } +} diff --git a/scripts/deploy.sh b/scripts/deploy.sh index aa04c86..0804965 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -9,7 +9,8 @@ RSYNC="rsync -arzh --progress" ssh $SERVER mkdir -p $REMOTE_DIR/{release,deploy} $RSYNC $ROOTDIR/release/u_server $REMOTE_PATH/release/u_server -$RSYNC --exclude="*.sh" $ROOTDIR/certs/ $REMOTE_PATH/certs +$RSYNC $ROOTDIR/certs/server.{crt,key} $REMOTE_PATH/certs +$RSYNC $ROOTDIR/certs/ca.crt $REMOTE_PATH/certs $RSYNC $ROOTDIR/migrations/ $REMOTE_PATH/migrations $RSYNC $ROOTDIR/.env* $REMOTE_PATH/ $RSYNC $ROOTDIR/integration/docker-compose.yml $REMOTE_PATH/deploy/ From ec3f78b8cdb4cdc73c9d679d3861d695bf04b8f8 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Mon, 25 Oct 2021 02:16:36 +0500 Subject: [PATCH 11/30] fuck web, tui init --- Cargo.toml | 4 +- bin/u_panel/Cargo.toml | 8 ++- bin/u_panel/src/argparse.rs | 7 +- bin/u_panel/src/main.rs | 1 + bin/u_panel/src/tui/mod.rs | 82 +++++++++++++++++++++++ bin/u_panel/src/tui/state.rs | 53 +++++++++++++++ bin/u_panel/src/tui/ui.rs | 26 +++++++ integration/tests/tests.rs | 14 ++-- lib/u_lib/src/errors/variants.rs | 3 + {integration => scripts}/get_docker_ip.sh | 0 10 files changed, 187 insertions(+), 11 deletions(-) create mode 100644 bin/u_panel/src/tui/mod.rs create mode 100644 bin/u_panel/src/tui/state.rs create mode 100644 bin/u_panel/src/tui/ui.rs rename {integration => scripts}/get_docker_ip.sh (100%) diff --git a/Cargo.toml b/Cargo.toml index 9cb212d..c22386b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,8 +2,8 @@ members = [ "bin/u_agent", "bin/u_panel", - "bin/u_panel/be", - "bin/u_panel/fe", + #"bin/u_panel/be", + #"bin/u_panel/fe", "bin/u_run", "bin/u_server", "lib/u_lib", diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 511cdbf..c357f5a 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -14,5 +14,9 @@ uuid = "0.6.5" serde_json = "1.0.4" serde = { version = "1.0.114", features = ["derive"] } tokio = "1.11.0" -be = { version = "*", path = "./be" } -u_lib = { version = "*", path = "../../lib/u_lib" } \ No newline at end of file +# be = { version = "*", path = "./be" } +u_lib = { version = "*", path = "../../lib/u_lib" } +tui = { version = "0.16", default-features = false, features = ['crossterm'] } +crossterm = "0.22.1" +anyhow = "1.0.44" +strum = { version = "0.22.0", features = ["derive"] } diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 03cfb2b..b3df203 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -19,7 +19,7 @@ enum Cmd { Agents(LD), Jobs(JobALD), Jobmap(JobMapALD), - Server, + TUI, } #[derive(StructOpt, Debug)] @@ -101,7 +101,7 @@ pub async fn process_cmd(args: Args) -> UResult<()> { } let token = env::var("ADMIN_AUTH_TOKEN").map_err(|_| UError::WrongToken)?; - let cli_handler = ClientHandler::new(None).password(token); + let cli_handler = ClientHandler::new(None).password(token.clone()); let printer = Printer { json: args.json }; match args.cmd { Cmd::Agents(action) => match action { @@ -131,7 +131,8 @@ pub async fn process_cmd(args: Args) -> UResult<()> { JobMapALD::List { uid } => printer.print(cli_handler.get_agent_jobs(uid).await), JobMapALD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await), }, - Cmd::Server => be::serve().unwrap(), + //Cmd::Server => be::serve().unwrap(), + Cmd::TUI => crate::tui::init_tui(token).map_err(|e| UError::TUIError(e.to_string()))?, } Ok(()) } diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index 0ab0a09..65fe81d 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -1,4 +1,5 @@ mod argparse; +mod tui; use argparse::{process_cmd, Args}; use std::process; diff --git a/bin/u_panel/src/tui/mod.rs b/bin/u_panel/src/tui/mod.rs new file mode 100644 index 0000000..2cd7c6b --- /dev/null +++ b/bin/u_panel/src/tui/mod.rs @@ -0,0 +1,82 @@ +mod state; +mod ui; + +use anyhow::Result; +use crossterm::event::{self, DisableMouseCapture, EnableMouseCapture, Event, KeyCode}; +use crossterm::execute; +use crossterm::terminal::{ + disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen, +}; +use state::State; +use std::panic::set_hook; +use std::process::exit; +use std::{ + io::{stdout, Stdout}, + sync::mpsc, + thread, + time::Duration, +}; +use tui::{backend::CrosstermBackend, Terminal}; + +type Frame<'f> = tui::Frame<'f, CrosstermBackend>; + +pub fn init_tui(token: String) -> Result<()> { + //TODO: fix this + set_hook(Box::new(|p| { + teardown().unwrap(); + eprintln!("{}", p); + exit(254); + })); + let mut state = State::new(token); + if let Err(e) = init(&mut state) { + teardown()?; + return Err(e); + } + Ok(()) +} + +fn init(state: &mut State) -> Result<()> { + let mut stdout = stdout(); + enable_raw_mode()?; + execute!(&mut stdout, EnterAlternateScreen, EnableMouseCapture)?; + + let backend = CrosstermBackend::new(stdout); + let mut terminal = Terminal::new(backend)?; + let (tx, rx) = mpsc::channel(); + + thread::spawn(move || loop { + if event::poll(Duration::from_millis(10)).unwrap() { + match event::read().unwrap() { + key @ Event::Key(_) => tx.send(key).unwrap(), + _ => (), + } + } + }); + + terminal.clear()?; + + loop { + terminal.draw(|f| ui::draw(f, state))?; + match rx.recv()? { + Event::Key(key) => match key.code { + KeyCode::Esc => { + teardown()?; + terminal.show_cursor()?; + break; + } + KeyCode::Left => state.prev_tab(), + KeyCode::Right => state.next_tab(), + _ => (), + }, + _ => unreachable!(), + } + } + Ok(()) +} + +fn teardown() -> Result<()> { + disable_raw_mode()?; + execute!(stdout(), LeaveAlternateScreen, DisableMouseCapture)?; + eprintln!("teardown"); + Ok(()) +} diff --git a/bin/u_panel/src/tui/state.rs b/bin/u_panel/src/tui/state.rs new file mode 100644 index 0000000..d72fbb4 --- /dev/null +++ b/bin/u_panel/src/tui/state.rs @@ -0,0 +1,53 @@ +use std::str::FromStr; +use strum::VariantNames; + +#[derive(strum::Display, strum::EnumVariantNames, strum::EnumString)] +pub enum UiTabs { + Agents, + Jobs, + Map, +} + +impl UiTabs { + pub fn variants() -> &'static [&'static str] { + Self::VARIANTS + } + + pub fn index(&self) -> usize { + let ss = self.to_string(); + Self::VARIANTS.iter().position(|el| **el == ss).unwrap() + } + + pub fn next(&self) -> Self { + let next_idx = (self.index() + 1) % Self::VARIANTS.len(); + Self::from_str(Self::VARIANTS[next_idx]).unwrap() + } + + pub fn prev(&self) -> Self { + let vlen = Self::VARIANTS.len(); + let next_idx = (self.index() + vlen - 1) % vlen; + Self::from_str(Self::VARIANTS[next_idx]).unwrap() + } +} + +pub struct State { + pub token: String, + pub active_tab: UiTabs, +} + +impl State { + pub fn new(token: String) -> Self { + State { + token, + active_tab: UiTabs::Agents, + } + } + + pub fn next_tab(&mut self) { + self.active_tab = self.active_tab.next() + } + + pub fn prev_tab(&mut self) { + self.active_tab = self.active_tab.prev() + } +} diff --git a/bin/u_panel/src/tui/ui.rs b/bin/u_panel/src/tui/ui.rs new file mode 100644 index 0000000..0341afe --- /dev/null +++ b/bin/u_panel/src/tui/ui.rs @@ -0,0 +1,26 @@ +use super::{ + state::{State, UiTabs}, + Frame, +}; +use tui::style::{Color, Style}; +use tui::text::Spans; +use tui::widgets::{Block, Borders, Tabs}; + +pub fn draw(f: &mut Frame, s: &mut State) { + let titles = UiTabs::variants() + .iter() + .cloned() + .map(Spans::from) + .collect(); + let tabs = Tabs::new(titles) + .block( + Block::default() + .title("The whole that you need to know") + .borders(Borders::ALL), + ) + .style(Style::default().fg(Color::White)) + .highlight_style(Style::default().fg(Color::Yellow)) + .divider("-") + .select(s.active_tab.index()); + f.render_widget(tabs, f.size()) +} diff --git a/integration/tests/tests.rs b/integration/tests/tests.rs index 32fcc9f..39293f6 100644 --- a/integration/tests/tests.rs +++ b/integration/tests/tests.rs @@ -11,11 +11,17 @@ extern crate rstest; #[tokio::test] async fn test_non_auth_connection_dropped() { let env_server = env::var("U_SERVER").unwrap(); - match reqwest::get(format!("https://{}:{}", env_server, MASTER_PORT)).await { + let client = reqwest::ClientBuilder::new() + .danger_accept_invalid_certs(true) + .build() + .unwrap(); + match client + .get(format!("https://{}:{}", env_server, MASTER_PORT)) + .send() + .await + { Err(e) => { - assert!(e - .to_string() - .contains("unable to get local issuer certificate")) + assert!(e.to_string().contains("channel closed")) } _ => panic!("no error occured on foreign client connection"), } diff --git a/lib/u_lib/src/errors/variants.rs b/lib/u_lib/src/errors/variants.rs index 0819a96..d84db3a 100644 --- a/lib/u_lib/src/errors/variants.rs +++ b/lib/u_lib/src/errors/variants.rs @@ -65,6 +65,9 @@ pub enum UError { #[error("Panicked: {0}")] Panic(String), + + #[error("UI init error: {0}")] + TUIError(String), } impl UError { diff --git a/integration/get_docker_ip.sh b/scripts/get_docker_ip.sh similarity index 100% rename from integration/get_docker_ip.sh rename to scripts/get_docker_ip.sh From c8dc747bccd76c00eade2a4240878393cf4fc4ed Mon Sep 17 00:00:00 2001 From: plazmoid Date: Tue, 26 Oct 2021 12:03:43 +0500 Subject: [PATCH 12/30] some ui progress --- bin/u_panel/Cargo.toml | 2 + bin/u_panel/src/argparse.rs | 28 +++-- bin/u_panel/src/main.rs | 3 + bin/u_panel/src/tui/impls.rs | 63 +++++++++++ bin/u_panel/src/tui/mod.rs | 29 ++++-- bin/u_panel/src/tui/state.rs | 195 ++++++++++++++++++++++++++++++++++- bin/u_panel/src/tui/ui.rs | 23 ++++- 7 files changed, 318 insertions(+), 25 deletions(-) create mode 100644 bin/u_panel/src/tui/impls.rs diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index c357f5a..50395a0 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -20,3 +20,5 @@ tui = { version = "0.16", default-features = false, features = ['crossterm'] } crossterm = "0.22.1" anyhow = "1.0.44" strum = { version = "0.22.0", features = ["derive"] } +async-trait = "0.1.51" +once_cell = "1.8.0" diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index b3df203..dec2ccb 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -1,3 +1,4 @@ +use once_cell::sync::Lazy; use std::env; use std::fmt; use structopt::StructOpt; @@ -74,6 +75,11 @@ enum LD { }, } +pub static CLIENT: Lazy = Lazy::new(|| { + let token = env::var("ADMIN_AUTH_TOKEN").expect("access token is not set"); + ClientHandler::new(None).password(token.clone()) +}); + fn parse_uuid(src: &str) -> Result { Uuid::parse_str(src).map_err(|e| e.to_string()) } @@ -100,13 +106,11 @@ pub async fn process_cmd(args: Args) -> UResult<()> { } } - let token = env::var("ADMIN_AUTH_TOKEN").map_err(|_| UError::WrongToken)?; - let cli_handler = ClientHandler::new(None).password(token.clone()); let printer = Printer { json: args.json }; match args.cmd { Cmd::Agents(action) => match action { - LD::List { uid } => printer.print(cli_handler.get_agents(uid).await), - LD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await), + LD::List { uid } => printer.print(CLIENT.get_agents(uid).await), + LD::Delete { uid } => printer.print(CLIENT.del(Some(uid)).await), }, Cmd::Jobs(action) => match action { JobALD::Add { @@ -118,21 +122,23 @@ pub async fn process_cmd(args: Args) -> UResult<()> { .with_shell(cmd.join(" ")) .with_alias(alias) .build()?; - printer.print(cli_handler.upload_jobs(&[job]).await); + printer.print(CLIENT.upload_jobs(&[job]).await); } - JobALD::LD(LD::List { uid }) => printer.print(cli_handler.get_jobs(uid).await), - JobALD::LD(LD::Delete { uid }) => printer.print(cli_handler.del(Some(uid)).await), + JobALD::LD(LD::List { uid }) => printer.print(CLIENT.get_jobs(uid).await), + JobALD::LD(LD::Delete { uid }) => printer.print(CLIENT.del(Some(uid)).await), }, Cmd::Jobmap(action) => match action { JobMapALD::Add { agent_uid, job_idents, - } => printer.print(cli_handler.set_jobs(Some(agent_uid), &job_idents).await), - JobMapALD::List { uid } => printer.print(cli_handler.get_agent_jobs(uid).await), - JobMapALD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await), + } => printer.print(CLIENT.set_jobs(Some(agent_uid), &job_idents).await), + JobMapALD::List { uid } => printer.print(CLIENT.get_agent_jobs(uid).await), + JobMapALD::Delete { uid } => printer.print(CLIENT.del(Some(uid)).await), }, //Cmd::Server => be::serve().unwrap(), - Cmd::TUI => crate::tui::init_tui(token).map_err(|e| UError::TUIError(e.to_string()))?, + Cmd::TUI => crate::tui::init_tui() + .await + .map_err(|e| UError::TUIError(e.to_string()))?, } Ok(()) } diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index 65fe81d..b48a2c1 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -1,6 +1,9 @@ mod argparse; mod tui; +#[macro_use] +extern crate async_trait; + use argparse::{process_cmd, Args}; use std::process; use structopt::StructOpt; diff --git a/bin/u_panel/src/tui/impls.rs b/bin/u_panel/src/tui/impls.rs new file mode 100644 index 0000000..5b660d1 --- /dev/null +++ b/bin/u_panel/src/tui/impls.rs @@ -0,0 +1,63 @@ +use crate::argparse::CLIENT; +use u_lib::models::{Agent, AssignedJob, JobMeta}; +use u_lib::UResult; +use uuid::Uuid; + +pub trait Id { + fn id(&self) -> Uuid; +} + +impl Id for Agent { + fn id(&self) -> Uuid { + self.id + } +} + +impl Id for JobMeta { + fn id(&self) -> Uuid { + self.id + } +} + +impl Id for AssignedJob { + fn id(&self) -> Uuid { + self.id + } +} + +#[async_trait] +pub trait CRUD: Id +where + Self: Sized, +{ + async fn read() -> UResult>; + + async fn delete(uid: Uuid) -> UResult { + CLIENT.del(Some(uid)).await + } + //TODO: other crud +} + +#[async_trait] +impl CRUD for Agent { + async fn read() -> UResult> { + CLIENT.get_agents(None).await.map(|r| r.into_builtin_vec()) + } +} + +#[async_trait] +impl CRUD for AssignedJob { + async fn read() -> UResult> { + CLIENT + .get_agent_jobs(None) + .await + .map(|r| r.into_builtin_vec()) + } +} + +#[async_trait] +impl CRUD for JobMeta { + async fn read() -> UResult> { + CLIENT.get_jobs(None).await.map(|r| r.into_builtin_vec()) + } +} diff --git a/bin/u_panel/src/tui/mod.rs b/bin/u_panel/src/tui/mod.rs index 2cd7c6b..5742407 100644 --- a/bin/u_panel/src/tui/mod.rs +++ b/bin/u_panel/src/tui/mod.rs @@ -1,3 +1,4 @@ +mod impls; mod state; mod ui; @@ -20,22 +21,27 @@ use tui::{backend::CrosstermBackend, Terminal}; type Frame<'f> = tui::Frame<'f, CrosstermBackend>; -pub fn init_tui(token: String) -> Result<()> { +enum InputEvent { + Key(I), + Tick, +} + +pub async fn init_tui() -> Result<()> { //TODO: fix this set_hook(Box::new(|p| { teardown().unwrap(); eprintln!("{}", p); exit(254); })); - let mut state = State::new(token); - if let Err(e) = init(&mut state) { + let mut state = State::default(); + if let Err(e) = init(&mut state).await { teardown()?; return Err(e); } Ok(()) } -fn init(state: &mut State) -> Result<()> { +async fn init(state: &mut State) -> Result<()> { let mut stdout = stdout(); enable_raw_mode()?; execute!(&mut stdout, EnterAlternateScreen, EnableMouseCapture)?; @@ -47,18 +53,21 @@ fn init(state: &mut State) -> Result<()> { thread::spawn(move || loop { if event::poll(Duration::from_millis(10)).unwrap() { match event::read().unwrap() { - key @ Event::Key(_) => tx.send(key).unwrap(), + key @ Event::Key(_) => tx.send(InputEvent::Key(key)).unwrap(), _ => (), } + } else { + tx.send(InputEvent::Tick).unwrap() } }); terminal.clear()?; loop { + state.check_updates().await; terminal.draw(|f| ui::draw(f, state))?; match rx.recv()? { - Event::Key(key) => match key.code { + InputEvent::Key(Event::Key(key)) => match key.code { KeyCode::Esc => { teardown()?; terminal.show_cursor()?; @@ -66,8 +75,16 @@ fn init(state: &mut State) -> Result<()> { } KeyCode::Left => state.prev_tab(), KeyCode::Right => state.next_tab(), + KeyCode::Up => state.on_up(), + KeyCode::Down => state.on_down(), + KeyCode::Delete => { + state.delete().await; + state.update_tab(); + } + KeyCode::F(5) => state.update_tab(), _ => (), }, + InputEvent::Tick => (), _ => unreachable!(), } } diff --git a/bin/u_panel/src/tui/state.rs b/bin/u_panel/src/tui/state.rs index d72fbb4..57b1109 100644 --- a/bin/u_panel/src/tui/state.rs +++ b/bin/u_panel/src/tui/state.rs @@ -1,5 +1,11 @@ -use std::str::FromStr; +use super::impls::CRUD; +use anyhow::Result as AResult; +use std::{fmt::Display, str::FromStr}; use strum::VariantNames; +use tokio::join; +use tui::widgets::ListState; +use u_lib::models::{Agent, AssignedJob, JobMeta}; +use uuid::Uuid; #[derive(strum::Display, strum::EnumVariantNames, strum::EnumString)] pub enum UiTabs { @@ -30,19 +36,73 @@ impl UiTabs { } } +pub struct StatefulList { + pub updated: bool, + pub inner: Vec, + pub state: ListState, +} + +impl StatefulList { + pub async fn update(&mut self) -> AResult<()> { + if !self.updated { + let new_values = ::read().await?; + self.inner = new_values; + self.updated = true; + } + Ok(()) + } + + pub async fn delete(&mut self) -> AResult<()> { + if let Some(s) = self.state.selected() { + let uid = self.inner[s].id(); + ::delete(uid).await?; + } + Ok(()) + } + + pub fn get(&self, id: Uuid) -> Option<&T> { + for item in self.inner.iter() { + if item.id() == id { + return Some(item); + } + } + None + } +} + +impl Default for StatefulList { + fn default() -> Self { + let mut state = ListState::default(); + state.select(Some(0)); + StatefulList { + updated: false, + inner: vec![], + state, + } + } +} + pub struct State { - pub token: String, pub active_tab: UiTabs, + pub last_error: Option, + pub agents: StatefulList, + pub jobs: StatefulList, + pub map: StatefulList, } -impl State { - pub fn new(token: String) -> Self { +impl Default for State { + fn default() -> Self { State { - token, active_tab: UiTabs::Agents, + last_error: None, + agents: Default::default(), + jobs: Default::default(), + map: Default::default(), } } +} +impl State { pub fn next_tab(&mut self) { self.active_tab = self.active_tab.next() } @@ -50,4 +110,129 @@ impl State { pub fn prev_tab(&mut self) { self.active_tab = self.active_tab.prev() } + + fn check_err(&mut self, res: AResult<()>) -> bool { + if let Err(e) = res { + self.last_error = Some(e.to_string()); + true + } else { + false + } + } + + pub async fn check_updates(&mut self) { + if !self.agents.updated || !self.jobs.updated || !self.map.updated { + let state = self.tab_list_state(); + if let None = state.selected() { + state.select(Some(0)) + } + + let (a, j, m) = join! { + self.agents.update(), + self.jobs.update(), + self.map.update() + }; + for res in [a, j, m] { + self.check_err(res); + } + } + } + + pub fn tab_data(&self) -> Vec { + match self.active_tab { + UiTabs::Agents => self + .agents + .inner + .iter() + .map(|i| format!("{}: {}-{}", crop(i.id, 6), i.username, i.hostname)) + .collect(), + UiTabs::Jobs => self + .jobs + .inner + .iter() + .map(|i| format!("{}: {}", crop(i.id, 6), i.alias.clone().unwrap_or_default())) + .collect(), + UiTabs::Map => self + .map + .inner + .iter() + .map(|i| { + let job = self.jobs.get(i.job_id).unwrap(); + let job_id = crop(i.job_id, 6); + let job_ident = if let Some(alias) = job.alias.as_ref() { + format!("{} ({})", alias, job_id) + } else { + format!("{}", job_id) + }; + let agent = self.agents.get(i.agent_id).unwrap(); + let agent_id = crop(i.agent_id, 6); + let agent_ident = if let Some(alias) = agent.alias.as_ref() { + format!("{} ({})", alias, agent_id) + } else { + format!("{}-{} ({})", agent.username, agent.hostname, agent_id) + }; + format!("{}: {} for {}", crop(i.id, 6), job_ident, agent_ident) + }) + .collect(), + } + } + + pub fn tab_list_state(&mut self) -> &mut ListState { + match self.active_tab { + UiTabs::Agents => &mut self.agents.state, + UiTabs::Jobs => &mut self.jobs.state, + UiTabs::Map => &mut self.map.state, + } + } + + pub fn update_tab(&mut self) { + match self.active_tab { + UiTabs::Agents => self.agents.updated = false, + UiTabs::Jobs => self.jobs.updated = false, + UiTabs::Map => self.map.updated = false, + } + } + + pub fn on_down(&mut self) { + let (list_len, list_state) = match self.active_tab { + UiTabs::Agents => (self.agents.inner.len(), &mut self.agents.state), + UiTabs::Jobs => (self.jobs.inner.len(), &mut self.jobs.state), + UiTabs::Map => (self.map.inner.len(), &mut self.map.state), + }; + if list_len == 0 { + list_state.select(None); + } else { + let selected = list_state.selected().unwrap_or(0); + list_state.select(Some((selected + 1) % list_len)); + } + } + + pub fn on_up(&mut self) { + let (list_len, list_state) = match self.active_tab { + UiTabs::Agents => (self.agents.inner.len(), &mut self.agents.state), + UiTabs::Jobs => (self.jobs.inner.len(), &mut self.jobs.state), + UiTabs::Map => (self.map.inner.len(), &mut self.map.state), + }; + if list_len == 0 { + list_state.select(None); + } else { + let selected = list_state.selected().unwrap_or(1); + list_state.select(Some((selected + list_len - 1) % list_len)); + } + } + + pub async fn delete(&mut self) { + let res = match self.active_tab { + UiTabs::Agents => self.agents.delete().await, + UiTabs::Jobs => self.jobs.delete().await, + UiTabs::Map => self.map.delete().await, + }; + if !self.check_err(res) { + self.on_up(); + } + } +} + +fn crop(data: T, retain: usize) -> String { + data.to_string()[..retain].to_string() } diff --git a/bin/u_panel/src/tui/ui.rs b/bin/u_panel/src/tui/ui.rs index 0341afe..8a97e65 100644 --- a/bin/u_panel/src/tui/ui.rs +++ b/bin/u_panel/src/tui/ui.rs @@ -2,11 +2,18 @@ use super::{ state::{State, UiTabs}, Frame, }; -use tui::style::{Color, Style}; +use tui::layout::{Constraint, Direction, Layout}; +use tui::style::{Color, Modifier, Style}; use tui::text::Spans; -use tui::widgets::{Block, Borders, Tabs}; +use tui::widgets::{Block, Borders, List, ListItem, Tabs}; pub fn draw(f: &mut Frame, s: &mut State) { + let size = f.size(); + let chunks = Layout::default() + .direction(Direction::Vertical) + .margin(1) + .constraints([Constraint::Length(3), Constraint::Min(0)].as_ref()) + .split(size); let titles = UiTabs::variants() .iter() .cloned() @@ -22,5 +29,15 @@ pub fn draw(f: &mut Frame, s: &mut State) { .highlight_style(Style::default().fg(Color::Yellow)) .divider("-") .select(s.active_tab.index()); - f.render_widget(tabs, f.size()) + f.render_widget(tabs, chunks[0]); + + let tab_data = s + .tab_data() + .into_iter() + .map(ListItem::new) + .collect::>(); + let list = List::new(tab_data) + .block(Block::default().borders(Borders::ALL)) + .highlight_style(Style::default().add_modifier(Modifier::BOLD)); + f.render_stateful_widget(list, chunks[1], s.tab_list_state()); } From 638ae4da6ede87c1159208880ff72815894bcbb5 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sun, 31 Oct 2021 07:03:33 +0500 Subject: [PATCH 13/30] new window manager --- bin/u_panel/Cargo.toml | 1 + bin/u_panel/src/argparse.rs | 4 +- bin/u_panel/src/tui/impls.rs | 30 +++--- bin/u_panel/src/tui/mod.rs | 65 ++++++------- bin/u_panel/src/tui/ui.rs | 43 --------- .../src/tui/{state.rs => windows/main_wnd.rs} | 85 ++++++++++++++++- bin/u_panel/src/tui/windows/mod.rs | 94 +++++++++++++++++++ bin/u_panel/src/tui/windows/processing.rs | 0 8 files changed, 217 insertions(+), 105 deletions(-) delete mode 100644 bin/u_panel/src/tui/ui.rs rename bin/u_panel/src/tui/{state.rs => windows/main_wnd.rs} (73%) create mode 100644 bin/u_panel/src/tui/windows/mod.rs create mode 100644 bin/u_panel/src/tui/windows/processing.rs diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 50395a0..14442d0 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +backtrace = "0.3.61" structopt = "0.3.21" log = "^0.4" env_logger = "0.7.1" diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index dec2ccb..53c93bc 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -19,7 +19,7 @@ pub struct Args { enum Cmd { Agents(LD), Jobs(JobALD), - Jobmap(JobMapALD), + Map(JobMapALD), TUI, } @@ -127,7 +127,7 @@ pub async fn process_cmd(args: Args) -> UResult<()> { JobALD::LD(LD::List { uid }) => printer.print(CLIENT.get_jobs(uid).await), JobALD::LD(LD::Delete { uid }) => printer.print(CLIENT.del(Some(uid)).await), }, - Cmd::Jobmap(action) => match action { + Cmd::Map(action) => match action { JobMapALD::Add { agent_uid, job_idents, diff --git a/bin/u_panel/src/tui/impls.rs b/bin/u_panel/src/tui/impls.rs index 5b660d1..9e29a48 100644 --- a/bin/u_panel/src/tui/impls.rs +++ b/bin/u_panel/src/tui/impls.rs @@ -7,23 +7,19 @@ pub trait Id { fn id(&self) -> Uuid; } -impl Id for Agent { - fn id(&self) -> Uuid { - self.id - } -} - -impl Id for JobMeta { - fn id(&self) -> Uuid { - self.id - } -} - -impl Id for AssignedJob { - fn id(&self) -> Uuid { - self.id - } -} +#[macro_export] +macro_rules! impl_id { + ($($type:ty),+) => { + $( + impl Id for $type { + fn id(&self) -> Uuid { + self.id + } + })+ + }; +} + +impl_id!(Agent, JobMeta, AssignedJob); #[async_trait] pub trait CRUD: Id diff --git a/bin/u_panel/src/tui/mod.rs b/bin/u_panel/src/tui/mod.rs index 5742407..8ffd734 100644 --- a/bin/u_panel/src/tui/mod.rs +++ b/bin/u_panel/src/tui/mod.rs @@ -1,14 +1,13 @@ mod impls; -mod state; -mod ui; +mod windows; -use anyhow::Result; -use crossterm::event::{self, DisableMouseCapture, EnableMouseCapture, Event, KeyCode}; +use anyhow::Result as AResult; +use backtrace::Backtrace; +use crossterm::event::{self, DisableMouseCapture, EnableMouseCapture, Event}; use crossterm::execute; use crossterm::terminal::{ disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen, }; -use state::State; use std::panic::set_hook; use std::process::exit; use std::{ @@ -18,36 +17,42 @@ use std::{ time::Duration, }; use tui::{backend::CrosstermBackend, Terminal}; +use windows::{MainWnd, WindowsHandler}; -type Frame<'f> = tui::Frame<'f, CrosstermBackend>; +pub type Backend = CrosstermBackend; +pub type Frame<'f> = tui::Frame<'f, Backend>; enum InputEvent { Key(I), Tick, } -pub async fn init_tui() -> Result<()> { +fn get_terminal() -> AResult> { + let backend = CrosstermBackend::new(stdout()); + Ok(Terminal::new(backend)?) +} + +pub async fn init_tui() -> AResult<()> { //TODO: fix this set_hook(Box::new(|p| { teardown().unwrap(); - eprintln!("{}", p); + get_terminal().unwrap().show_cursor().unwrap(); + eprintln!("{}\n{:?}", p, Backtrace::new()); exit(254); })); - let mut state = State::default(); - if let Err(e) = init(&mut state).await { + if let Err(e) = init().await { teardown()?; return Err(e); } Ok(()) } -async fn init(state: &mut State) -> Result<()> { - let mut stdout = stdout(); +async fn init() -> AResult<()> { enable_raw_mode()?; - execute!(&mut stdout, EnterAlternateScreen, EnableMouseCapture)?; + execute!(stdout(), EnterAlternateScreen, EnableMouseCapture)?; - let backend = CrosstermBackend::new(stdout); - let mut terminal = Terminal::new(backend)?; + let mut wh = WindowsHandler::new(get_terminal()?); + wh.push::(); let (tx, rx) = mpsc::channel(); thread::spawn(move || loop { @@ -61,39 +66,23 @@ async fn init(state: &mut State) -> Result<()> { } }); - terminal.clear()?; + wh.clear()?; loop { - state.check_updates().await; - terminal.draw(|f| ui::draw(f, state))?; + wh.draw()?; + wh.update().await?; match rx.recv()? { - InputEvent::Key(Event::Key(key)) => match key.code { - KeyCode::Esc => { - teardown()?; - terminal.show_cursor()?; - break; - } - KeyCode::Left => state.prev_tab(), - KeyCode::Right => state.next_tab(), - KeyCode::Up => state.on_up(), - KeyCode::Down => state.on_down(), - KeyCode::Delete => { - state.delete().await; - state.update_tab(); - } - KeyCode::F(5) => state.update_tab(), - _ => (), - }, + InputEvent::Key(Event::Key(key)) => { + wh.handle_kbd(key.code).await?; + } InputEvent::Tick => (), _ => unreachable!(), } } - Ok(()) } -fn teardown() -> Result<()> { +fn teardown() -> AResult<()> { disable_raw_mode()?; execute!(stdout(), LeaveAlternateScreen, DisableMouseCapture)?; - eprintln!("teardown"); Ok(()) } diff --git a/bin/u_panel/src/tui/ui.rs b/bin/u_panel/src/tui/ui.rs deleted file mode 100644 index 8a97e65..0000000 --- a/bin/u_panel/src/tui/ui.rs +++ /dev/null @@ -1,43 +0,0 @@ -use super::{ - state::{State, UiTabs}, - Frame, -}; -use tui::layout::{Constraint, Direction, Layout}; -use tui::style::{Color, Modifier, Style}; -use tui::text::Spans; -use tui::widgets::{Block, Borders, List, ListItem, Tabs}; - -pub fn draw(f: &mut Frame, s: &mut State) { - let size = f.size(); - let chunks = Layout::default() - .direction(Direction::Vertical) - .margin(1) - .constraints([Constraint::Length(3), Constraint::Min(0)].as_ref()) - .split(size); - let titles = UiTabs::variants() - .iter() - .cloned() - .map(Spans::from) - .collect(); - let tabs = Tabs::new(titles) - .block( - Block::default() - .title("The whole that you need to know") - .borders(Borders::ALL), - ) - .style(Style::default().fg(Color::White)) - .highlight_style(Style::default().fg(Color::Yellow)) - .divider("-") - .select(s.active_tab.index()); - f.render_widget(tabs, chunks[0]); - - let tab_data = s - .tab_data() - .into_iter() - .map(ListItem::new) - .collect::>(); - let list = List::new(tab_data) - .block(Block::default().borders(Borders::ALL)) - .highlight_style(Style::default().add_modifier(Modifier::BOLD)); - f.render_stateful_widget(list, chunks[1], s.tab_list_state()); -} diff --git a/bin/u_panel/src/tui/state.rs b/bin/u_panel/src/tui/windows/main_wnd.rs similarity index 73% rename from bin/u_panel/src/tui/state.rs rename to bin/u_panel/src/tui/windows/main_wnd.rs index 57b1109..4cebc8f 100644 --- a/bin/u_panel/src/tui/state.rs +++ b/bin/u_panel/src/tui/windows/main_wnd.rs @@ -1,5 +1,7 @@ -use super::impls::CRUD; +use super::Window; +use crate::tui::{impls::CRUD, Frame}; use anyhow::Result as AResult; +use crossterm::event::KeyCode; use std::{fmt::Display, str::FromStr}; use strum::VariantNames; use tokio::join; @@ -7,6 +9,11 @@ use tui::widgets::ListState; use u_lib::models::{Agent, AssignedJob, JobMeta}; use uuid::Uuid; +use tui::layout::{Constraint, Direction, Layout}; +use tui::style::{Color, Modifier, Style}; +use tui::text::Spans; +use tui::widgets::{Block, Borders, List, ListItem, Tabs}; + #[derive(strum::Display, strum::EnumVariantNames, strum::EnumString)] pub enum UiTabs { Agents, @@ -82,7 +89,7 @@ impl Default for StatefulList { } } -pub struct State { +pub struct MainWnd { pub active_tab: UiTabs, pub last_error: Option, pub agents: StatefulList, @@ -90,9 +97,9 @@ pub struct State { pub map: StatefulList, } -impl Default for State { +impl Default for MainWnd { fn default() -> Self { - State { + MainWnd { active_tab: UiTabs::Agents, last_error: None, agents: Default::default(), @@ -102,7 +109,7 @@ impl Default for State { } } -impl State { +impl MainWnd { pub fn next_tab(&mut self) { self.active_tab = self.active_tab.next() } @@ -233,6 +240,74 @@ impl State { } } +#[async_trait] +impl Window for MainWnd { + async fn handle_kbd(&mut self, k: KeyCode) -> AResult<()> { + match k { + KeyCode::Esc => { + /*teardown()?; + wh.show_cursor()?; + break;*/ + } + KeyCode::Left => self.prev_tab(), + KeyCode::Right => self.next_tab(), + KeyCode::Up => self.on_up(), + KeyCode::Down => self.on_down(), + KeyCode::Delete => { + self.delete().await; + self.update_tab(); + } + KeyCode::F(5) => self.update_tab(), + _ => (), + }; + Ok(()) + } + + async fn update(&mut self) -> AResult<()> { + self.check_updates().await; + Ok(()) + } + + async fn close(&mut self) { + //self.show_cursor() + } + + fn draw(&mut self, f: &mut Frame) { + let size = f.size(); + let chunks = Layout::default() + .direction(Direction::Vertical) + .margin(1) + .constraints([Constraint::Length(3), Constraint::Min(0)].as_ref()) + .split(size); + let titles = UiTabs::variants() + .iter() + .cloned() + .map(Spans::from) + .collect(); + let tabs = Tabs::new(titles) + .block( + Block::default() + .title("The whole that you need to know") + .borders(Borders::ALL), + ) + .style(Style::default().fg(Color::White)) + .highlight_style(Style::default().fg(Color::Yellow)) + .divider("-") + .select(self.active_tab.index()); + f.render_widget(tabs, chunks[0]); + + let tab_data = self + .tab_data() + .into_iter() + .map(ListItem::new) + .collect::>(); + let list = List::new(tab_data) + .block(Block::default().borders(Borders::ALL)) + .highlight_style(Style::default().add_modifier(Modifier::BOLD)); + f.render_stateful_widget(list, chunks[1], self.tab_list_state()); + } +} + fn crop(data: T, retain: usize) -> String { data.to_string()[..retain].to_string() } diff --git a/bin/u_panel/src/tui/windows/mod.rs b/bin/u_panel/src/tui/windows/mod.rs new file mode 100644 index 0000000..3cc7e31 --- /dev/null +++ b/bin/u_panel/src/tui/windows/mod.rs @@ -0,0 +1,94 @@ +mod main_wnd; +mod processing; +pub use main_wnd::MainWnd; + +use crate::tui::{Backend, Frame}; +use anyhow::Result as AResult; +use crossterm::event::KeyCode; +use std::cell::RefCell; +use std::process::exit; +use std::rc::Rc; +use tui::Terminal; + +type Wnd = Rc>; + +#[async_trait] +pub trait Window { + async fn handle_kbd(&mut self, k: KeyCode) -> AResult<()>; + async fn update(&mut self) -> AResult<()>; + async fn close(&mut self); + fn draw(&mut self, f: &mut Frame); +} + +pub struct WindowsHandler { + queue: Vec, + term: Terminal, + redraw_all: bool, +} + +impl WindowsHandler { + fn get_last_wnd(&self) -> Wnd { + self.queue.last().expect("No windows found").clone() + } + + pub fn new(term: Terminal) -> Self { + Self { + term, + queue: vec![], + redraw_all: true, + } + } + + pub fn push(&mut self) { + let window = Rc::new(RefCell::new(W::default())); + self.queue.push(window); + } + + pub fn clear(&mut self) -> AResult<()> { + self.term.clear()?; + Ok(()) + } + + pub fn show_cursor(&mut self) -> AResult<()> { + self.term.show_cursor()?; + Ok(()) + } + + pub fn draw(&mut self) -> AResult<()> { + if self.redraw_all { + for wnd in self.queue.iter() { + self.term.draw(|f| wnd.borrow_mut().draw(f))?; + } + } else { + let wnd = self.get_last_wnd(); + self.term.draw(|f| wnd.borrow_mut().draw(f))?; + } + Ok(()) + } + + pub async fn close(&mut self) { + let wnd = self.queue.pop().unwrap(); + wnd.borrow_mut().close().await; + self.redraw_all = true; + if self.queue.is_empty() { + self.show_cursor().unwrap(); + exit(0); + } + } + + pub async fn handle_kbd(&mut self, k: KeyCode) -> AResult<()> { + if let KeyCode::Esc = k { + self.close().await; + } else { + let current_wnd = self.get_last_wnd(); + current_wnd.borrow_mut().handle_kbd(k).await?; + } + Ok(()) + } + + pub async fn update(&mut self) -> AResult<()> { + let current_wnd = self.get_last_wnd(); + current_wnd.borrow_mut().update().await?; + Ok(()) + } +} diff --git a/bin/u_panel/src/tui/windows/processing.rs b/bin/u_panel/src/tui/windows/processing.rs new file mode 100644 index 0000000..e69de29 From bda30e2a72c8da33a6752a5e625d141e929a0397 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sun, 5 Dec 2021 23:29:27 +0500 Subject: [PATCH 14/30] biiiig tui, but still not working --- .gitignore | 12 +- bin/u_agent/src/lib.rs | 2 +- bin/u_panel/Cargo.toml | 1 + bin/u_panel/src/argparse.rs | 18 +- bin/u_panel/src/main.rs | 8 + bin/u_panel/src/tui/impls.rs | 18 +- bin/u_panel/src/tui/mod.rs | 54 +++--- bin/u_panel/src/tui/retval.rs | 42 +++++ bin/u_panel/src/tui/utils.rs | 31 ++++ bin/u_panel/src/tui/windows/confirm.rs | 133 +++++++++++++++ bin/u_panel/src/tui/windows/main_wnd.rs | 50 +++--- bin/u_panel/src/tui/windows/mod.rs | 199 ++++++++++++++++++---- bin/u_panel/src/tui/windows/processing.rs | 0 integration/tests/behaviour.rs | 4 +- integration/tests/helpers/panel.rs | 6 +- lib/u_lib/src/builder.rs | 4 +- lib/u_lib/src/cache.rs | 16 +- lib/u_lib/src/models/jobs/assigned.rs | 2 +- 18 files changed, 481 insertions(+), 119 deletions(-) create mode 100644 bin/u_panel/src/tui/retval.rs create mode 100644 bin/u_panel/src/tui/utils.rs create mode 100644 bin/u_panel/src/tui/windows/confirm.rs delete mode 100644 bin/u_panel/src/tui/windows/processing.rs diff --git a/.gitignore b/.gitignore index 8946455..3b8e7ff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,14 @@ target/ -**/*.rs.bk .idea/ data/ +certs/ +static/ +.vscode/ +release/ + +**/*.rs.bk **/*.pyc -certs/* *.log echoer .env.private -*.lock -static/ -.vscode/ \ No newline at end of file +*.lock \ No newline at end of file diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index a0b56f1..3f422b3 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -29,7 +29,7 @@ const ITERATION_LATENCY: u64 = 5; pub async fn process_request(job_requests: Vec, client: &ClientHandler) { if !job_requests.is_empty() { for jr in &job_requests { - if !JobCache::contains(&jr.job_id) { + if !JobCache::contains(jr.job_id) { debug!("Fetching job: {}", &jr.job_id); let fetched_job = loop { match client.get_jobs(Some(jr.job_id)).await { diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 14442d0..924ac7c 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -23,3 +23,4 @@ anyhow = "1.0.44" strum = { version = "0.22.0", features = ["derive"] } async-trait = "0.1.51" once_cell = "1.8.0" +crossbeam = "0.8.1" diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 53c93bc..3ee527a 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -1,10 +1,7 @@ -use once_cell::sync::Lazy; -use std::env; +use crate::CLIENT; use std::fmt; use structopt::StructOpt; -use u_lib::{ - api::ClientHandler, datatypes::DataResult, messaging::AsMsg, models::JobMeta, UError, UResult, -}; +use u_lib::{datatypes::DataResult, messaging::AsMsg, models::JobMeta, UError, UResult}; use uuid::Uuid; #[derive(StructOpt, Debug)] @@ -26,9 +23,8 @@ enum Cmd { #[derive(StructOpt, Debug)] enum JobALD { Add { - #[structopt(long, parse(try_from_str = parse_uuid))] - agent: Option, - + //#[structopt(long, parse(try_from_str = parse_uuid))] + //agent: Option, #[structopt(long)] alias: String, @@ -75,11 +71,6 @@ enum LD { }, } -pub static CLIENT: Lazy = Lazy::new(|| { - let token = env::var("ADMIN_AUTH_TOKEN").expect("access token is not set"); - ClientHandler::new(None).password(token.clone()) -}); - fn parse_uuid(src: &str) -> Result { Uuid::parse_str(src).map_err(|e| e.to_string()) } @@ -135,7 +126,6 @@ pub async fn process_cmd(args: Args) -> UResult<()> { JobMapALD::List { uid } => printer.print(CLIENT.get_agent_jobs(uid).await), JobMapALD::Delete { uid } => printer.print(CLIENT.del(Some(uid)).await), }, - //Cmd::Server => be::serve().unwrap(), Cmd::TUI => crate::tui::init_tui() .await .map_err(|e| UError::TUIError(e.to_string()))?, diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index b48a2c1..29a1275 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -5,10 +5,18 @@ mod tui; extern crate async_trait; use argparse::{process_cmd, Args}; +use once_cell::sync::Lazy; +use std::env; use std::process; use structopt::StructOpt; +use u_lib::api::ClientHandler; use u_lib::utils::init_env; +pub static CLIENT: Lazy = Lazy::new(|| { + let token = env::var("ADMIN_AUTH_TOKEN").expect("access token is not set"); + ClientHandler::new(None).password(token.clone()) +}); + #[tokio::main] async fn main() { init_env(); diff --git a/bin/u_panel/src/tui/impls.rs b/bin/u_panel/src/tui/impls.rs index 9e29a48..a20adb4 100644 --- a/bin/u_panel/src/tui/impls.rs +++ b/bin/u_panel/src/tui/impls.rs @@ -1,28 +1,30 @@ -use crate::argparse::CLIENT; +use crate::tui::windows::{ConfirmWnd, MainWnd, WndId}; +use crate::CLIENT; use u_lib::models::{Agent, AssignedJob, JobMeta}; use u_lib::UResult; use uuid::Uuid; -pub trait Id { - fn id(&self) -> Uuid; +pub trait Id { + fn id(&self) -> T; } #[macro_export] macro_rules! impl_id { - ($($type:ty),+) => { + ($id_type:tt => $($type:tt),+) => { $( - impl Id for $type { - fn id(&self) -> Uuid { + impl Id<$id_type> for $type { + fn id(&self) -> $id_type { self.id } })+ }; } -impl_id!(Agent, JobMeta, AssignedJob); +impl_id!(Uuid => Agent, JobMeta, AssignedJob); +impl_id!(WndId => MainWnd, ConfirmWnd); #[async_trait] -pub trait CRUD: Id +pub trait CRUD: Id where Self: Sized, { diff --git a/bin/u_panel/src/tui/mod.rs b/bin/u_panel/src/tui/mod.rs index 8ffd734..4c414f2 100644 --- a/bin/u_panel/src/tui/mod.rs +++ b/bin/u_panel/src/tui/mod.rs @@ -1,29 +1,39 @@ mod impls; +mod retval; +mod utils; mod windows; use anyhow::Result as AResult; use backtrace::Backtrace; -use crossterm::event::{self, DisableMouseCapture, EnableMouseCapture, Event}; +use crossterm::event::{self, DisableMouseCapture, EnableMouseCapture, Event, KeyCode}; use crossterm::execute; use crossterm::terminal::{ disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen, }; +use once_cell::sync::Lazy; +use retval::{RetVal, ReturnValue}; use std::panic::set_hook; use std::process::exit; use std::{ io::{stdout, Stdout}, - sync::mpsc, thread, time::Duration, }; use tui::{backend::CrosstermBackend, Terminal}; -use windows::{MainWnd, WindowsHandler}; +use utils::Channel; +use windows::{MainWnd, SharedWnd, WindowsHandler, WndId}; pub type Backend = CrosstermBackend; pub type Frame<'f> = tui::Frame<'f, Backend>; -enum InputEvent { - Key(I), +const EVENT_GEN_PERIOD: Duration = Duration::from_millis(120); + +static GENERAL_EVENT_CHANNEL: Lazy> = Lazy::new(|| Channel::new()); + +enum GEvent { + CreateWnd(SharedWnd), + CloseWnd { wid: WndId, force: bool }, + Key(KeyCode), Tick, } @@ -51,32 +61,38 @@ async fn init() -> AResult<()> { enable_raw_mode()?; execute!(stdout(), EnterAlternateScreen, EnableMouseCapture)?; - let mut wh = WindowsHandler::new(get_terminal()?); - wh.push::(); - let (tx, rx) = mpsc::channel(); + WindowsHandler::lock().await.push_new::().await; thread::spawn(move || loop { - if event::poll(Duration::from_millis(10)).unwrap() { + if event::poll(EVENT_GEN_PERIOD).unwrap() { match event::read().unwrap() { - key @ Event::Key(_) => tx.send(InputEvent::Key(key)).unwrap(), + Event::Key(key) => GENERAL_EVENT_CHANNEL.send(GEvent::Key(key.code)), _ => (), } } else { - tx.send(InputEvent::Tick).unwrap() + GENERAL_EVENT_CHANNEL.send(GEvent::Tick) } }); - wh.clear()?; + WindowsHandler::lock().await.clear()?; loop { - wh.draw()?; - wh.update().await?; - match rx.recv()? { - InputEvent::Key(Event::Key(key)) => { - wh.handle_kbd(key.code).await?; + match GENERAL_EVENT_CHANNEL.recv() { + GEvent::Tick => { + let mut wh = WindowsHandler::lock().await; + wh.update().await?; + wh.draw().await?; + } + GEvent::CloseWnd { wid, force } => { + let mut wh = WindowsHandler::lock().await; + wh.close(wid, force).await; + } + GEvent::Key(key) => { + WindowsHandler::lock().await.send_handle_kbd(key); + } + GEvent::CreateWnd(wnd) => { + WindowsHandler::lock().await.push_dyn(wnd).await; } - InputEvent::Tick => (), - _ => unreachable!(), } } } diff --git a/bin/u_panel/src/tui/retval.rs b/bin/u_panel/src/tui/retval.rs new file mode 100644 index 0000000..6d470b7 --- /dev/null +++ b/bin/u_panel/src/tui/retval.rs @@ -0,0 +1,42 @@ +use once_cell::sync::Lazy; +use std::any::{type_name, Any}; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::sync::Mutex; + +type RV = Box; +static LAST_RETVAL: Lazy>> = Lazy::new(|| Mutex::new(None)); + +pub struct ReturnValue; + +impl ReturnValue { + pub async fn set(t: RV) { + *LAST_RETVAL.lock().await = Some(t); + } + + pub async fn get() -> Box { + ReturnValue + .await + .downcast::() + .expect(&format!("wrong type {}", type_name::())) + } +} + +impl Future for ReturnValue { + type Output = Box; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if let Poll::Ready(mut rv) = unsafe { Pin::new_unchecked(&mut LAST_RETVAL.lock()) }.poll(cx) + { + if let Some(rv) = rv.take() { + return Poll::Ready(rv); + } + } + Poll::Pending + } +} + +pub trait RetVal { + fn retval(&self) -> RV; +} diff --git a/bin/u_panel/src/tui/utils.rs b/bin/u_panel/src/tui/utils.rs new file mode 100644 index 0000000..f1a1f2c --- /dev/null +++ b/bin/u_panel/src/tui/utils.rs @@ -0,0 +1,31 @@ +use crossbeam::channel::{unbounded, Receiver, Sender}; + +pub struct Channel { + pub tx: Sender, + pub rx: Receiver, +} + +impl Channel { + pub fn new() -> Self { + let (tx, rx) = unbounded::(); + Self { tx, rx } + } + + pub fn send(&self, msg: T) { + self.tx.send(msg).unwrap() + } + + pub fn recv(&self) -> T { + self.rx.recv().unwrap() + } + + pub fn is_empty(&self) -> bool { + self.rx.is_empty() + } +} + +impl Default for Channel { + fn default() -> Self { + Channel::new() + } +} diff --git a/bin/u_panel/src/tui/windows/confirm.rs b/bin/u_panel/src/tui/windows/confirm.rs new file mode 100644 index 0000000..41ca76e --- /dev/null +++ b/bin/u_panel/src/tui/windows/confirm.rs @@ -0,0 +1,133 @@ +use super::{Window, WndId}; +use crate::tui::{Frame, GEvent, RetVal, ReturnValue, GENERAL_EVENT_CHANNEL}; +use anyhow::Result as AResult; +use crossterm::event::KeyCode; +use std::any::Any; +use tui::layout::{Alignment, Constraint, Direction, Layout, Rect}; +use tui::style::{Modifier, Style}; +use tui::widgets::{Block, Clear, List, ListItem, ListState, Paragraph}; + +#[derive(Default)] +pub struct ConfirmWnd { + pub id: WndId, + msg: String, + variants: Vec, + state: ListState, +} + +impl ConfirmWnd { + pub fn new_yn(msg: impl Into, variants: Option>) -> Self { + let default = vec!["Yes".to_string(), "No".to_string()]; + let variants = match variants { + Some(v) if !v.is_empty() => v, + _ => default, + }; + let mut this = Self { + msg: msg.into(), + variants, + ..Default::default() + }; + this.state.select(Some(0)); + this + } + + pub fn on_right(&mut self) { + let selected = self.state.selected().unwrap_or(0); + self.state + .select(Some((selected + 1).rem_euclid(self.variants.len()))); + } + + pub fn on_left(&mut self) { + let selected = self.state.selected().unwrap_or(0); + let vars_len = self.variants.len(); + self.state + .select(Some((selected + vars_len - 1).rem_euclid(vars_len))); + } +} + +impl RetVal for ConfirmWnd { + fn retval(&self) -> Box { + let value = self + .variants + .get(self.state.selected().unwrap()) + .unwrap() + .to_owned(); + Box::new(value) + } +} + +#[async_trait] +impl Window for ConfirmWnd { + async fn handle_kbd(&mut self, k: KeyCode) -> AResult<()> { + match k { + KeyCode::Right => self.on_right(), + KeyCode::Left => self.on_left(), + KeyCode::Enter | KeyCode::Esc => self.close(false).await, + _ => (), + } + Ok(()) + } + + async fn handle_update(&mut self) -> AResult<()> { + Ok(()) + } + + async fn handle_close(&mut self) -> bool { + true + } + + fn draw(&mut self, f: &mut Frame) { + let size = f.size(); + let rect = centered_rect(60, 40, size); + f.render_widget(Clear, rect); + + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints(vec![Constraint::Percentage(70), Constraint::Percentage(30)]) + .split(rect); + let msg = Paragraph::new(self.msg.as_ref()); + f.render_widget(msg, chunks[0]); + + let options = self + .variants + .iter() + .map(AsRef::as_ref) + .map(ListItem::new) + .collect::>(); + let list = + List::new(options).highlight_style(Style::default().add_modifier(Modifier::BOLD)); + f.render_stateful_widget(list, chunks[1], &mut self.state); + } +} + +pub async fn confirm_wnd(msg: impl Into) -> bool { + let wnd = ConfirmWnd::new_yn(msg.into(), None); + GENERAL_EVENT_CHANNEL.send(GEvent::CreateWnd(wnd.into_shared())); + *ReturnValue::get().await +} + +fn centered_rect(percent_x: u16, percent_y: u16, r: Rect) -> Rect { + let popup_layout = Layout::default() + .direction(Direction::Vertical) + .constraints( + [ + Constraint::Percentage((100 - percent_y) / 2), + Constraint::Percentage(percent_y), + Constraint::Percentage((100 - percent_y) / 2), + ] + .as_ref(), + ) + .split(r); + + Layout::default() + .direction(Direction::Horizontal) + .constraints( + [ + Constraint::Percentage((100 - percent_x) / 2), + Constraint::Percentage(percent_x), + Constraint::Percentage((100 - percent_x) / 2), + ] + .as_ref(), + ) + .split(popup_layout[1])[1] +} diff --git a/bin/u_panel/src/tui/windows/main_wnd.rs b/bin/u_panel/src/tui/windows/main_wnd.rs index 4cebc8f..e595e1f 100644 --- a/bin/u_panel/src/tui/windows/main_wnd.rs +++ b/bin/u_panel/src/tui/windows/main_wnd.rs @@ -1,7 +1,8 @@ -use super::Window; -use crate::tui::{impls::CRUD, Frame}; +use super::{confirm_wnd, Window}; +use crate::tui::{impls::CRUD, windows::WndId, Frame, RetVal}; use anyhow::Result as AResult; use crossterm::event::KeyCode; +use std::any::Any; use std::{fmt::Display, str::FromStr}; use strum::VariantNames; use tokio::join; @@ -32,13 +33,13 @@ impl UiTabs { } pub fn next(&self) -> Self { - let next_idx = (self.index() + 1) % Self::VARIANTS.len(); + let next_idx = (self.index() + 1).rem_euclid(Self::VARIANTS.len()); Self::from_str(Self::VARIANTS[next_idx]).unwrap() } pub fn prev(&self) -> Self { let vlen = Self::VARIANTS.len(); - let next_idx = (self.index() + vlen - 1) % vlen; + let next_idx = (self.index() + vlen - 1).rem_euclid(vlen); Self::from_str(Self::VARIANTS[next_idx]).unwrap() } } @@ -52,7 +53,7 @@ pub struct StatefulList { impl StatefulList { pub async fn update(&mut self) -> AResult<()> { if !self.updated { - let new_values = ::read().await?; + let new_values = T::read().await?; self.inner = new_values; self.updated = true; } @@ -62,7 +63,7 @@ impl StatefulList { pub async fn delete(&mut self) -> AResult<()> { if let Some(s) = self.state.selected() { let uid = self.inner[s].id(); - ::delete(uid).await?; + T::delete(uid).await?; } Ok(()) } @@ -90,6 +91,7 @@ impl Default for StatefulList { } pub struct MainWnd { + pub id: WndId, pub active_tab: UiTabs, pub last_error: Option, pub agents: StatefulList, @@ -105,6 +107,7 @@ impl Default for MainWnd { agents: Default::default(), jobs: Default::default(), map: Default::default(), + id: Default::default(), } } } @@ -194,7 +197,10 @@ impl MainWnd { pub fn update_tab(&mut self) { match self.active_tab { - UiTabs::Agents => self.agents.updated = false, + UiTabs::Agents => { + self.agents.updated = false; + self.jobs.updated = false; + } UiTabs::Jobs => self.jobs.updated = false, UiTabs::Map => self.map.updated = false, } @@ -209,8 +215,8 @@ impl MainWnd { if list_len == 0 { list_state.select(None); } else { - let selected = list_state.selected().unwrap_or(0); - list_state.select(Some((selected + 1) % list_len)); + let selected = list_state.selected().unwrap_or(list_len - 1); + list_state.select(Some((selected + 1).rem_euclid(list_len))); } } @@ -224,7 +230,7 @@ impl MainWnd { list_state.select(None); } else { let selected = list_state.selected().unwrap_or(1); - list_state.select(Some((selected + list_len - 1) % list_len)); + list_state.select(Some((selected + list_len - 1).rem_euclid(list_len))); } } @@ -244,18 +250,16 @@ impl MainWnd { impl Window for MainWnd { async fn handle_kbd(&mut self, k: KeyCode) -> AResult<()> { match k { - KeyCode::Esc => { - /*teardown()?; - wh.show_cursor()?; - break;*/ - } + KeyCode::Esc => self.close(false).await, KeyCode::Left => self.prev_tab(), KeyCode::Right => self.next_tab(), KeyCode::Up => self.on_up(), KeyCode::Down => self.on_down(), KeyCode::Delete => { - self.delete().await; - self.update_tab(); + if confirm_wnd("Delete?").await { + self.delete().await; + self.update_tab(); + } } KeyCode::F(5) => self.update_tab(), _ => (), @@ -263,13 +267,13 @@ impl Window for MainWnd { Ok(()) } - async fn update(&mut self) -> AResult<()> { + async fn handle_update(&mut self) -> AResult<()> { self.check_updates().await; Ok(()) } - async fn close(&mut self) { - //self.show_cursor() + async fn handle_close(&mut self) -> bool { + true } fn draw(&mut self, f: &mut Frame) { @@ -308,6 +312,12 @@ impl Window for MainWnd { } } +impl RetVal for MainWnd { + fn retval(&self) -> Box { + Box::new(()) + } +} + fn crop(data: T, retain: usize) -> String { data.to_string()[..retain].to_string() } diff --git a/bin/u_panel/src/tui/windows/mod.rs b/bin/u_panel/src/tui/windows/mod.rs index 3cc7e31..87c430b 100644 --- a/bin/u_panel/src/tui/windows/mod.rs +++ b/bin/u_panel/src/tui/windows/mod.rs @@ -1,47 +1,158 @@ +mod confirm; mod main_wnd; -mod processing; +pub use confirm::{confirm_wnd, ConfirmWnd}; +use crossbeam::channel::Sender; pub use main_wnd::MainWnd; -use crate::tui::{Backend, Frame}; +use crate::tui::{ + get_terminal, impls::Id, teardown, utils::Channel, Backend, Frame, GEvent, RetVal, ReturnValue, + GENERAL_EVENT_CHANNEL, +}; use anyhow::Result as AResult; use crossterm::event::KeyCode; -use std::cell::RefCell; +use once_cell::sync::{Lazy, OnceCell}; +use std::collections::BTreeMap; use std::process::exit; -use std::rc::Rc; +use std::sync::Arc; +use std::sync::{Mutex as StdMutex, MutexGuard as StdMutexGuard}; +use tokio::sync::{Mutex, MutexGuard}; +use tokio::task::{self, JoinHandle}; use tui::Terminal; -type Wnd = Rc>; +static WINDOWS: Lazy>> = + Lazy::new(|| Arc::new(Mutex::new(WindowsHandler::new(get_terminal().unwrap())))); + +static LAST_WND_ID: OnceCell> = OnceCell::new(); + +static WND_EVENT_CHANNEL: Lazy> = Lazy::new(|| Channel::new()); + +pub type SharedWnd = Arc>; + +enum WndEvent { + Key(KeyCode), +} + +#[derive(PartialEq, PartialOrd, Eq, Ord, Hash, Copy, Clone, Debug)] +pub struct WndId(u64); + +impl WndId { + pub fn last_wid() -> StdMutexGuard<'static, WndId> { + LAST_WND_ID + .get_or_init(|| StdMutex::new(WndId(0))) + .lock() + .unwrap() + } +} + +impl Default for WndId { + fn default() -> Self { + let mut wid = Self::last_wid(); + wid.0 += 1; + *wid + } +} #[async_trait] -pub trait Window { +pub trait Window: Id + RetVal + Send { + async fn check_wnd_events(&mut self) { + if !WND_EVENT_CHANNEL.is_empty() { + match WND_EVENT_CHANNEL.recv() { + WndEvent::Key(k) => { + self.handle_kbd(k).await.unwrap(); + } + } + } + } + + async fn close(&mut self, force: bool) { + let rv = self.retval(); + ReturnValue::set(rv).await; + GENERAL_EVENT_CHANNEL.send(GEvent::CloseWnd { + wid: self.id(), + force, + }); + } + + fn into_shared(self) -> SharedWnd + where + Self: Sized + 'static, + { + Arc::new(Mutex::new(self)) + } + + async fn handle_update(&mut self) -> AResult<()>; + + async fn handle_close(&mut self) -> bool; + async fn handle_kbd(&mut self, k: KeyCode) -> AResult<()>; - async fn update(&mut self) -> AResult<()>; - async fn close(&mut self); + fn draw(&mut self, f: &mut Frame); } +pub struct WndLoop { + upd_tx: Sender<()>, + updater: JoinHandle<()>, + window: SharedWnd, +} + +impl WndLoop { + pub fn with_wnd(window: SharedWnd) -> Self { + let Channel { tx, rx } = Channel::new(); + let wnd = window.clone(); + let wnd_loop = async move { + loop { + rx.recv().unwrap(); + wnd.lock().await.check_wnd_events().await; + } + }; + WndLoop { + upd_tx: tx, + window, + updater: task::spawn(wnd_loop), + } + } + + pub fn send_update(&self) { + self.upd_tx.send(()).unwrap(); + } +} + pub struct WindowsHandler { - queue: Vec, + queue: BTreeMap, term: Terminal, redraw_all: bool, } impl WindowsHandler { - fn get_last_wnd(&self) -> Wnd { - self.queue.last().expect("No windows found").clone() + fn get_last_wnd(&self) -> &WndLoop { + let last_id = self.queue.keys().rev().next().expect("No windows found"); + self.queue.get(last_id).unwrap() + } + + pub async fn lock<'l>() -> MutexGuard<'l, Self> { + WINDOWS.lock().await } - pub fn new(term: Terminal) -> Self { + fn new(term: Terminal) -> Self { Self { term, - queue: vec![], + queue: BTreeMap::new(), redraw_all: true, } } - pub fn push(&mut self) { - let window = Rc::new(RefCell::new(W::default())); - self.queue.push(window); + pub async fn push(&mut self, window: W) -> SharedWnd { + self.push_dyn(window.into_shared()).await + } + + pub async fn push_new(&mut self) -> SharedWnd { + self.push(W::default()).await + } + + pub async fn push_dyn(&mut self, window: SharedWnd) -> SharedWnd { + let wid = window.lock().await.id(); + self.queue.insert(wid, WndLoop::with_wnd(window.clone())); + window } pub fn clear(&mut self) -> AResult<()> { @@ -54,41 +165,57 @@ impl WindowsHandler { Ok(()) } - pub fn draw(&mut self) -> AResult<()> { - if self.redraw_all { - for wnd in self.queue.iter() { - self.term.draw(|f| wnd.borrow_mut().draw(f))?; - } + pub async fn draw(&mut self) -> AResult<()> { + let wids_to_redraw = if self.redraw_all { + self.redraw_all = false; + let mut wids = self.queue.keys().cloned().collect::>(); + wids.sort(); + wids } else { - let wnd = self.get_last_wnd(); - self.term.draw(|f| wnd.borrow_mut().draw(f))?; + vec![*WndId::last_wid()] + }; + for wid in wids_to_redraw { + let mut wnd_locked = match self.queue.get(&wid) { + Some(w) => w.window.lock().await, + None => { + eprintln!("Can't redraw window {:?}, not found", wid); + continue; + } + }; + self.term.draw(move |f| wnd_locked.draw(f))?; } Ok(()) } - pub async fn close(&mut self) { - let wnd = self.queue.pop().unwrap(); - wnd.borrow_mut().close().await; - self.redraw_all = true; + pub async fn close(&mut self, wid: WndId, force: bool) { + let wnd = match self.queue.get(&wid) { + Some(w) => w.clone(), + None => { + eprintln!("Can't close window {:?}, not found", wid); + return; + } + }; + if wnd.window.lock().await.handle_close().await || force { + let WndLoop { updater, .. } = self.queue.remove(&wid).unwrap(); + updater.abort(); + self.redraw_all = true; + } + if self.queue.is_empty() { self.show_cursor().unwrap(); + teardown().unwrap(); exit(0); } } - pub async fn handle_kbd(&mut self, k: KeyCode) -> AResult<()> { - if let KeyCode::Esc = k { - self.close().await; - } else { - let current_wnd = self.get_last_wnd(); - current_wnd.borrow_mut().handle_kbd(k).await?; - } - Ok(()) + pub fn send_handle_kbd(&self, k: KeyCode) { + WND_EVENT_CHANNEL.send(WndEvent::Key(k)); } pub async fn update(&mut self) -> AResult<()> { let current_wnd = self.get_last_wnd(); - current_wnd.borrow_mut().update().await?; + current_wnd.send_update(); + current_wnd.window.lock().await.handle_update().await?; Ok(()) } } diff --git a/bin/u_panel/src/tui/windows/processing.rs b/bin/u_panel/src/tui/windows/processing.rs deleted file mode 100644 index e69de29..0000000 diff --git a/integration/tests/behaviour.rs b/integration/tests/behaviour.rs index 38e88f2..fcc3c63 100644 --- a/integration/tests/behaviour.rs +++ b/integration/tests/behaviour.rs @@ -29,11 +29,11 @@ async fn test_setup_tasks() -> TestResult { let job_alias = "passwd_contents"; let cmd = format!("jobs add --alias {} 'cat /etc/passwd'", job_alias); Panel::check_status(cmd); - let cmd = format!("jobmap add {} {}", agent_uid, job_alias); + let cmd = format!("map add {} {}", agent_uid, job_alias); let assigned_uids: Vec = Panel::check_output(cmd); for _ in 0..3 { let result: Vec = - Panel::check_output(format!("jobmap list {}", assigned_uids[0])); + Panel::check_output(format!("map list {}", assigned_uids[0])); if result[0].state == JobState::Finished { return Ok(()); } else { diff --git a/integration/tests/helpers/panel.rs b/integration/tests/helpers/panel.rs index 929715a..9406d90 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration/tests/helpers/panel.rs @@ -35,7 +35,8 @@ impl Panel { }) } - pub fn output(args: impl Into) -> PanelResult { + pub fn output(args: impl Into + Display) -> PanelResult { + println!("Executing 'u_panel {}'", &args); let splitted = split(args.into().as_ref()).unwrap(); Self::output_argv( splitted @@ -54,12 +55,11 @@ impl Panel { } pub fn check_status(args: impl Into + Display) { - println!("Panel: executing '{}'", &args); let result: PanelResult = Self::output(args); Self::status_is_ok(result); } - pub fn check_output(args: impl Into) -> T { + pub fn check_output(args: impl Into + Display) -> T { let result = Self::output(args); Self::status_is_ok(result) } diff --git a/lib/u_lib/src/builder.rs b/lib/u_lib/src/builder.rs index 726d74a..e6ce08f 100644 --- a/lib/u_lib/src/builder.rs +++ b/lib/u_lib/src/builder.rs @@ -12,7 +12,7 @@ impl JobBuilder { let mut prepared: Vec = vec![]; let mut result = CombinedResult::::new(); for req in job_requests { - let job_meta = JobCache::get(&req.job_id); + let job_meta = JobCache::get(req.job_id); if job_meta.is_none() { result.err(UError::NoJob(req.job_id).into_bt()); continue; @@ -22,7 +22,7 @@ impl JobBuilder { let built_req = (|| -> UResult<()> { Ok(match job_meta.exec_type { JobType::Shell => { - let meta = JobCache::get(&req.job_id).ok_or(UError::NoJob(req.job_id))?; + let meta = JobCache::get(req.job_id).ok_or(UError::NoJob(req.job_id))?; let curr_platform = guess_host_triple().unwrap_or("unknown").to_string(); //TODO: extend platform checking (partial check) if meta.platform != curr_platform { diff --git a/lib/u_lib/src/cache.rs b/lib/u_lib/src/cache.rs index 3af1724..9d173ee 100644 --- a/lib/u_lib/src/cache.rs +++ b/lib/u_lib/src/cache.rs @@ -20,11 +20,11 @@ impl JobCache { JOB_CACHE.write().unwrap().insert(job_meta.id, job_meta); } - pub fn contains(uid: &Uuid) -> bool { - JOB_CACHE.read().unwrap().contains_key(uid) + pub fn contains(uid: Uuid) -> bool { + JOB_CACHE.read().unwrap().contains_key(&uid) } - pub fn get(uid: &Uuid) -> Option { + pub fn get<'jh>(uid: Uuid) -> Option> { if !Self::contains(uid) { return None; } @@ -32,17 +32,17 @@ impl JobCache { Some(JobCacheHolder(lock, uid)) } - pub fn remove(uid: &Uuid) { - JOB_CACHE.write().unwrap().remove(uid); + pub fn remove(uid: Uuid) { + JOB_CACHE.write().unwrap().remove(&uid); } } -pub struct JobCacheHolder<'jm>(pub RwLockReadGuard<'jm, Cache>, pub &'jm Uuid); +pub struct JobCacheHolder<'jh>(pub RwLockReadGuard<'jh, Cache>, pub Uuid); -impl<'jm> Deref for JobCacheHolder<'jm> { +impl<'jh> Deref for JobCacheHolder<'jh> { type Target = JobMeta; fn deref(&self) -> &Self::Target { - self.0.get(self.1).unwrap() + self.0.get(&self.1).unwrap() } } diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index 3182796..8893977 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -80,7 +80,7 @@ impl AssignedJob { pub async fn run(mut self) -> Reportable { use tokio::process::Command; let (argv, _payload) = { - let meta = JobCache::get(&self.job_id).unwrap(); + let meta = JobCache::get(self.job_id).unwrap(); if let Some(ref payload) = meta.payload { let extracted_payload = match TempFile::write_exec(payload) { Ok(p) => p, From 5df2b09ceb1880414eb6699b6785376f80a3c26f Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sat, 18 Dec 2021 03:42:50 +0500 Subject: [PATCH 15/30] IT WORKS --- bin/u_panel/Cargo.toml | 7 +- bin/u_panel/src/argparse.rs | 10 +- bin/u_panel/src/main.rs | 5 +- bin/u_panel/src/tui/mod.rs | 154 ++++++++++++++++++------ bin/u_panel/src/tui/retval.rs | 31 ++--- bin/u_panel/src/tui/utils.rs | 33 +++++ bin/u_panel/src/tui/windows/confirm.rs | 15 ++- bin/u_panel/src/tui/windows/main_wnd.rs | 14 ++- bin/u_panel/src/tui/windows/mod.rs | 71 +++++------ scripts/build_musl_libs.sh | 18 +-- 10 files changed, 242 insertions(+), 116 deletions(-) diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 924ac7c..4fb5761 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -14,7 +14,7 @@ env_logger = "0.7.1" uuid = "0.6.5" serde_json = "1.0.4" serde = { version = "1.0.114", features = ["derive"] } -tokio = "1.11.0" +tokio = { version = "1.11.0", features = ["rt", "rt-multi-thread"] } # be = { version = "*", path = "./be" } u_lib = { version = "*", path = "../../lib/u_lib" } tui = { version = "0.16", default-features = false, features = ['crossterm'] } @@ -24,3 +24,8 @@ strum = { version = "0.22.0", features = ["derive"] } async-trait = "0.1.51" once_cell = "1.8.0" crossbeam = "0.8.1" +async-channel = "1.6.1" +tracing = "0.1.29" +tracing-subscriber = { version = "0.3.3", features = ["env-filter"]} +signal-hook = "0.3.12" +tracing-appender = "0.2.0" diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 3ee527a..651e50d 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -17,7 +17,13 @@ enum Cmd { Agents(LD), Jobs(JobALD), Map(JobMapALD), - TUI, + TUI(TUIArgs), +} + +#[derive(StructOpt, Debug)] +pub struct TUIArgs { + #[structopt(long)] + pub nogui: bool, } #[derive(StructOpt, Debug)] @@ -126,7 +132,7 @@ pub async fn process_cmd(args: Args) -> UResult<()> { JobMapALD::List { uid } => printer.print(CLIENT.get_agent_jobs(uid).await), JobMapALD::Delete { uid } => printer.print(CLIENT.del(Some(uid)).await), }, - Cmd::TUI => crate::tui::init_tui() + Cmd::TUI(args) => crate::tui::init_tui(&args) .await .map_err(|e| UError::TUIError(e.to_string()))?, } diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index 29a1275..63e9e06 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -4,6 +4,9 @@ mod tui; #[macro_use] extern crate async_trait; +#[macro_use] +extern crate tracing; + use argparse::{process_cmd, Args}; use once_cell::sync::Lazy; use std::env; @@ -17,7 +20,7 @@ pub static CLIENT: Lazy = Lazy::new(|| { ClientHandler::new(None).password(token.clone()) }); -#[tokio::main] +#[tokio::main(flavor = "multi_thread")] async fn main() { init_env(); let args: Args = Args::from_args(); diff --git a/bin/u_panel/src/tui/mod.rs b/bin/u_panel/src/tui/mod.rs index 4c414f2..5cd9942 100644 --- a/bin/u_panel/src/tui/mod.rs +++ b/bin/u_panel/src/tui/mod.rs @@ -3,6 +3,7 @@ mod retval; mod utils; mod windows; +use crate::argparse::TUIArgs; use anyhow::Result as AResult; use backtrace::Backtrace; use crossterm::event::{self, DisableMouseCapture, EnableMouseCapture, Event, KeyCode}; @@ -12,15 +13,17 @@ use crossterm::terminal::{ }; use once_cell::sync::Lazy; use retval::{RetVal, ReturnValue}; -use std::panic::set_hook; -use std::process::exit; use std::{ + env, io::{stdout, Stdout}, + panic::set_hook, + process::exit, + sync::atomic::{AtomicBool, Ordering}, thread, time::Duration, }; use tui::{backend::CrosstermBackend, Terminal}; -use utils::Channel; +use utils::{AsyncChannel, Channel}; use windows::{MainWnd, SharedWnd, WindowsHandler, WndId}; pub type Backend = CrosstermBackend; @@ -30,9 +33,12 @@ const EVENT_GEN_PERIOD: Duration = Duration::from_millis(120); static GENERAL_EVENT_CHANNEL: Lazy> = Lazy::new(|| Channel::new()); +static ACTIVE_LOOP: AtomicBool = AtomicBool::new(true); + enum GEvent { CreateWnd(SharedWnd), CloseWnd { wid: WndId, force: bool }, + Exit, Key(KeyCode), Tick, } @@ -42,63 +48,139 @@ fn get_terminal() -> AResult> { Ok(Terminal::new(backend)?) } -pub async fn init_tui() -> AResult<()> { - //TODO: fix this - set_hook(Box::new(|p| { - teardown().unwrap(); - get_terminal().unwrap().show_cursor().unwrap(); - eprintln!("{}\n{:?}", p, Backtrace::new()); - exit(254); - })); - if let Err(e) = init().await { - teardown()?; - return Err(e); - } - Ok(()) -} +pub async fn init_tui(args: &TUIArgs) -> AResult<()> { + init_logger(); + info!("Initializing u_panel"); -async fn init() -> AResult<()> { - enable_raw_mode()?; - execute!(stdout(), EnterAlternateScreen, EnableMouseCapture)?; + let gui = !args.nogui; + init_signal_handlers(gui); + init_panic_handler(gui); + term_setup(gui)?; + info!("Starting loop"); + + let result = init_loop(args).await; + info!("Exiting"); + term_teardown(gui)?; + result +} +async fn init_loop(args: &TUIArgs) -> AResult<()> { + let gui = !args.nogui; + if gui { + WindowsHandler::lock().await.clear()?; + } WindowsHandler::lock().await.push_new::().await; - thread::spawn(move || loop { - if event::poll(EVENT_GEN_PERIOD).unwrap() { - match event::read().unwrap() { - Event::Key(key) => GENERAL_EVENT_CHANNEL.send(GEvent::Key(key.code)), - _ => (), + thread::spawn(move || { + while is_running() { + if event::poll(EVENT_GEN_PERIOD).unwrap() { + match event::read().unwrap() { + Event::Key(key) => GENERAL_EVENT_CHANNEL.send(GEvent::Key(key.code)), + _ => (), + } + } else { + GENERAL_EVENT_CHANNEL.send(GEvent::Tick) } - } else { - GENERAL_EVENT_CHANNEL.send(GEvent::Tick) } }); - WindowsHandler::lock().await.clear()?; - - loop { + while is_running() { match GENERAL_EVENT_CHANNEL.recv() { GEvent::Tick => { let mut wh = WindowsHandler::lock().await; wh.update().await?; - wh.draw().await?; + if gui { + wh.draw().await?; + } } GEvent::CloseWnd { wid, force } => { - let mut wh = WindowsHandler::lock().await; - wh.close(wid, force).await; + WindowsHandler::lock().await.close(wid, force).await; } GEvent::Key(key) => { - WindowsHandler::lock().await.send_handle_kbd(key); + info!(?key, "pressed"); + if let KeyCode::Char('q') = key { + break_global(); + } else { + WindowsHandler::lock().await.send_handle_kbd(key).await; + } } GEvent::CreateWnd(wnd) => { WindowsHandler::lock().await.push_dyn(wnd).await; } + GEvent::Exit => { + break_global(); + } } } + Ok(()) +} + +#[inline] +fn is_running() -> bool { + ACTIVE_LOOP.load(Ordering::Relaxed) +} + +fn break_global() { + ACTIVE_LOOP.store(false, Ordering::Relaxed); } -fn teardown() -> AResult<()> { +fn init_signal_handlers(gui: bool) { + use signal_hook::{ + consts::{SIGINT, SIGTERM}, + iterator::Signals, + }; + + thread::spawn(move || { + let mut signals = Signals::new(&[SIGINT, SIGTERM]).unwrap(); + for sig in signals.forever() { + match sig { + SIGINT => break_global(), + SIGTERM => { + break_global(); + term_teardown(gui).ok(); + exit(3); + } + _ => (), + } + } + }); +} + +fn init_logger() { + use tracing_appender::rolling::{RollingFileAppender, Rotation}; + use tracing_subscriber::EnvFilter; + + if env::var("RUST_LOG").is_err() { + env::set_var("RUST_LOG", "info") + } + + tracing_subscriber::fmt::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .with_writer(|| RollingFileAppender::new(Rotation::NEVER, ".", "u_panel.log")) + .init(); +} + +fn init_panic_handler(gui: bool) { + set_hook(Box::new(move |panic_info| { + term_teardown(gui).ok(); + eprintln!("{}\n{:?}", panic_info, Backtrace::new()); + exit(254); + })); +} + +fn term_setup(gui: bool) -> AResult<()> { + enable_raw_mode()?; + if gui { + execute!(stdout(), EnterAlternateScreen, EnableMouseCapture)?; + } + Ok(()) +} + +fn term_teardown(gui: bool) -> AResult<()> { disable_raw_mode()?; - execute!(stdout(), LeaveAlternateScreen, DisableMouseCapture)?; + if gui { + get_terminal()?.show_cursor()?; + execute!(stdout(), LeaveAlternateScreen, DisableMouseCapture)?; + } Ok(()) } diff --git a/bin/u_panel/src/tui/retval.rs b/bin/u_panel/src/tui/retval.rs index 6d470b7..3874dfc 100644 --- a/bin/u_panel/src/tui/retval.rs +++ b/bin/u_panel/src/tui/retval.rs @@ -1,42 +1,25 @@ +use crate::tui::Channel; use once_cell::sync::Lazy; use std::any::{type_name, Any}; -use std::future::Future; -use std::pin::Pin; -use std::task::{Context, Poll}; -use tokio::sync::Mutex; type RV = Box; -static LAST_RETVAL: Lazy>> = Lazy::new(|| Mutex::new(None)); +static RETVAL: Lazy> = Lazy::new(|| Channel::new()); pub struct ReturnValue; impl ReturnValue { - pub async fn set(t: RV) { - *LAST_RETVAL.lock().await = Some(t); + pub fn set(t: RV) { + RETVAL.send(t); } - pub async fn get() -> Box { - ReturnValue - .await + pub fn get() -> Box { + RETVAL + .recv() .downcast::() .expect(&format!("wrong type {}", type_name::())) } } -impl Future for ReturnValue { - type Output = Box; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - if let Poll::Ready(mut rv) = unsafe { Pin::new_unchecked(&mut LAST_RETVAL.lock()) }.poll(cx) - { - if let Some(rv) = rv.take() { - return Poll::Ready(rv); - } - } - Poll::Pending - } -} - pub trait RetVal { fn retval(&self) -> RV; } diff --git a/bin/u_panel/src/tui/utils.rs b/bin/u_panel/src/tui/utils.rs index f1a1f2c..e8e8542 100644 --- a/bin/u_panel/src/tui/utils.rs +++ b/bin/u_panel/src/tui/utils.rs @@ -1,3 +1,6 @@ +use async_channel::{ + unbounded as unbounded_async, Receiver as AsyncReceiver, Sender as AsyncSender, +}; use crossbeam::channel::{unbounded, Receiver, Sender}; pub struct Channel { @@ -29,3 +32,33 @@ impl Default for Channel { Channel::new() } } + +pub struct AsyncChannel { + pub tx: AsyncSender, + pub rx: AsyncReceiver, +} + +impl AsyncChannel { + pub fn new() -> Self { + let (tx, rx) = unbounded_async::(); + Self { tx, rx } + } + + pub async fn send(&self, msg: T) { + self.tx.send(msg).await.unwrap() + } + + pub async fn recv(&self) -> T { + self.rx.recv().await.unwrap() + } + + pub fn is_empty(&self) -> bool { + self.rx.is_empty() + } +} + +impl Default for AsyncChannel { + fn default() -> Self { + AsyncChannel::new() + } +} diff --git a/bin/u_panel/src/tui/windows/confirm.rs b/bin/u_panel/src/tui/windows/confirm.rs index 41ca76e..8272ffa 100644 --- a/bin/u_panel/src/tui/windows/confirm.rs +++ b/bin/u_panel/src/tui/windows/confirm.rs @@ -1,7 +1,10 @@ use super::{Window, WndId}; -use crate::tui::{Frame, GEvent, RetVal, ReturnValue, GENERAL_EVENT_CHANNEL}; +use crate::tui::{ + windows::WndEvent, AsyncChannel, Frame, GEvent, RetVal, ReturnValue, GENERAL_EVENT_CHANNEL, +}; use anyhow::Result as AResult; use crossterm::event::KeyCode; +use once_cell::sync::OnceCell; use std::any::Any; use tui::layout::{Alignment, Constraint, Direction, Layout, Rect}; use tui::style::{Modifier, Style}; @@ -79,7 +82,6 @@ impl Window for ConfirmWnd { fn draw(&mut self, f: &mut Frame) { let size = f.size(); let rect = centered_rect(60, 40, size); - f.render_widget(Clear, rect); let chunks = Layout::default() .direction(Direction::Vertical) @@ -98,12 +100,17 @@ impl Window for ConfirmWnd { List::new(options).highlight_style(Style::default().add_modifier(Modifier::BOLD)); f.render_stateful_widget(list, chunks[1], &mut self.state); } + + fn get_chan(&self) -> &'static AsyncChannel { + static EV_CHAN: OnceCell> = OnceCell::new(); + EV_CHAN.get_or_init(|| AsyncChannel::new()) + } } -pub async fn confirm_wnd(msg: impl Into) -> bool { +pub async fn confirm_wnd(msg: impl Into) -> String { let wnd = ConfirmWnd::new_yn(msg.into(), None); GENERAL_EVENT_CHANNEL.send(GEvent::CreateWnd(wnd.into_shared())); - *ReturnValue::get().await + *ReturnValue::get() } fn centered_rect(percent_x: u16, percent_y: u16, r: Rect) -> Rect { diff --git a/bin/u_panel/src/tui/windows/main_wnd.rs b/bin/u_panel/src/tui/windows/main_wnd.rs index e595e1f..1a7c40a 100644 --- a/bin/u_panel/src/tui/windows/main_wnd.rs +++ b/bin/u_panel/src/tui/windows/main_wnd.rs @@ -1,7 +1,12 @@ use super::{confirm_wnd, Window}; -use crate::tui::{impls::CRUD, windows::WndId, Frame, RetVal}; +use crate::tui::{ + impls::CRUD, + windows::{WndEvent, WndId}, + AsyncChannel, Frame, RetVal, +}; use anyhow::Result as AResult; use crossterm::event::KeyCode; +use once_cell::sync::OnceCell; use std::any::Any; use std::{fmt::Display, str::FromStr}; use strum::VariantNames; @@ -256,7 +261,7 @@ impl Window for MainWnd { KeyCode::Up => self.on_up(), KeyCode::Down => self.on_down(), KeyCode::Delete => { - if confirm_wnd("Delete?").await { + if &confirm_wnd("Delete?").await == "Yes" { self.delete().await; self.update_tab(); } @@ -310,6 +315,11 @@ impl Window for MainWnd { .highlight_style(Style::default().add_modifier(Modifier::BOLD)); f.render_stateful_widget(list, chunks[1], self.tab_list_state()); } + + fn get_chan(&self) -> &'static AsyncChannel { + static EV_CHAN: OnceCell> = OnceCell::new(); + EV_CHAN.get_or_init(|| AsyncChannel::new()) + } } impl RetVal for MainWnd { diff --git a/bin/u_panel/src/tui/windows/mod.rs b/bin/u_panel/src/tui/windows/mod.rs index 87c430b..c4b18a5 100644 --- a/bin/u_panel/src/tui/windows/mod.rs +++ b/bin/u_panel/src/tui/windows/mod.rs @@ -1,18 +1,17 @@ mod confirm; mod main_wnd; +use async_channel::Sender; pub use confirm::{confirm_wnd, ConfirmWnd}; -use crossbeam::channel::Sender; pub use main_wnd::MainWnd; use crate::tui::{ - get_terminal, impls::Id, teardown, utils::Channel, Backend, Frame, GEvent, RetVal, ReturnValue, + get_terminal, impls::Id, AsyncChannel, Backend, Frame, GEvent, RetVal, ReturnValue, GENERAL_EVENT_CHANNEL, }; use anyhow::Result as AResult; use crossterm::event::KeyCode; use once_cell::sync::{Lazy, OnceCell}; use std::collections::BTreeMap; -use std::process::exit; use std::sync::Arc; use std::sync::{Mutex as StdMutex, MutexGuard as StdMutexGuard}; use tokio::sync::{Mutex, MutexGuard}; @@ -24,11 +23,10 @@ static WINDOWS: Lazy>> = static LAST_WND_ID: OnceCell> = OnceCell::new(); -static WND_EVENT_CHANNEL: Lazy> = Lazy::new(|| Channel::new()); - pub type SharedWnd = Arc>; -enum WndEvent { +#[derive(Clone, Debug)] +pub enum WndEvent { Key(KeyCode), } @@ -36,7 +34,7 @@ enum WndEvent { pub struct WndId(u64); impl WndId { - pub fn last_wid() -> StdMutexGuard<'static, WndId> { + fn last_wid() -> StdMutexGuard<'static, WndId> { LAST_WND_ID .get_or_init(|| StdMutex::new(WndId(0))) .lock() @@ -54,25 +52,25 @@ impl Default for WndId { #[async_trait] pub trait Window: Id + RetVal + Send { - async fn check_wnd_events(&mut self) { - if !WND_EVENT_CHANNEL.is_empty() { - match WND_EVENT_CHANNEL.recv() { - WndEvent::Key(k) => { - self.handle_kbd(k).await.unwrap(); - } + async fn handle_event(&mut self, ev: WndEvent) { + match ev { + WndEvent::Key(k) => { + self.handle_kbd(k).await.unwrap(); } } } async fn close(&mut self, force: bool) { let rv = self.retval(); - ReturnValue::set(rv).await; + ReturnValue::set(rv); GENERAL_EVENT_CHANNEL.send(GEvent::CloseWnd { wid: self.id(), force, }); } + fn get_chan(&self) -> &'static AsyncChannel; + fn into_shared(self) -> SharedWnd where Self: Sized + 'static, @@ -90,30 +88,35 @@ pub trait Window: Id + RetVal + Send { } pub struct WndLoop { - upd_tx: Sender<()>, + upd_tx: Sender, updater: JoinHandle<()>, window: SharedWnd, } impl WndLoop { - pub fn with_wnd(window: SharedWnd) -> Self { - let Channel { tx, rx } = Channel::new(); + pub async fn with_wnd(window: SharedWnd) -> Self { let wnd = window.clone(); + let AsyncChannel { tx, rx } = wnd.lock().await.get_chan(); let wnd_loop = async move { loop { - rx.recv().unwrap(); - wnd.lock().await.check_wnd_events().await; + match rx.recv().await { + Ok(ev) => wnd.lock().await.handle_event(ev).await, + Err(_) => break, + } } }; WndLoop { - upd_tx: tx, + upd_tx: tx.clone(), window, updater: task::spawn(wnd_loop), } } - pub fn send_update(&self) { - self.upd_tx.send(()).unwrap(); + pub async fn send(&self, ev: WndEvent) { + let wnd_id = self.window.lock().await.id(); + let event = ev.clone(); + debug!(?event, ?wnd_id, "sending"); + self.upd_tx.send(ev).await.expect("big zhopa"); } } @@ -151,7 +154,8 @@ impl WindowsHandler { pub async fn push_dyn(&mut self, window: SharedWnd) -> SharedWnd { let wid = window.lock().await.id(); - self.queue.insert(wid, WndLoop::with_wnd(window.clone())); + self.queue + .insert(wid, WndLoop::with_wnd(window.clone()).await); window } @@ -160,19 +164,12 @@ impl WindowsHandler { Ok(()) } - pub fn show_cursor(&mut self) -> AResult<()> { - self.term.show_cursor()?; - Ok(()) - } - pub async fn draw(&mut self) -> AResult<()> { let wids_to_redraw = if self.redraw_all { self.redraw_all = false; - let mut wids = self.queue.keys().cloned().collect::>(); - wids.sort(); - wids + self.queue.keys().collect::>() } else { - vec![*WndId::last_wid()] + vec![self.queue.keys().last().unwrap()] }; for wid in wids_to_redraw { let mut wnd_locked = match self.queue.get(&wid) { @@ -202,19 +199,17 @@ impl WindowsHandler { } if self.queue.is_empty() { - self.show_cursor().unwrap(); - teardown().unwrap(); - exit(0); + GENERAL_EVENT_CHANNEL.send(GEvent::Exit); } } - pub fn send_handle_kbd(&self, k: KeyCode) { - WND_EVENT_CHANNEL.send(WndEvent::Key(k)); + pub async fn send_handle_kbd(&self, k: KeyCode) { + let current_wnd = self.get_last_wnd(); + current_wnd.send(WndEvent::Key(k)).await; } pub async fn update(&mut self) -> AResult<()> { let current_wnd = self.get_last_wnd(); - current_wnd.send_update(); current_wnd.window.lock().await.handle_update().await?; Ok(()) } diff --git a/scripts/build_musl_libs.sh b/scripts/build_musl_libs.sh index b63211d..8c688a5 100755 --- a/scripts/build_musl_libs.sh +++ b/scripts/build_musl_libs.sh @@ -5,11 +5,13 @@ ARGS=$@ STATIC_LIBS=./static DOCKER_EXCHG=/musl-share IMAGE=unki/musllibs -mkdir -p $STATIC_LIBS -cd $ROOTDIR/images && docker build -t $IMAGE . -f musl-libs.Dockerfile -docker run \ - -v $ROOTDIR/$STATIC_LIBS:$DOCKER_EXCHG \ - -w /volume \ - -it \ - $IMAGE \ - bash -c "[[ \$(ls -A $DOCKER_EXCHG) ]] || cp /musl/. $DOCKER_EXCHG -r" +if [[ ! -d ./static ]]; then + mkdir $STATIC_LIBS + cd $ROOTDIR/images && docker build -t $IMAGE . -f musl-libs.Dockerfile + docker run \ + -v $ROOTDIR/$STATIC_LIBS:$DOCKER_EXCHG \ + -w /volume \ + -it \ + $IMAGE \ + bash -c "[[ \$(ls -A $DOCKER_EXCHG) ]] || cp /musl/. $DOCKER_EXCHG -r" +fi From 16514b7d1a43db82954ebebea89656dd38881e08 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sun, 23 Jan 2022 15:40:40 +0500 Subject: [PATCH 16/30] i realized that arch is shitty so switch to web frontend --- bin/u_panel/src/tui/mod.rs | 29 +++++---- bin/u_panel/src/tui/retval.rs | 25 ------- bin/u_panel/src/tui/utils.rs | 20 ++---- bin/u_panel/src/tui/windows/confirm.rs | 61 +++++++---------- bin/u_panel/src/tui/windows/main_wnd.rs | 34 ++++------ bin/u_panel/src/tui/windows/mod.rs | 87 +++++++++++++------------ lib/u_lib/src/builder.rs | 13 +++- lib/u_lib/src/errors/variants.rs | 2 +- 8 files changed, 118 insertions(+), 153 deletions(-) delete mode 100644 bin/u_panel/src/tui/retval.rs diff --git a/bin/u_panel/src/tui/mod.rs b/bin/u_panel/src/tui/mod.rs index 5cd9942..80b0a08 100644 --- a/bin/u_panel/src/tui/mod.rs +++ b/bin/u_panel/src/tui/mod.rs @@ -1,5 +1,4 @@ mod impls; -mod retval; mod utils; mod windows; @@ -12,7 +11,6 @@ use crossterm::terminal::{ disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen, }; use once_cell::sync::Lazy; -use retval::{RetVal, ReturnValue}; use std::{ env, io::{stdout, Stdout}, @@ -24,15 +22,14 @@ use std::{ }; use tui::{backend::CrosstermBackend, Terminal}; use utils::{AsyncChannel, Channel}; -use windows::{MainWnd, SharedWnd, WindowsHandler, WndId}; +use windows::{MainWnd, SharedWnd, WndId, WM}; pub type Backend = CrosstermBackend; pub type Frame<'f> = tui::Frame<'f, Backend>; -const EVENT_GEN_PERIOD: Duration = Duration::from_millis(120); +const EVENT_GEN_PERIOD: Duration = Duration::from_millis(70); static GENERAL_EVENT_CHANNEL: Lazy> = Lazy::new(|| Channel::new()); - static ACTIVE_LOOP: AtomicBool = AtomicBool::new(true); enum GEvent { @@ -49,10 +46,10 @@ fn get_terminal() -> AResult> { } pub async fn init_tui(args: &TUIArgs) -> AResult<()> { + let gui = !args.nogui; init_logger(); info!("Initializing u_panel"); - let gui = !args.nogui; init_signal_handlers(gui); init_panic_handler(gui); term_setup(gui)?; @@ -67,15 +64,18 @@ pub async fn init_tui(args: &TUIArgs) -> AResult<()> { async fn init_loop(args: &TUIArgs) -> AResult<()> { let gui = !args.nogui; if gui { - WindowsHandler::lock().await.clear()?; + WM::lock().await.clear()?; } - WindowsHandler::lock().await.push_new::().await; + WM::lock().await.push_new::().await; thread::spawn(move || { while is_running() { if event::poll(EVENT_GEN_PERIOD).unwrap() { match event::read().unwrap() { - Event::Key(key) => GENERAL_EVENT_CHANNEL.send(GEvent::Key(key.code)), + Event::Key(key) => { + GENERAL_EVENT_CHANNEL.send(GEvent::Key(key.code)); + GENERAL_EVENT_CHANNEL.send(GEvent::Tick); + } _ => (), } } else { @@ -87,25 +87,25 @@ async fn init_loop(args: &TUIArgs) -> AResult<()> { while is_running() { match GENERAL_EVENT_CHANNEL.recv() { GEvent::Tick => { - let mut wh = WindowsHandler::lock().await; + let mut wh = WM::lock().await; wh.update().await?; if gui { wh.draw().await?; } } GEvent::CloseWnd { wid, force } => { - WindowsHandler::lock().await.close(wid, force).await; + WM::lock().await.close(wid, force).await; } GEvent::Key(key) => { info!(?key, "pressed"); if let KeyCode::Char('q') = key { break_global(); } else { - WindowsHandler::lock().await.send_handle_kbd(key).await; + WM::lock().await.send_handle_kbd(key).await; } } GEvent::CreateWnd(wnd) => { - WindowsHandler::lock().await.push_dyn(wnd).await; + WM::lock().await.push_dyn(wnd).await; } GEvent::Exit => { break_global(); @@ -163,7 +163,8 @@ fn init_logger() { fn init_panic_handler(gui: bool) { set_hook(Box::new(move |panic_info| { term_teardown(gui).ok(); - eprintln!("{}\n{:?}", panic_info, Backtrace::new()); + eprintln!("CRITICAL PANIK ENCOUNTRD OMFG!11! go see logz lul"); + error!("{}\n{:?}", panic_info, Backtrace::new()); exit(254); })); } diff --git a/bin/u_panel/src/tui/retval.rs b/bin/u_panel/src/tui/retval.rs deleted file mode 100644 index 3874dfc..0000000 --- a/bin/u_panel/src/tui/retval.rs +++ /dev/null @@ -1,25 +0,0 @@ -use crate::tui::Channel; -use once_cell::sync::Lazy; -use std::any::{type_name, Any}; - -type RV = Box; -static RETVAL: Lazy> = Lazy::new(|| Channel::new()); - -pub struct ReturnValue; - -impl ReturnValue { - pub fn set(t: RV) { - RETVAL.send(t); - } - - pub fn get() -> Box { - RETVAL - .recv() - .downcast::() - .expect(&format!("wrong type {}", type_name::())) - } -} - -pub trait RetVal { - fn retval(&self) -> RV; -} diff --git a/bin/u_panel/src/tui/utils.rs b/bin/u_panel/src/tui/utils.rs index e8e8542..6b508be 100644 --- a/bin/u_panel/src/tui/utils.rs +++ b/bin/u_panel/src/tui/utils.rs @@ -1,5 +1,6 @@ use async_channel::{ - unbounded as unbounded_async, Receiver as AsyncReceiver, Sender as AsyncSender, + unbounded as unbounded_async, Receiver as AsyncReceiver, RecvError, SendError, + Sender as AsyncSender, }; use crossbeam::channel::{unbounded, Receiver, Sender}; @@ -21,10 +22,6 @@ impl Channel { pub fn recv(&self) -> T { self.rx.recv().unwrap() } - - pub fn is_empty(&self) -> bool { - self.rx.is_empty() - } } impl Default for Channel { @@ -33,6 +30,7 @@ impl Default for Channel { } } +#[derive(Clone)] pub struct AsyncChannel { pub tx: AsyncSender, pub rx: AsyncReceiver, @@ -44,16 +42,12 @@ impl AsyncChannel { Self { tx, rx } } - pub async fn send(&self, msg: T) { - self.tx.send(msg).await.unwrap() - } - - pub async fn recv(&self) -> T { - self.rx.recv().await.unwrap() + pub async fn send(&self, msg: T) -> Result<(), SendError> { + self.tx.send(msg).await } - pub fn is_empty(&self) -> bool { - self.rx.is_empty() + pub async fn recv(&self) -> Result { + self.rx.recv().await } } diff --git a/bin/u_panel/src/tui/windows/confirm.rs b/bin/u_panel/src/tui/windows/confirm.rs index 8272ffa..319c10c 100644 --- a/bin/u_panel/src/tui/windows/confirm.rs +++ b/bin/u_panel/src/tui/windows/confirm.rs @@ -1,14 +1,10 @@ -use super::{Window, WndId}; -use crate::tui::{ - windows::WndEvent, AsyncChannel, Frame, GEvent, RetVal, ReturnValue, GENERAL_EVENT_CHANNEL, -}; +use super::{ReturnVal, Window, WndId}; +use crate::tui::Frame; use anyhow::Result as AResult; use crossterm::event::KeyCode; -use once_cell::sync::OnceCell; -use std::any::Any; use tui::layout::{Alignment, Constraint, Direction, Layout, Rect}; -use tui::style::{Modifier, Style}; -use tui::widgets::{Block, Clear, List, ListItem, ListState, Paragraph}; +use tui::style::{Color, Modifier, Style}; +use tui::widgets::{Block, Borders, Clear, List, ListItem, ListState, Paragraph}; #[derive(Default)] pub struct ConfirmWnd { @@ -19,15 +15,15 @@ pub struct ConfirmWnd { } impl ConfirmWnd { - pub fn new_yn(msg: impl Into, variants: Option>) -> Self { - let default = vec!["Yes".to_string(), "No".to_string()]; - let variants = match variants { - Some(v) if !v.is_empty() => v, - _ => default, + pub fn new(msg: impl Into, variants: &[&str]) -> Self { + let variants = if !variants.is_empty() { + variants + } else { + &["Yes", "No"] }; let mut this = Self { msg: msg.into(), - variants, + variants: variants.into_iter().map(|s| s.to_string()).collect(), ..Default::default() }; this.state.select(Some(0)); @@ -48,24 +44,14 @@ impl ConfirmWnd { } } -impl RetVal for ConfirmWnd { - fn retval(&self) -> Box { - let value = self - .variants - .get(self.state.selected().unwrap()) - .unwrap() - .to_owned(); - Box::new(value) - } -} - #[async_trait] impl Window for ConfirmWnd { async fn handle_kbd(&mut self, k: KeyCode) -> AResult<()> { match k { KeyCode::Right => self.on_right(), KeyCode::Left => self.on_left(), - KeyCode::Enter | KeyCode::Esc => self.close(false).await, + KeyCode::Enter => self.close(false, true).await, + KeyCode::Esc => self.close(false, false).await, _ => (), } Ok(()) @@ -82,10 +68,13 @@ impl Window for ConfirmWnd { fn draw(&mut self, f: &mut Frame) { let size = f.size(); let rect = centered_rect(60, 40, size); + let popup = Block::default().title("Popup").borders(Borders::ALL); + f.render_widget(Clear, rect); + f.render_widget(popup, rect); let chunks = Layout::default() .direction(Direction::Vertical) - .constraints(vec![Constraint::Percentage(70), Constraint::Percentage(30)]) + .constraints(vec![Constraint::Percentage(40), Constraint::Percentage(30)]) .split(rect); let msg = Paragraph::new(self.msg.as_ref()); f.render_widget(msg, chunks[0]); @@ -97,22 +86,20 @@ impl Window for ConfirmWnd { .map(ListItem::new) .collect::>(); let list = - List::new(options).highlight_style(Style::default().add_modifier(Modifier::BOLD)); + List::new(options).highlight_style(Style::default().bg(Color::Gray).fg(Color::Black)); f.render_stateful_widget(list, chunks[1], &mut self.state); } - fn get_chan(&self) -> &'static AsyncChannel { - static EV_CHAN: OnceCell> = OnceCell::new(); - EV_CHAN.get_or_init(|| AsyncChannel::new()) + fn retval(&self) -> ReturnVal { + let value = self + .variants + .get(self.state.selected().unwrap()) + .unwrap() + .to_owned(); + ReturnVal::String(value) } } -pub async fn confirm_wnd(msg: impl Into) -> String { - let wnd = ConfirmWnd::new_yn(msg.into(), None); - GENERAL_EVENT_CHANNEL.send(GEvent::CreateWnd(wnd.into_shared())); - *ReturnValue::get() -} - fn centered_rect(percent_x: u16, percent_y: u16, r: Rect) -> Rect { let popup_layout = Layout::default() .direction(Direction::Vertical) diff --git a/bin/u_panel/src/tui/windows/main_wnd.rs b/bin/u_panel/src/tui/windows/main_wnd.rs index 1a7c40a..d6494ee 100644 --- a/bin/u_panel/src/tui/windows/main_wnd.rs +++ b/bin/u_panel/src/tui/windows/main_wnd.rs @@ -1,13 +1,7 @@ -use super::{confirm_wnd, Window}; -use crate::tui::{ - impls::CRUD, - windows::{WndEvent, WndId}, - AsyncChannel, Frame, RetVal, -}; +use super::{ConfirmWnd, ReturnVal, Window}; +use crate::tui::{impls::CRUD, windows::WndId, Frame}; use anyhow::Result as AResult; use crossterm::event::KeyCode; -use once_cell::sync::OnceCell; -use std::any::Any; use std::{fmt::Display, str::FromStr}; use strum::VariantNames; use tokio::join; @@ -205,6 +199,7 @@ impl MainWnd { UiTabs::Agents => { self.agents.updated = false; self.jobs.updated = false; + self.map.updated = false; } UiTabs::Jobs => self.jobs.updated = false, UiTabs::Map => self.map.updated = false, @@ -255,15 +250,19 @@ impl MainWnd { impl Window for MainWnd { async fn handle_kbd(&mut self, k: KeyCode) -> AResult<()> { match k { - KeyCode::Esc => self.close(false).await, + KeyCode::Esc => self.close(false, false).await, KeyCode::Left => self.prev_tab(), KeyCode::Right => self.next_tab(), KeyCode::Up => self.on_up(), KeyCode::Down => self.on_down(), KeyCode::Delete => { - if &confirm_wnd("Delete?").await == "Yes" { - self.delete().await; - self.update_tab(); + if let ReturnVal::String(ref s) = + ConfirmWnd::new("Delete?", &["Yes", "No"]).wait_retval() + { + if s == "Yes" { + self.delete().await; + self.update_tab(); + } } } KeyCode::F(5) => self.update_tab(), @@ -316,15 +315,8 @@ impl Window for MainWnd { f.render_stateful_widget(list, chunks[1], self.tab_list_state()); } - fn get_chan(&self) -> &'static AsyncChannel { - static EV_CHAN: OnceCell> = OnceCell::new(); - EV_CHAN.get_or_init(|| AsyncChannel::new()) - } -} - -impl RetVal for MainWnd { - fn retval(&self) -> Box { - Box::new(()) + fn retval(&self) -> ReturnVal { + ReturnVal::None } } diff --git a/bin/u_panel/src/tui/windows/mod.rs b/bin/u_panel/src/tui/windows/mod.rs index c4b18a5..ec86b3f 100644 --- a/bin/u_panel/src/tui/windows/mod.rs +++ b/bin/u_panel/src/tui/windows/mod.rs @@ -1,30 +1,34 @@ mod confirm; mod main_wnd; -use async_channel::Sender; -pub use confirm::{confirm_wnd, ConfirmWnd}; +pub use confirm::ConfirmWnd; pub use main_wnd::MainWnd; use crate::tui::{ - get_terminal, impls::Id, AsyncChannel, Backend, Frame, GEvent, RetVal, ReturnValue, - GENERAL_EVENT_CHANNEL, + get_terminal, impls::Id, AsyncChannel, Backend, Channel, Frame, GEvent, GENERAL_EVENT_CHANNEL, }; use anyhow::Result as AResult; use crossterm::event::KeyCode; use once_cell::sync::{Lazy, OnceCell}; use std::collections::BTreeMap; -use std::sync::Arc; -use std::sync::{Mutex as StdMutex, MutexGuard as StdMutexGuard}; +use std::sync::{Arc, Mutex as StdMutex, MutexGuard as StdMutexGuard}; use tokio::sync::{Mutex, MutexGuard}; use tokio::task::{self, JoinHandle}; use tui::Terminal; -static WINDOWS: Lazy>> = - Lazy::new(|| Arc::new(Mutex::new(WindowsHandler::new(get_terminal().unwrap())))); +static WINDOWS: Lazy>> = + Lazy::new(|| Arc::new(Mutex::new(WM::new(get_terminal().unwrap())))); static LAST_WND_ID: OnceCell> = OnceCell::new(); +static RETVAL: Lazy> = Lazy::new(|| Channel::new()); + pub type SharedWnd = Arc>; +pub enum ReturnVal { + String(String), + None, +} + #[derive(Clone, Debug)] pub enum WndEvent { Key(KeyCode), @@ -51,7 +55,7 @@ impl Default for WndId { } #[async_trait] -pub trait Window: Id + RetVal + Send { +pub trait Window: Id + Send { async fn handle_event(&mut self, ev: WndEvent) { match ev { WndEvent::Key(k) => { @@ -60,22 +64,27 @@ pub trait Window: Id + RetVal + Send { } } - async fn close(&mut self, force: bool) { - let rv = self.retval(); - ReturnValue::set(rv); + async fn close(&mut self, force: bool, save_retval: bool) { + let rv = if save_retval { + self.retval() + } else { + ReturnVal::None + }; + RETVAL.send(rv); GENERAL_EVENT_CHANNEL.send(GEvent::CloseWnd { wid: self.id(), force, }); } - fn get_chan(&self) -> &'static AsyncChannel; + fn retval(&self) -> ReturnVal; - fn into_shared(self) -> SharedWnd + fn wait_retval(self) -> ReturnVal where Self: Sized + 'static, { - Arc::new(Mutex::new(self)) + GENERAL_EVENT_CHANNEL.send(GEvent::CreateWnd(Arc::new(Mutex::new(self)))); + RETVAL.recv() } async fn handle_update(&mut self) -> AResult<()>; @@ -88,27 +97,28 @@ pub trait Window: Id + RetVal + Send { } pub struct WndLoop { - upd_tx: Sender, - updater: JoinHandle<()>, + chan: AsyncChannel, + wnd_loop: JoinHandle<()>, window: SharedWnd, } impl WndLoop { pub async fn with_wnd(window: SharedWnd) -> Self { let wnd = window.clone(); - let AsyncChannel { tx, rx } = wnd.lock().await.get_chan(); + let chan = AsyncChannel::::new(); + let ch = chan.clone(); let wnd_loop = async move { loop { - match rx.recv().await { + match ch.recv().await { Ok(ev) => wnd.lock().await.handle_event(ev).await, Err(_) => break, } } }; WndLoop { - upd_tx: tx.clone(), + chan, window, - updater: task::spawn(wnd_loop), + wnd_loop: task::spawn(wnd_loop), } } @@ -116,17 +126,16 @@ impl WndLoop { let wnd_id = self.window.lock().await.id(); let event = ev.clone(); debug!(?event, ?wnd_id, "sending"); - self.upd_tx.send(ev).await.expect("big zhopa"); + self.chan.send(ev).await.expect("send failed"); } } -pub struct WindowsHandler { +pub struct WM { queue: BTreeMap, term: Terminal, - redraw_all: bool, } -impl WindowsHandler { +impl WM { fn get_last_wnd(&self) -> &WndLoop { let last_id = self.queue.keys().rev().next().expect("No windows found"); self.queue.get(last_id).unwrap() @@ -140,12 +149,11 @@ impl WindowsHandler { Self { term, queue: BTreeMap::new(), - redraw_all: true, } } pub async fn push(&mut self, window: W) -> SharedWnd { - self.push_dyn(window.into_shared()).await + self.push_dyn(Arc::new(Mutex::new(window))).await } pub async fn push_new(&mut self) -> SharedWnd { @@ -165,17 +173,17 @@ impl WindowsHandler { } pub async fn draw(&mut self) -> AResult<()> { - let wids_to_redraw = if self.redraw_all { - self.redraw_all = false; - self.queue.keys().collect::>() - } else { - vec![self.queue.keys().last().unwrap()] - }; - for wid in wids_to_redraw { + for wid in self.queue.keys().collect::>() { let mut wnd_locked = match self.queue.get(&wid) { - Some(w) => w.window.lock().await, + Some(w) => match w.window.try_lock() { + Ok(w) => w, + Err(_) => { + warn!("Can't lock window {:?}", wid); + continue; + } + }, None => { - eprintln!("Can't redraw window {:?}, not found", wid); + warn!("Can't redraw window {:?}, not found", wid); continue; } }; @@ -188,14 +196,13 @@ impl WindowsHandler { let wnd = match self.queue.get(&wid) { Some(w) => w.clone(), None => { - eprintln!("Can't close window {:?}, not found", wid); + warn!("Can't close window {:?}, not found", wid); return; } }; if wnd.window.lock().await.handle_close().await || force { - let WndLoop { updater, .. } = self.queue.remove(&wid).unwrap(); - updater.abort(); - self.redraw_all = true; + let WndLoop { wnd_loop, .. } = self.queue.remove(&wid).unwrap(); + wnd_loop.abort(); } if self.queue.is_empty() { diff --git a/lib/u_lib/src/builder.rs b/lib/u_lib/src/builder.rs index e6ce08f..746cc12 100644 --- a/lib/u_lib/src/builder.rs +++ b/lib/u_lib/src/builder.rs @@ -1,4 +1,13 @@ -use crate::{UError, UErrorBt, UResult, cache::JobCache, executor::{Waiter, DynFut}, messaging::Reportable, models::{Agent, AssignedJob, JobMeta, JobType}, utils::{CombinedResult, OneOrVec}}; +use crate::{ + UError, + UErrorBt, + UResult, + cache::JobCache, + executor::{Waiter, DynFut}, + messaging::Reportable, + models::{Agent, AssignedJob, JobMeta, JobType}, + utils::{CombinedResult, OneOrVec} +}; use guess_host_triple::guess_host_triple; use std::collections::HashMap; @@ -6,7 +15,7 @@ pub struct JobBuilder { waiter: Waiter, } -impl JobBuilder { +impl JobBuilder { pub fn from_request(job_requests: impl OneOrVec) -> CombinedResult { let job_requests = job_requests.into_vec(); let mut prepared: Vec = vec![]; diff --git a/lib/u_lib/src/errors/variants.rs b/lib/u_lib/src/errors/variants.rs index d84db3a..48d2fbb 100644 --- a/lib/u_lib/src/errors/variants.rs +++ b/lib/u_lib/src/errors/variants.rs @@ -66,7 +66,7 @@ pub enum UError { #[error("Panicked: {0}")] Panic(String), - #[error("UI init error: {0}")] + #[error("UI error: {0}")] TUIError(String), } From 88f17eab0281ee8ceb94ac253fe24a39ea03aa20 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sun, 27 Mar 2022 17:31:42 +0500 Subject: [PATCH 17/30] added web frontend --- .cargo/config.toml | 6 +- .gitignore | 1 + Cargo.toml | 5 +- Makefile.toml | 10 +- bin/u_agent/Cargo.toml | 2 +- bin/u_panel/Cargo.toml | 4 +- bin/u_panel/be/Cargo.toml | 10 -- bin/u_panel/be/src/lib.rs | 23 ---- bin/u_panel/fe/Cargo.toml | 12 -- bin/u_panel/fe/index.html | 9 -- bin/u_panel/fe/src/main.rs | 51 -------- bin/u_panel/src/argparse.rs | 4 +- bin/u_panel/src/main.rs | 1 + bin/u_panel/src/server/fe/README.md | 27 +++++ bin/u_panel/src/server/fe/angular.json | 111 ++++++++++++++++++ bin/u_panel/src/server/fe/karma.conf.js | 44 +++++++ bin/u_panel/src/server/fe/package.json | 39 ++++++ .../server/fe/src/app/app-routing.module.ts | 10 ++ .../src/server/fe/src/app/app.component.html | 1 + .../src/server/fe/src/app/app.component.less | 0 .../server/fe/src/app/app.component.spec.ts | 35 ++++++ .../src/server/fe/src/app/app.component.ts | 10 ++ .../src/server/fe/src/app/app.module.ts | 18 +++ .../fe/src/environments/environment.prod.ts | 3 + .../server/fe/src/environments/environment.ts | 16 +++ bin/u_panel/src/server/fe/src/favicon.ico | Bin 0 -> 948 bytes bin/u_panel/src/server/fe/src/index.html | 13 ++ bin/u_panel/src/server/fe/src/main.ts | 12 ++ bin/u_panel/src/server/fe/src/polyfills.ts | 53 +++++++++ bin/u_panel/src/server/fe/src/styles.less | 1 + bin/u_panel/src/server/fe/src/test.ts | 26 ++++ bin/u_panel/src/server/fe/tsconfig.app.json | 15 +++ bin/u_panel/src/server/fe/tsconfig.json | 32 +++++ bin/u_panel/src/server/fe/tsconfig.spec.json | 18 +++ bin/u_panel/src/server/mod.rs | 56 +++++++++ bin/u_server/Cargo.toml | 2 +- lib/u_lib/src/errors/variants.rs | 4 +- 37 files changed, 567 insertions(+), 117 deletions(-) delete mode 100644 bin/u_panel/be/Cargo.toml delete mode 100644 bin/u_panel/be/src/lib.rs delete mode 100644 bin/u_panel/fe/Cargo.toml delete mode 100644 bin/u_panel/fe/index.html delete mode 100644 bin/u_panel/fe/src/main.rs create mode 100644 bin/u_panel/src/server/fe/README.md create mode 100644 bin/u_panel/src/server/fe/angular.json create mode 100644 bin/u_panel/src/server/fe/karma.conf.js create mode 100644 bin/u_panel/src/server/fe/package.json create mode 100644 bin/u_panel/src/server/fe/src/app/app-routing.module.ts create mode 100644 bin/u_panel/src/server/fe/src/app/app.component.html create mode 100644 bin/u_panel/src/server/fe/src/app/app.component.less create mode 100644 bin/u_panel/src/server/fe/src/app/app.component.spec.ts create mode 100644 bin/u_panel/src/server/fe/src/app/app.component.ts create mode 100644 bin/u_panel/src/server/fe/src/app/app.module.ts create mode 100644 bin/u_panel/src/server/fe/src/environments/environment.prod.ts create mode 100644 bin/u_panel/src/server/fe/src/environments/environment.ts create mode 100644 bin/u_panel/src/server/fe/src/favicon.ico create mode 100644 bin/u_panel/src/server/fe/src/index.html create mode 100644 bin/u_panel/src/server/fe/src/main.ts create mode 100644 bin/u_panel/src/server/fe/src/polyfills.ts create mode 100644 bin/u_panel/src/server/fe/src/styles.less create mode 100644 bin/u_panel/src/server/fe/src/test.ts create mode 100644 bin/u_panel/src/server/fe/tsconfig.app.json create mode 100644 bin/u_panel/src/server/fe/tsconfig.json create mode 100644 bin/u_panel/src/server/fe/tsconfig.spec.json create mode 100644 bin/u_panel/src/server/mod.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 21748f1..433fea9 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,6 @@ [build] -rustflags = ["-L", "/home/ortem/src/rust/unki/static/lib"] \ No newline at end of file +rustflags = [ + "-L", "/home/ortem/src/rust/unki/static/lib", + "--remap-path-prefix=/home/ortem/src/rust/unki=src", + "--remap-path-prefix=/home/ortem/.cargo=cargo" +] diff --git a/.gitignore b/.gitignore index 3b8e7ff..fd55330 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ certs/ static/ .vscode/ release/ +**/node_modules/ **/*.rs.bk **/*.pyc diff --git a/Cargo.toml b/Cargo.toml index c22386b..a9f8f02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,8 +2,6 @@ members = [ "bin/u_agent", "bin/u_panel", - #"bin/u_panel/be", - #"bin/u_panel/fe", "bin/u_run", "bin/u_server", "lib/u_lib", @@ -13,7 +11,8 @@ members = [ [profile.release] panic = "abort" +strip = "symbols" [profile.dev] debug = true # Добавляет флаг `-g` для компилятора; -opt-level = 0 \ No newline at end of file +opt-level = 0 diff --git a/Makefile.toml b/Makefile.toml index 45c2842..4c11b71 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -13,12 +13,18 @@ OPENSSL_DIR = "${PREFIX}" [tasks.build_static_libs] script = "./scripts/build_musl_libs.sh" +[tasks.build_frontend] +script = ''' +cd ./bin/u_panel/src/server/fe +ng build +''' + [tasks.clean] command = "${CARGO}" args = ["clean"] [tasks.cargo_build] -dependencies = ["build_static_libs"] +dependencies = ["build_static_libs", "build_frontend"] command = "${CARGO}" args = ["build", "--target", "${TARGET}", "${@}"] @@ -39,7 +45,7 @@ args = [] [tasks.run] script = ''' -echo "Only integration tests are supported." +echo "wtf are you running? run binaries dud!" exit 1 ''' diff --git a/bin/u_agent/Cargo.toml b/bin/u_agent/Cargo.toml index 3f7a884..500fdb0 100644 --- a/bin/u_agent/Cargo.toml +++ b/bin/u_agent/Cargo.toml @@ -17,4 +17,4 @@ openssl = "*" u_lib = { version = "*", path = "../../lib/u_lib" } [build-dependencies] -openssl = "*" \ No newline at end of file +openssl = "*" diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 4fb5761..2bccf84 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +actix-web = "3.3.2" backtrace = "0.3.61" structopt = "0.3.21" log = "^0.4" @@ -15,7 +16,6 @@ uuid = "0.6.5" serde_json = "1.0.4" serde = { version = "1.0.114", features = ["derive"] } tokio = { version = "1.11.0", features = ["rt", "rt-multi-thread"] } -# be = { version = "*", path = "./be" } u_lib = { version = "*", path = "../../lib/u_lib" } tui = { version = "0.16", default-features = false, features = ['crossterm'] } crossterm = "0.22.1" @@ -29,3 +29,5 @@ tracing = "0.1.29" tracing-subscriber = { version = "0.3.3", features = ["env-filter"]} signal-hook = "0.3.12" tracing-appender = "0.2.0" +rust-embed = { version = "6.3.0", features = ["debug-embed", "compression"] } +mime_guess = "2.0.4" diff --git a/bin/u_panel/be/Cargo.toml b/bin/u_panel/be/Cargo.toml deleted file mode 100644 index 8831379..0000000 --- a/bin/u_panel/be/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "be" -version = "0.1.0" -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -actix-web = "3.3.2" -u_lib = { version = "*", path = "../../../lib/u_lib" } \ No newline at end of file diff --git a/bin/u_panel/be/src/lib.rs b/bin/u_panel/be/src/lib.rs deleted file mode 100644 index d7c5117..0000000 --- a/bin/u_panel/be/src/lib.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* -Tabs: Agents, Tasks, Summary -every tab has list page and item page with more info/actions - -Agents: - -| id | alias | ..see struct | tasks done -| stripped | alias | ... | clickable number of assigned jobs - -almost all fields are editable, rows are deletable - - -*/ - -use actix_web::{web, App, HttpResponse, HttpServer}; - -#[actix_web::main] -pub async fn serve() -> std::io::Result<()> { - let addr = "127.0.0.1:8080"; - let app = || App::new().route("/", web::get().to(|| HttpResponse::Ok().body("ok"))); - println!("Serving at http://{}", addr); - HttpServer::new(app).bind(addr)?.run().await -} diff --git a/bin/u_panel/fe/Cargo.toml b/bin/u_panel/fe/Cargo.toml deleted file mode 100644 index f84fa64..0000000 --- a/bin/u_panel/fe/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "fe" -version = "0.1.0" -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -u_lib = { version = "*", path = "../../../lib/u_lib" } -wasm-bindgen = "0.2.78" -yew = "0.18.0" -yew-router = "0.15.0" diff --git a/bin/u_panel/fe/index.html b/bin/u_panel/fe/index.html deleted file mode 100644 index de24c98..0000000 --- a/bin/u_panel/fe/index.html +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - Yew App - - - \ No newline at end of file diff --git a/bin/u_panel/fe/src/main.rs b/bin/u_panel/fe/src/main.rs deleted file mode 100644 index c7e8f03..0000000 --- a/bin/u_panel/fe/src/main.rs +++ /dev/null @@ -1,51 +0,0 @@ -use wasm_bindgen::prelude::*; -use yew::prelude::*; -enum Msg { - AddOne, -} -struct Model { - // `ComponentLink` is like a reference to a component. - // It can be used to send messages to the component - link: ComponentLink, - value: i64, -} - -impl Component for Model { - type Message = Msg; - type Properties = (); - - fn create(_props: Self::Properties, link: ComponentLink) -> Self { - Self { link, value: 0 } - } - - fn update(&mut self, msg: Self::Message) -> ShouldRender { - match msg { - Msg::AddOne => { - self.value += 1; - // the value has changed so we need to - // re-render for it to appear on the page - true - } - } - } - - fn change(&mut self, _props: Self::Properties) -> ShouldRender { - // Should only return "true" if new properties are different to - // previously received properties. - // This component has no properties so we will always return "false". - false - } - - fn view(&self) -> Html { - html! { -
- -

{ self.value }

-
- } - } -} - -pub fn main() { - yew::start_app::(); -} diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 651e50d..879ccbb 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -18,6 +18,7 @@ enum Cmd { Jobs(JobALD), Map(JobMapALD), TUI(TUIArgs), + Serve, } #[derive(StructOpt, Debug)] @@ -134,7 +135,8 @@ pub async fn process_cmd(args: Args) -> UResult<()> { }, Cmd::TUI(args) => crate::tui::init_tui(&args) .await - .map_err(|e| UError::TUIError(e.to_string()))?, + .map_err(|e| UError::PanelError(e.to_string()))?, + Cmd::Serve => crate::server::serve().map_err(|e| UError::PanelError(e.to_string()))?, } Ok(()) } diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index 63e9e06..d4ba19a 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -1,5 +1,6 @@ mod argparse; mod tui; +mod server; #[macro_use] extern crate async_trait; diff --git a/bin/u_panel/src/server/fe/README.md b/bin/u_panel/src/server/fe/README.md new file mode 100644 index 0000000..d16d556 --- /dev/null +++ b/bin/u_panel/src/server/fe/README.md @@ -0,0 +1,27 @@ +# Fe + +This project was generated with [Angular CLI](https://github.com/angular/angular-cli) version 13.1.2. + +## Development server + +Run `ng serve` for a dev server. Navigate to `http://localhost:4200/`. The app will automatically reload if you change any of the source files. + +## Code scaffolding + +Run `ng generate component component-name` to generate a new component. You can also use `ng generate directive|pipe|service|class|guard|interface|enum|module`. + +## Build + +Run `ng build` to build the project. The build artifacts will be stored in the `dist/` directory. + +## Running unit tests + +Run `ng test` to execute the unit tests via [Karma](https://karma-runner.github.io). + +## Running end-to-end tests + +Run `ng e2e` to execute the end-to-end tests via a platform of your choice. To use this command, you need to first add a package that implements end-to-end testing capabilities. + +## Further help + +To get more help on the Angular CLI use `ng help` or go check out the [Angular CLI Overview and Command Reference](https://angular.io/cli) page. diff --git a/bin/u_panel/src/server/fe/angular.json b/bin/u_panel/src/server/fe/angular.json new file mode 100644 index 0000000..5bb4463 --- /dev/null +++ b/bin/u_panel/src/server/fe/angular.json @@ -0,0 +1,111 @@ +{ + "$schema": "./node_modules/@angular/cli/lib/config/schema.json", + "version": 1, + "newProjectRoot": "projects", + "projects": { + "fe": { + "projectType": "application", + "schematics": { + "@schematics/angular:component": { + "style": "less" + }, + "@schematics/angular:application": { + "strict": true + } + }, + "root": "", + "sourceRoot": "src", + "prefix": "app", + "architect": { + "build": { + "builder": "@angular-devkit/build-angular:browser", + "options": { + "outputPath": "dist/fe", + "index": "src/index.html", + "main": "src/main.ts", + "polyfills": "src/polyfills.ts", + "tsConfig": "tsconfig.app.json", + "inlineStyleLanguage": "less", + "assets": [ + "src/favicon.ico", + "src/assets" + ], + "styles": [ + "src/styles.less" + ], + "scripts": [] + }, + "configurations": { + "production": { + "budgets": [ + { + "type": "initial", + "maximumWarning": "500kb", + "maximumError": "1mb" + }, + { + "type": "anyComponentStyle", + "maximumWarning": "2kb", + "maximumError": "4kb" + } + ], + "fileReplacements": [ + { + "replace": "src/environments/environment.ts", + "with": "src/environments/environment.prod.ts" + } + ], + "outputHashing": "all" + }, + "development": { + "buildOptimizer": false, + "optimization": false, + "vendorChunk": true, + "extractLicenses": false, + "sourceMap": true, + "namedChunks": true + } + }, + "defaultConfiguration": "production" + }, + "serve": { + "builder": "@angular-devkit/build-angular:dev-server", + "configurations": { + "production": { + "browserTarget": "fe:build:production" + }, + "development": { + "browserTarget": "fe:build:development" + } + }, + "defaultConfiguration": "development" + }, + "extract-i18n": { + "builder": "@angular-devkit/build-angular:extract-i18n", + "options": { + "browserTarget": "fe:build" + } + }, + "test": { + "builder": "@angular-devkit/build-angular:karma", + "options": { + "main": "src/test.ts", + "polyfills": "src/polyfills.ts", + "tsConfig": "tsconfig.spec.json", + "karmaConfig": "karma.conf.js", + "inlineStyleLanguage": "less", + "assets": [ + "src/favicon.ico", + "src/assets" + ], + "styles": [ + "src/styles.less" + ], + "scripts": [] + } + } + } + } + }, + "defaultProject": "fe" +} diff --git a/bin/u_panel/src/server/fe/karma.conf.js b/bin/u_panel/src/server/fe/karma.conf.js new file mode 100644 index 0000000..07a1538 --- /dev/null +++ b/bin/u_panel/src/server/fe/karma.conf.js @@ -0,0 +1,44 @@ +// Karma configuration file, see link for more information +// https://karma-runner.github.io/1.0/config/configuration-file.html + +module.exports = function (config) { + config.set({ + basePath: '', + frameworks: ['jasmine', '@angular-devkit/build-angular'], + plugins: [ + require('karma-jasmine'), + require('karma-chrome-launcher'), + require('karma-jasmine-html-reporter'), + require('karma-coverage'), + require('@angular-devkit/build-angular/plugins/karma') + ], + client: { + jasmine: { + // you can add configuration options for Jasmine here + // the possible options are listed at https://jasmine.github.io/api/edge/Configuration.html + // for example, you can disable the random execution with `random: false` + // or set a specific seed with `seed: 4321` + }, + clearContext: false // leave Jasmine Spec Runner output visible in browser + }, + jasmineHtmlReporter: { + suppressAll: true // removes the duplicated traces + }, + coverageReporter: { + dir: require('path').join(__dirname, './coverage/fe'), + subdir: '.', + reporters: [ + { type: 'html' }, + { type: 'text-summary' } + ] + }, + reporters: ['progress', 'kjhtml'], + port: 9876, + colors: true, + logLevel: config.LOG_INFO, + autoWatch: true, + browsers: ['Chrome'], + singleRun: false, + restartOnFileChange: true + }); +}; diff --git a/bin/u_panel/src/server/fe/package.json b/bin/u_panel/src/server/fe/package.json new file mode 100644 index 0000000..ae1f9de --- /dev/null +++ b/bin/u_panel/src/server/fe/package.json @@ -0,0 +1,39 @@ +{ + "name": "fe", + "version": "0.0.0", + "scripts": { + "ng": "ng", + "start": "ng serve", + "build": "ng build", + "watch": "ng build --watch --configuration development", + "test": "ng test" + }, + "private": true, + "dependencies": { + "@angular/animations": "~13.1.0", + "@angular/common": "~13.1.0", + "@angular/compiler": "~13.1.0", + "@angular/core": "~13.1.0", + "@angular/forms": "~13.1.0", + "@angular/platform-browser": "~13.1.0", + "@angular/platform-browser-dynamic": "~13.1.0", + "@angular/router": "~13.1.0", + "rxjs": "~7.4.0", + "tslib": "^2.3.0", + "zone.js": "~0.11.4" + }, + "devDependencies": { + "@angular-devkit/build-angular": "~13.1.2", + "@angular/cli": "~13.1.2", + "@angular/compiler-cli": "~13.1.0", + "@types/jasmine": "~3.10.0", + "@types/node": "^12.11.1", + "jasmine-core": "~3.10.0", + "karma": "~6.3.0", + "karma-chrome-launcher": "~3.1.0", + "karma-coverage": "~2.1.0", + "karma-jasmine": "~4.0.0", + "karma-jasmine-html-reporter": "~1.7.0", + "typescript": "~4.5.2" + } +} diff --git a/bin/u_panel/src/server/fe/src/app/app-routing.module.ts b/bin/u_panel/src/server/fe/src/app/app-routing.module.ts new file mode 100644 index 0000000..0297262 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/app-routing.module.ts @@ -0,0 +1,10 @@ +import { NgModule } from '@angular/core'; +import { RouterModule, Routes } from '@angular/router'; + +const routes: Routes = []; + +@NgModule({ + imports: [RouterModule.forRoot(routes)], + exports: [RouterModule] +}) +export class AppRoutingModule { } diff --git a/bin/u_panel/src/server/fe/src/app/app.component.html b/bin/u_panel/src/server/fe/src/app/app.component.html new file mode 100644 index 0000000..171ac03 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/app.component.html @@ -0,0 +1 @@ +{{ title }} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/app.component.less b/bin/u_panel/src/server/fe/src/app/app.component.less new file mode 100644 index 0000000..e69de29 diff --git a/bin/u_panel/src/server/fe/src/app/app.component.spec.ts b/bin/u_panel/src/server/fe/src/app/app.component.spec.ts new file mode 100644 index 0000000..d2ec0e1 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/app.component.spec.ts @@ -0,0 +1,35 @@ +import { TestBed } from '@angular/core/testing'; +import { RouterTestingModule } from '@angular/router/testing'; +import { AppComponent } from './app.component'; + +describe('AppComponent', () => { + beforeEach(async () => { + await TestBed.configureTestingModule({ + imports: [ + RouterTestingModule + ], + declarations: [ + AppComponent + ], + }).compileComponents(); + }); + + it('should create the app', () => { + const fixture = TestBed.createComponent(AppComponent); + const app = fixture.componentInstance; + expect(app).toBeTruthy(); + }); + + it(`should have as title 'fe'`, () => { + const fixture = TestBed.createComponent(AppComponent); + const app = fixture.componentInstance; + expect(app.title).toEqual('fe'); + }); + + it('should render title', () => { + const fixture = TestBed.createComponent(AppComponent); + fixture.detectChanges(); + const compiled = fixture.nativeElement as HTMLElement; + expect(compiled.querySelector('.content span')?.textContent).toContain('fe app is running!'); + }); +}); diff --git a/bin/u_panel/src/server/fe/src/app/app.component.ts b/bin/u_panel/src/server/fe/src/app/app.component.ts new file mode 100644 index 0000000..25a2bce --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/app.component.ts @@ -0,0 +1,10 @@ +import { Component } from '@angular/core'; + +@Component({ + selector: 'app-root', + templateUrl: './app.component.html', + styleUrls: ['./app.component.less'] +}) +export class AppComponent { + title = 'ты лох'; +} diff --git a/bin/u_panel/src/server/fe/src/app/app.module.ts b/bin/u_panel/src/server/fe/src/app/app.module.ts new file mode 100644 index 0000000..b1c6c96 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/app.module.ts @@ -0,0 +1,18 @@ +import { NgModule } from '@angular/core'; +import { BrowserModule } from '@angular/platform-browser'; + +import { AppRoutingModule } from './app-routing.module'; +import { AppComponent } from './app.component'; + +@NgModule({ + declarations: [ + AppComponent + ], + imports: [ + BrowserModule, + AppRoutingModule + ], + providers: [], + bootstrap: [AppComponent] +}) +export class AppModule { } diff --git a/bin/u_panel/src/server/fe/src/environments/environment.prod.ts b/bin/u_panel/src/server/fe/src/environments/environment.prod.ts new file mode 100644 index 0000000..3612073 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/environments/environment.prod.ts @@ -0,0 +1,3 @@ +export const environment = { + production: true +}; diff --git a/bin/u_panel/src/server/fe/src/environments/environment.ts b/bin/u_panel/src/server/fe/src/environments/environment.ts new file mode 100644 index 0000000..f56ff47 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/environments/environment.ts @@ -0,0 +1,16 @@ +// This file can be replaced during build by using the `fileReplacements` array. +// `ng build` replaces `environment.ts` with `environment.prod.ts`. +// The list of file replacements can be found in `angular.json`. + +export const environment = { + production: false +}; + +/* + * For easier debugging in development mode, you can import the following file + * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`. + * + * This import should be commented out in production mode because it will have a negative impact + * on performance if an error is thrown. + */ +// import 'zone.js/plugins/zone-error'; // Included with Angular CLI. diff --git a/bin/u_panel/src/server/fe/src/favicon.ico b/bin/u_panel/src/server/fe/src/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..997406ad22c29aae95893fb3d666c30258a09537 GIT binary patch literal 948 zcmV;l155mgP)CBYU7IjCFmI-B}4sMJt3^s9NVg!P0 z6hDQy(L`XWMkB@zOLgN$4KYz;j0zZxq9KKdpZE#5@k0crP^5f9KO};h)ZDQ%ybhht z%t9#h|nu0K(bJ ztIkhEr!*UyrZWQ1k2+YkGqDi8Z<|mIN&$kzpKl{cNP=OQzXHz>vn+c)F)zO|Bou>E z2|-d_=qY#Y+yOu1a}XI?cU}%04)zz%anD(XZC{#~WreV!a$7k2Ug`?&CUEc0EtrkZ zL49MB)h!_K{H(*l_93D5tO0;BUnvYlo+;yss%n^&qjt6fZOa+}+FDO(~2>G z2dx@=JZ?DHP^;b7*Y1as5^uphBsh*s*z&MBd?e@I>-9kU>63PjP&^#5YTOb&x^6Cf z?674rmSHB5Fk!{Gv7rv!?qX#ei_L(XtwVqLX3L}$MI|kJ*w(rhx~tc&L&xP#?cQow zX_|gx$wMr3pRZIIr_;;O|8fAjd;1`nOeu5K(pCu7>^3E&D2OBBq?sYa(%S?GwG&_0-s%_v$L@R!5H_fc)lOb9ZoOO#p`Nn`KU z3LTTBtjwo`7(HA6 z7gmO$yTR!5L>Bsg!X8616{JUngg_@&85%>W=mChTR;x4`P=?PJ~oPuy5 zU-L`C@_!34D21{fD~Y8NVnR3t;aqZI3fIhmgmx}$oc-dKDC6Ap$Gy>a!`A*x2L1v0 WcZ@i?LyX}70000 + + + + Fe + + + + + + + + diff --git a/bin/u_panel/src/server/fe/src/main.ts b/bin/u_panel/src/server/fe/src/main.ts new file mode 100644 index 0000000..c7b673c --- /dev/null +++ b/bin/u_panel/src/server/fe/src/main.ts @@ -0,0 +1,12 @@ +import { enableProdMode } from '@angular/core'; +import { platformBrowserDynamic } from '@angular/platform-browser-dynamic'; + +import { AppModule } from './app/app.module'; +import { environment } from './environments/environment'; + +if (environment.production) { + enableProdMode(); +} + +platformBrowserDynamic().bootstrapModule(AppModule) + .catch(err => console.error(err)); diff --git a/bin/u_panel/src/server/fe/src/polyfills.ts b/bin/u_panel/src/server/fe/src/polyfills.ts new file mode 100644 index 0000000..429bb9e --- /dev/null +++ b/bin/u_panel/src/server/fe/src/polyfills.ts @@ -0,0 +1,53 @@ +/** + * This file includes polyfills needed by Angular and is loaded before the app. + * You can add your own extra polyfills to this file. + * + * This file is divided into 2 sections: + * 1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers. + * 2. Application imports. Files imported after ZoneJS that should be loaded before your main + * file. + * + * The current setup is for so-called "evergreen" browsers; the last versions of browsers that + * automatically update themselves. This includes recent versions of Safari, Chrome (including + * Opera), Edge on the desktop, and iOS and Chrome on mobile. + * + * Learn more in https://angular.io/guide/browser-support + */ + +/*************************************************************************************************** + * BROWSER POLYFILLS + */ + +/** + * By default, zone.js will patch all possible macroTask and DomEvents + * user can disable parts of macroTask/DomEvents patch by setting following flags + * because those flags need to be set before `zone.js` being loaded, and webpack + * will put import in the top of bundle, so user need to create a separate file + * in this directory (for example: zone-flags.ts), and put the following flags + * into that file, and then add the following code before importing zone.js. + * import './zone-flags'; + * + * The flags allowed in zone-flags.ts are listed here. + * + * The following flags will work for all browsers. + * + * (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame + * (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick + * (window as any).__zone_symbol__UNPATCHED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames + * + * in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js + * with the following flag, it will bypass `zone.js` patch for IE/Edge + * + * (window as any).__Zone_enable_cross_context_check = true; + * + */ + +/*************************************************************************************************** + * Zone JS is required by default for Angular itself. + */ +import 'zone.js'; // Included with Angular CLI. + + +/*************************************************************************************************** + * APPLICATION IMPORTS + */ diff --git a/bin/u_panel/src/server/fe/src/styles.less b/bin/u_panel/src/server/fe/src/styles.less new file mode 100644 index 0000000..90d4ee0 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/styles.less @@ -0,0 +1 @@ +/* You can add global styles to this file, and also import other style files */ diff --git a/bin/u_panel/src/server/fe/src/test.ts b/bin/u_panel/src/server/fe/src/test.ts new file mode 100644 index 0000000..00025da --- /dev/null +++ b/bin/u_panel/src/server/fe/src/test.ts @@ -0,0 +1,26 @@ +// This file is required by karma.conf.js and loads recursively all the .spec and framework files + +import 'zone.js/testing'; +import { getTestBed } from '@angular/core/testing'; +import { + BrowserDynamicTestingModule, + platformBrowserDynamicTesting +} from '@angular/platform-browser-dynamic/testing'; + +declare const require: { + context(path: string, deep?: boolean, filter?: RegExp): { + (id: string): T; + keys(): string[]; + }; +}; + +// First, initialize the Angular testing environment. +getTestBed().initTestEnvironment( + BrowserDynamicTestingModule, + platformBrowserDynamicTesting(), +); + +// Then we find all the tests. +const context = require.context('./', true, /\.spec\.ts$/); +// And load the modules. +context.keys().map(context); diff --git a/bin/u_panel/src/server/fe/tsconfig.app.json b/bin/u_panel/src/server/fe/tsconfig.app.json new file mode 100644 index 0000000..82d91dc --- /dev/null +++ b/bin/u_panel/src/server/fe/tsconfig.app.json @@ -0,0 +1,15 @@ +/* To learn more about this file see: https://angular.io/config/tsconfig. */ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./out-tsc/app", + "types": [] + }, + "files": [ + "src/main.ts", + "src/polyfills.ts" + ], + "include": [ + "src/**/*.d.ts" + ] +} diff --git a/bin/u_panel/src/server/fe/tsconfig.json b/bin/u_panel/src/server/fe/tsconfig.json new file mode 100644 index 0000000..f531992 --- /dev/null +++ b/bin/u_panel/src/server/fe/tsconfig.json @@ -0,0 +1,32 @@ +/* To learn more about this file see: https://angular.io/config/tsconfig. */ +{ + "compileOnSave": false, + "compilerOptions": { + "baseUrl": "./", + "outDir": "./dist/out-tsc", + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "sourceMap": true, + "declaration": false, + "downlevelIteration": true, + "experimentalDecorators": true, + "moduleResolution": "node", + "importHelpers": true, + "target": "es2017", + "module": "es2020", + "lib": [ + "es2020", + "dom" + ] + }, + "angularCompilerOptions": { + "enableI18nLegacyMessageIdFormat": false, + "strictInjectionParameters": true, + "strictInputAccessModifiers": true, + "strictTemplates": true + } +} diff --git a/bin/u_panel/src/server/fe/tsconfig.spec.json b/bin/u_panel/src/server/fe/tsconfig.spec.json new file mode 100644 index 0000000..092345b --- /dev/null +++ b/bin/u_panel/src/server/fe/tsconfig.spec.json @@ -0,0 +1,18 @@ +/* To learn more about this file see: https://angular.io/config/tsconfig. */ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./out-tsc/spec", + "types": [ + "jasmine" + ] + }, + "files": [ + "src/test.ts", + "src/polyfills.ts" + ], + "include": [ + "src/**/*.spec.ts", + "src/**/*.d.ts" + ] +} diff --git a/bin/u_panel/src/server/mod.rs b/bin/u_panel/src/server/mod.rs new file mode 100644 index 0000000..58aef14 --- /dev/null +++ b/bin/u_panel/src/server/mod.rs @@ -0,0 +1,56 @@ +/* +Tabs: Agents, Tasks, Summary +every tab has list page and item page with more info/actions + +Agents: + +| id | alias | ..see struct | tasks done +| stripped | alias | ... | clickable number of assigned jobs + +almost all fields are editable, rows are deletable + +*/ + +use actix_web::{get, web, App, HttpResponse, HttpServer, Responder}; +use rust_embed::RustEmbed; +use std::borrow::Cow; +use u_lib::unwrap_enum; + +#[derive(RustEmbed)] +#[folder = "./src/server/fe/dist/fe/"] +struct Files; + +impl Files { + pub fn get_static(path: impl AsRef) -> Option<&'static [u8]> { + let file = Self::get(path.as_ref())?.data; + Some(unwrap_enum!(file, Cow::Borrowed)) + } +} + +#[get("/")] +async fn main_page() -> impl Responder { + let index = Files::get_static("index.html").unwrap(); + HttpResponse::Ok().body(index) +} + +#[get("/{file}")] +async fn static_files_adapter(file: web::Path<(String,)>) -> impl Responder { + let file = file.into_inner().0; + let mimetype = mime_guess::from_path(&file).first_or_octet_stream(); + match Files::get_static(file) { + Some(data) => HttpResponse::Ok() + .content_type(mimetype.to_string()) + .body(data), + None => HttpResponse::NotFound().finish(), + } +} + +#[actix_web::main] +pub async fn serve() -> std::io::Result<()> { + let addr = "127.0.0.1:8080"; + println!("Serving at http://{}", addr); + HttpServer::new(|| App::new().service(main_page).service(static_files_adapter)) + .bind(addr)? + .run() + .await +} diff --git a/bin/u_server/Cargo.toml b/bin/u_server/Cargo.toml index 043022a..d848850 100644 --- a/bin/u_server/Cargo.toml +++ b/bin/u_server/Cargo.toml @@ -41,4 +41,4 @@ path = "src/u_server.rs" [[bin]] name = "u_server" -path = "src/main.rs" \ No newline at end of file +path = "src/main.rs" diff --git a/lib/u_lib/src/errors/variants.rs b/lib/u_lib/src/errors/variants.rs index 48d2fbb..103afe1 100644 --- a/lib/u_lib/src/errors/variants.rs +++ b/lib/u_lib/src/errors/variants.rs @@ -66,8 +66,8 @@ pub enum UError { #[error("Panicked: {0}")] Panic(String), - #[error("UI error: {0}")] - TUIError(String), + #[error("Panel error: {0}")] + PanelError(String), } impl UError { From 5d04aa61d6edffefb235a5a0d4a2ca3838b96b40 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sat, 30 Apr 2022 23:17:35 +0500 Subject: [PATCH 18/30] improve env parsing & optimize integration tests --- bin/u_agent/src/lib.rs | 17 ++---- bin/u_panel/src/argparse.rs | 36 ++++++++---- bin/u_panel/src/main.rs | 25 +++++---- bin/u_panel/src/server/mod.rs | 10 ++-- bin/u_server/src/db.rs | 23 ++++---- bin/u_server/src/{filters.rs => init.rs} | 42 +++++++++++--- bin/u_server/src/u_server.rs | 56 +++++-------------- .../integration-tests/tests_base.Dockerfile | 11 ++++ .../integration-tests/tests_runner.Dockerfile | 12 +++- integration/Cargo.toml | 2 +- integration/docker-compose.yml | 1 - integration/docker.py | 38 +++++++------ integration/integration_tests.sh | 2 + integration/tests/helpers/panel.rs | 2 +- .../tests/{ => integration}/behaviour.rs | 2 +- integration/tests/integration/mod.rs | 1 + integration/tests/{tests.rs => lib.rs} | 8 +-- lib/u_lib/Cargo.toml | 1 + lib/u_lib/src/api.rs | 13 ++--- lib/u_lib/src/config.rs | 1 - lib/u_lib/src/utils/env.rs | 36 ++++++++++++ lib/u_lib/src/utils/misc.rs | 7 --- lib/u_lib/src/utils/mod.rs | 2 + 23 files changed, 205 insertions(+), 143 deletions(-) rename bin/u_server/src/{filters.rs => init.rs} (69%) create mode 100644 images/integration-tests/tests_base.Dockerfile rename integration/tests/{ => integration}/behaviour.rs (94%) create mode 100644 integration/tests/integration/mod.rs rename integration/tests/{tests.rs => lib.rs} (77%) create mode 100644 lib/u_lib/src/utils/env.rs diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 3f422b3..9403868 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -7,21 +7,12 @@ extern crate log; extern crate env_logger; -use std::env; use std::panic; use std::sync::Arc; use tokio::time::{sleep, Duration}; use u_lib::{ - api::ClientHandler, - builder::JobBuilder, - cache::JobCache, - errors::ErrChan, - executor::pop_completed, - messaging::Reportable, - models::AssignedJob, - UError, - UID, - //daemonize + api::ClientHandler, builder::JobBuilder, cache::JobCache, errors::ErrChan, + executor::pop_completed, messaging::Reportable, models::AssignedJob, utils::Env, UError, UID, }; const ITERATION_LATENCY: u64 = 5; @@ -98,8 +89,8 @@ async fn do_stuff(client: Arc) -> ! { pub async fn run_forever() { //daemonize(); env_logger::init(); - let arg_ip = env::args().nth(1); - let client = Arc::new(ClientHandler::new(arg_ip.as_deref())); + let env = Env::init_default().unwrap(); + let client = Arc::new(ClientHandler::new(&env.u_server)); panic::set_hook(Box::new(|panic_info| { ErrChan::send(UError::Panic(panic_info.to_string())) })); diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 879ccbb..c8f9b38 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -17,7 +17,7 @@ enum Cmd { Agents(LD), Jobs(JobALD), Map(JobMapALD), - TUI(TUIArgs), + //TUI(TUIArgs), Serve, } @@ -107,8 +107,8 @@ pub async fn process_cmd(args: Args) -> UResult<()> { let printer = Printer { json: args.json }; match args.cmd { Cmd::Agents(action) => match action { - LD::List { uid } => printer.print(CLIENT.get_agents(uid).await), - LD::Delete { uid } => printer.print(CLIENT.del(Some(uid)).await), + LD::List { uid } => printer.print(CLIENT.get().unwrap().get_agents(uid).await), + LD::Delete { uid } => printer.print(CLIENT.get().unwrap().del(Some(uid)).await), }, Cmd::Jobs(action) => match action { JobALD::Add { @@ -120,22 +120,34 @@ pub async fn process_cmd(args: Args) -> UResult<()> { .with_shell(cmd.join(" ")) .with_alias(alias) .build()?; - printer.print(CLIENT.upload_jobs(&[job]).await); + printer.print(CLIENT.get().unwrap().upload_jobs(&[job]).await); + } + JobALD::LD(LD::List { uid }) => { + printer.print(CLIENT.get().unwrap().get_jobs(uid).await) + } + JobALD::LD(LD::Delete { uid }) => { + printer.print(CLIENT.get().unwrap().del(Some(uid)).await) } - JobALD::LD(LD::List { uid }) => printer.print(CLIENT.get_jobs(uid).await), - JobALD::LD(LD::Delete { uid }) => printer.print(CLIENT.del(Some(uid)).await), }, Cmd::Map(action) => match action { JobMapALD::Add { agent_uid, job_idents, - } => printer.print(CLIENT.set_jobs(Some(agent_uid), &job_idents).await), - JobMapALD::List { uid } => printer.print(CLIENT.get_agent_jobs(uid).await), - JobMapALD::Delete { uid } => printer.print(CLIENT.del(Some(uid)).await), + } => printer.print( + CLIENT + .get() + .unwrap() + .set_jobs(Some(agent_uid), &job_idents) + .await, + ), + JobMapALD::List { uid } => { + printer.print(CLIENT.get().unwrap().get_agent_jobs(uid).await) + } + JobMapALD::Delete { uid } => printer.print(CLIENT.get().unwrap().del(Some(uid)).await), }, - Cmd::TUI(args) => crate::tui::init_tui(&args) - .await - .map_err(|e| UError::PanelError(e.to_string()))?, + /*Cmd::TUI(args) => crate::tui::init_tui(&args) + .await + .map_err(|e| UError::PanelError(e.to_string()))?,*/ Cmd::Serve => crate::server::serve().map_err(|e| UError::PanelError(e.to_string()))?, } Ok(()) diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index d4ba19a..3296915 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -1,6 +1,6 @@ mod argparse; -mod tui; mod server; +//mod tui; #[macro_use] extern crate async_trait; @@ -9,21 +9,26 @@ extern crate async_trait; extern crate tracing; use argparse::{process_cmd, Args}; -use once_cell::sync::Lazy; -use std::env; +use once_cell::sync::OnceCell; +use serde::Deserialize; use std::process; use structopt::StructOpt; use u_lib::api::ClientHandler; -use u_lib::utils::init_env; +use u_lib::utils::Env; -pub static CLIENT: Lazy = Lazy::new(|| { - let token = env::var("ADMIN_AUTH_TOKEN").expect("access token is not set"); - ClientHandler::new(None).password(token.clone()) -}); +pub static CLIENT: OnceCell = OnceCell::new(); -#[tokio::main(flavor = "multi_thread")] +#[derive(Deserialize)] +struct AccessEnv { + admin_auth_token: String, +} + +#[tokio::main] async fn main() { - init_env(); + let env = Env::::init().unwrap(); + + CLIENT.get_or_init(|| ClientHandler::new(&env.u_server).password(env.inner.admin_auth_token)); + let args: Args = Args::from_args(); if let Err(e) = process_cmd(args).await { eprintln!("Error: {}", e); diff --git a/bin/u_panel/src/server/mod.rs b/bin/u_panel/src/server/mod.rs index 58aef14..aaebf5a 100644 --- a/bin/u_panel/src/server/mod.rs +++ b/bin/u_panel/src/server/mod.rs @@ -33,11 +33,11 @@ async fn main_page() -> impl Responder { HttpResponse::Ok().body(index) } -#[get("/{file}")] -async fn static_files_adapter(file: web::Path<(String,)>) -> impl Responder { - let file = file.into_inner().0; - let mimetype = mime_guess::from_path(&file).first_or_octet_stream(); - match Files::get_static(file) { +#[get("/{path}")] +async fn static_files_adapter(path: web::Path<(String,)>) -> impl Responder { + let path = path.into_inner().0; + let mimetype = mime_guess::from_path(&path).first_or_octet_stream(); + match Files::get_static(path) { Some(data) => HttpResponse::Ok() .content_type(mimetype.to_string()) .body(data), diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index c2a2af1..a5b3796 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -1,11 +1,10 @@ use diesel::{pg::PgConnection, prelude::*, result::Error as DslError}; use once_cell::sync::OnceCell; -use std::{ - env, - sync::{Arc, Mutex, MutexGuard}, -}; +use serde::Deserialize; +use std::sync::{Arc, Mutex, MutexGuard}; use u_lib::{ models::{schema, Agent, AgentError, AssignedJob, JobMeta, JobState}, + utils::Env, ULocalError, ULocalResult, }; use uuid::Uuid; @@ -16,18 +15,22 @@ pub struct UDB { static DB: OnceCell>> = OnceCell::new(); +#[derive(Deserialize)] +struct DBEnv { + db_host: String, + db_name: String, + db_user: String, + db_password: String, +} + #[cfg_attr(test, automock)] impl UDB { pub fn lock_db() -> MutexGuard<'static, UDB> { DB.get_or_init(|| { - let _getenv = |v| env::var(v).unwrap(); - let db_host = _getenv("DB_HOST"); - let db_name = _getenv("DB_NAME"); - let db_user = _getenv("DB_USER"); - let db_password = _getenv("DB_PASSWORD"); + let env = Env::::init().unwrap(); let db_url = format!( "postgres://{}:{}@{}/{}", - db_user, db_password, db_host, db_name + env.inner.db_user, env.inner.db_password, env.inner.db_host, env.inner.db_name ); let conn = PgConnection::establish(&db_url).unwrap(); let instance = UDB { conn }; diff --git a/bin/u_server/src/filters.rs b/bin/u_server/src/init.rs similarity index 69% rename from bin/u_server/src/filters.rs rename to bin/u_server/src/init.rs index 44b40b8..60b3396 100644 --- a/bin/u_server/src/filters.rs +++ b/bin/u_server/src/init.rs @@ -1,6 +1,7 @@ +use crate::db::UDB; use crate::handlers::Endpoints; use serde::de::DeserializeOwned; -use std::env; +use std::path::PathBuf; use u_lib::{ messaging::{AsMsg, BaseMessage, Reportable}, models::*, @@ -15,7 +16,9 @@ where body::content_length_limit(1024 * 64).and(body::json::>()) } -pub fn make_filters() -> impl Filter + Clone { +pub fn init_filters( + auth_token: &str, +) -> impl Filter + Clone { let infallible_none = |_| async { Ok::<(Option,), std::convert::Infallible>((None,)) }; let get_agents = warp::get() @@ -70,11 +73,7 @@ pub fn make_filters() -> impl Filter .and(warp::path("report")) .and(get_content::>().and_then(Endpoints::report)); - let auth_token = format!( - "Bearer {}", - env::var("ADMIN_AUTH_TOKEN").expect("No auth token provided") - ) - .into_boxed_str(); + let auth_token = format!("Bearer {auth_token}",).into_boxed_str(); let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); let auth_zone = (get_agents @@ -89,3 +88,32 @@ pub fn make_filters() -> impl Filter auth_zone.or(agent_zone) } + +pub fn prefill_jobs() { + let agent_hello = JobMeta::builder() + .with_type(misc::JobType::Manage) + .with_alias("agent_hello") + .build() + .unwrap(); + UDB::lock_db().insert_jobs(&[agent_hello]).unwrap(); +} + +pub fn init_logger() { + use simplelog::*; + use std::fs::OpenOptions; + let log_cfg = ConfigBuilder::new() + .set_time_format_str("%x %X") + .set_time_to_local(true) + .build(); + let logfile = OpenOptions::new() + .append(true) + .create(true) + .open(PathBuf::from("logs").join("u_server.log")) + .unwrap(); + let level = LevelFilter::Info; + let loggers = vec![ + WriteLogger::new(level, log_cfg.clone(), logfile) as Box, + TermLogger::new(level, log_cfg, TerminalMode::Stderr, ColorChoice::Auto), + ]; + CombinedLogger::init(loggers).unwrap(); +} diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index adfeb1c..d9bbd03 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -13,56 +13,27 @@ extern crate diesel; // in this block mod db; -mod filters; mod handlers; +mod init; -use db::UDB; -use filters::make_filters; +use init::*; +use serde::Deserialize; use std::path::PathBuf; -use u_lib::{config::MASTER_PORT, models::*, utils::init_env}; +use u_lib::{config::MASTER_PORT, utils::Env}; use warp::Filter; -const LOGFILE: &str = "u_server.log"; - -fn prefill_jobs() { - let agent_hello = JobMeta::builder() - .with_type(misc::JobType::Manage) - .with_alias("agent_hello") - .build() - .unwrap(); - UDB::lock_db().insert_jobs(&[agent_hello]).unwrap(); -} - -fn init_logger() { - use simplelog::*; - use std::fs::OpenOptions; - let log_cfg = ConfigBuilder::new() - .set_time_format_str("%x %X") - .set_time_to_local(true) - .build(); - let logfile = OpenOptions::new() - .append(true) - .create(true) - .open(PathBuf::from("logs").join(LOGFILE)) - .unwrap(); - let level = LevelFilter::Info; - let loggers = vec![ - WriteLogger::new(level, log_cfg.clone(), logfile) as Box, - TermLogger::new(level, log_cfg, TerminalMode::Stderr, ColorChoice::Auto), - ]; - CombinedLogger::init(loggers).unwrap(); +#[derive(Deserialize)] +struct ServEnv { + admin_auth_token: String, } -fn init_all() { +//TODO: tracing-subscriber +pub async fn serve() -> Result<(), String> { init_logger(); - init_env(); prefill_jobs(); -} -//TODO: tracing-subscriber -pub async fn serve() { - init_all(); - let routes = make_filters(); + let env = Env::::init().map_err(|e| e.to_string())?; + let routes = init_filters(&env.inner.admin_auth_token); let certs_dir = PathBuf::from("certs"); warp::serve(routes.with(warp::log("warp"))) .tls() @@ -71,6 +42,7 @@ pub async fn serve() { .client_auth_required_path(certs_dir.join("ca.crt")) .run(([0, 0, 0, 0], MASTER_PORT)) .await; + Ok(()) } #[cfg(test)] @@ -97,7 +69,7 @@ mod tests { uid.map(|u| u.simple().to_string()).unwrap_or(String::new()) )) .method("GET") - .filter(&make_filters()) + .filter(&init_filters("")) .await .unwrap(); mock.checkpoint(); @@ -113,7 +85,7 @@ mod tests { .path("/report/") .method("POST") .json(&vec![Reportable::Dummy].as_message()) - .filter(&make_filters()) + .filter(&init_filters("")) .await .unwrap(); mock.checkpoint(); diff --git a/images/integration-tests/tests_base.Dockerfile b/images/integration-tests/tests_base.Dockerfile new file mode 100644 index 0000000..81fee07 --- /dev/null +++ b/images/integration-tests/tests_base.Dockerfile @@ -0,0 +1,11 @@ +# build from lib/ +FROM rust:1.60 as chef + +RUN rustup target add x86_64-unknown-linux-musl +RUN cargo install cargo-chef + +COPY u_lib /lib/u_lib +COPY u_api_proc_macro /lib/u_api_proc_macro +COPY certs /certs + +WORKDIR /app \ No newline at end of file diff --git a/images/integration-tests/tests_runner.Dockerfile b/images/integration-tests/tests_runner.Dockerfile index 4bf8287..ae60226 100644 --- a/images/integration-tests/tests_runner.Dockerfile +++ b/images/integration-tests/tests_runner.Dockerfile @@ -1,4 +1,12 @@ -FROM rust:1.55 +# build from integration/ +FROM unki/tests_base as chef + +FROM chef as planner +COPY . /app +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef as builder +COPY --from=planner /app/recipe.json recipe.json +RUN cargo chef cook --release --recipe-path recipe.json -RUN rustup target add x86_64-unknown-linux-musl CMD ["sleep", "3600"] \ No newline at end of file diff --git a/integration/Cargo.toml b/integration/Cargo.toml index 5bffbf6..14fb0f6 100644 --- a/integration/Cargo.toml +++ b/integration/Cargo.toml @@ -25,4 +25,4 @@ version = "*" [[test]] name = "integration" -path = "tests/tests.rs" +path = "tests/lib.rs" \ No newline at end of file diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml index 003460d..9d2e218 100644 --- a/integration/docker-compose.yml +++ b/integration/docker-compose.yml @@ -71,7 +71,6 @@ services: networks: - u_net volumes: - - ~/.cargo/registry:/root/.cargo/registry - ./:/tests/ - ../certs:/certs - ../release/u_panel:/u_panel diff --git a/integration/docker.py b/integration/docker.py index 0e1c163..2d47517 100644 --- a/integration/docker.py +++ b/integration/docker.py @@ -3,24 +3,29 @@ from utils import * BASE_IMAGE_DIR = '../images/integration-tests' -DOCKERFILES = { - 'u_agent': { +# do not reorder +DOCKERFILES = [ + { + 'name': 'u_agent', 'ctx': BASE_IMAGE_DIR, - 'dockerfile_prefix': 'u_agent' }, - 'u_server': { + { + 'name': 'u_server', 'ctx': BASE_IMAGE_DIR, - 'dockerfile_prefix': 'u_server' }, - 'u_db': { + { + 'name': 'u_db', 'ctx': BASE_IMAGE_DIR, - 'dockerfile_prefix': 'u_db' }, - 'tests_runner': { - 'ctx': BASE_IMAGE_DIR, - 'dockerfile_prefix': 'tests_runner' + { + 'name': 'tests_base', + 'ctx': '../lib', + }, + { + 'name': 'tests_runner', + 'ctx': '../integration', }, -} +] def docker(args): @@ -62,11 +67,11 @@ def check_state(containers): def rebuild_images_if_needed(force_rebuild=False): - for img_name, data in DOCKERFILES.items(): - ctx = data['ctx'] - df_prefix = data.get('dockerfile_prefix') + for img in DOCKERFILES: + ctx = img['ctx'] + name = img.get('name') df_suffix = 'Dockerfile' - img_name = f'unki/{img_name}' + img_name = f'unki/{name}' log(f'Building docker image {img_name}') cmd = [ 'build', @@ -74,8 +79,7 @@ def rebuild_images_if_needed(force_rebuild=False): img_name, ctx, ] - if df_prefix: - cmd += ['-f', f'{ctx}/{df_prefix}.{df_suffix}'] + cmd += ['-f', f'{BASE_IMAGE_DIR}/{name}.{df_suffix}'] if force_rebuild: cmd += ['--no-cache'] docker(cmd) diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 3f3eca1..374299b 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -1,3 +1,5 @@ #!/bin/bash set -e +cp -r ../certs ../lib/certs python integration_tests.py $@ +rm -rf ../lib/certs diff --git a/integration/tests/helpers/panel.rs b/integration/tests/helpers/panel.rs index 9406d90..2ddf4f0 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration/tests/helpers/panel.rs @@ -36,7 +36,7 @@ impl Panel { } pub fn output(args: impl Into + Display) -> PanelResult { - println!("Executing 'u_panel {}'", &args); + println!("Executing '{PANEL_BINARY} {}'", &args); let splitted = split(args.into().as_ref()).unwrap(); Self::output_argv( splitted diff --git a/integration/tests/behaviour.rs b/integration/tests/integration/behaviour.rs similarity index 94% rename from integration/tests/behaviour.rs rename to integration/tests/integration/behaviour.rs index fcc3c63..6ff1bf3 100644 --- a/integration/tests/behaviour.rs +++ b/integration/tests/integration/behaviour.rs @@ -27,7 +27,7 @@ async fn test_setup_tasks() -> TestResult { let agents: Vec = Panel::check_output("agents list"); let agent_uid = agents[0].id; let job_alias = "passwd_contents"; - let cmd = format!("jobs add --alias {} 'cat /etc/passwd'", job_alias); + let cmd = format!("jobs add --alias {job_alias} 'cat /etc/passwd'"); Panel::check_status(cmd); let cmd = format!("map add {} {}", agent_uid, job_alias); let assigned_uids: Vec = Panel::check_output(cmd); diff --git a/integration/tests/integration/mod.rs b/integration/tests/integration/mod.rs new file mode 100644 index 0000000..b3a413f --- /dev/null +++ b/integration/tests/integration/mod.rs @@ -0,0 +1 @@ +mod behaviour; diff --git a/integration/tests/tests.rs b/integration/tests/lib.rs similarity index 77% rename from integration/tests/tests.rs rename to integration/tests/lib.rs index 39293f6..70019bb 100644 --- a/integration/tests/tests.rs +++ b/integration/tests/lib.rs @@ -1,22 +1,22 @@ -mod behaviour; mod fixtures; mod helpers; +mod integration; -use std::env; use u_lib::config::MASTER_PORT; +use u_lib::utils::Env; #[macro_use] extern crate rstest; #[tokio::test] async fn test_non_auth_connection_dropped() { - let env_server = env::var("U_SERVER").unwrap(); + let env = Env::init_default().unwrap(); let client = reqwest::ClientBuilder::new() .danger_accept_invalid_certs(true) .build() .unwrap(); match client - .get(format!("https://{}:{}", env_server, MASTER_PORT)) + .get(format!("https://{}:{}", &env.u_server, MASTER_PORT)) .send() .await { diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index a2afaf6..adfc2c4 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -27,6 +27,7 @@ u_api_proc_macro = { version = "*", path = "../u_api_proc_macro" } crossbeam = "0.8.1" backtrace = "0.3.61" diesel = { version = "1.4.5", features = ["postgres", "uuid"] } +envy = "0.4.2" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] reqwest = { version = "0.11", features = ["json", "native-tls"] } diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 7dbb774..35af4d3 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -1,14 +1,11 @@ -use crate::messaging; -//#[allow(non_upper_case_globals)] use crate::{ - config::{MASTER_PORT, MASTER_SERVER}, - messaging::{AsMsg, BaseMessage}, + config::MASTER_PORT, + messaging::{self, AsMsg, BaseMessage}, models, utils::{opt_to_string, VecDisplay}, UError, UResult, }; use reqwest::{Certificate, Client, Identity, RequestBuilder, Url}; -use std::env; use u_api_proc_macro::api_route; use uuid::Uuid; @@ -22,9 +19,7 @@ pub struct ClientHandler { } impl ClientHandler { - pub fn new(server: Option<&str>) -> Self { - let env_server = env::var("U_SERVER").unwrap_or(String::from(MASTER_SERVER)); - let master_server = server.unwrap_or(env_server.as_str()); + pub fn new(server: &str) -> Self { let identity = Identity::from_pkcs12_der(AGENT_IDENTITY, "").unwrap(); let client = Client::builder() .identity(identity) @@ -33,7 +28,7 @@ impl ClientHandler { .unwrap(); Self { client, - base_url: Url::parse(&format!("https://{}:{}", master_server, MASTER_PORT)).unwrap(), + base_url: Url::parse(&format!("https://{}:{}", server, MASTER_PORT)).unwrap(), password: None, } } diff --git a/lib/u_lib/src/config.rs b/lib/u_lib/src/config.rs index ed9052d..e9d649a 100644 --- a/lib/u_lib/src/config.rs +++ b/lib/u_lib/src/config.rs @@ -1,7 +1,6 @@ use lazy_static::lazy_static; use uuid::Uuid; -pub const MASTER_SERVER: &str = "ortem.xyz"; //Ipv4Addr::new(3,9,16,40) pub const MASTER_PORT: u16 = 63714; lazy_static! { diff --git a/lib/u_lib/src/utils/env.rs b/lib/u_lib/src/utils/env.rs new file mode 100644 index 0000000..a5a7352 --- /dev/null +++ b/lib/u_lib/src/utils/env.rs @@ -0,0 +1,36 @@ +use envy::{from_env, Result as EnvResult}; +use serde::{de::DeserializeOwned, Deserialize}; + +#[derive(Deserialize)] +pub struct NoneEnv; + +#[derive(Deserialize)] +pub struct Env { + #[serde(default = "default_host")] + pub u_server: String, + pub inner: E, +} + +impl Env { + pub fn init_default() -> EnvResult { + let envs = [".env", ".env.private"]; + for envfile in &envs { + dotenv::from_filename(envfile).ok(); + } + from_env() + } +} + +impl Env { + pub fn init() -> EnvResult { + let envs = [".env", ".env.private"]; + for envfile in &envs { + dotenv::from_filename(envfile).ok(); + } + from_env() + } +} + +fn default_host() -> String { + "ortem.xyz".to_string() +} diff --git a/lib/u_lib/src/utils/misc.rs b/lib/u_lib/src/utils/misc.rs index 8981bd3..7fddf4c 100644 --- a/lib/u_lib/src/utils/misc.rs +++ b/lib/u_lib/src/utils/misc.rs @@ -24,10 +24,3 @@ macro_rules! unwrap_enum { } }; } - -pub fn init_env() { - let envs = [".env", ".env.private"]; - for envfile in &envs { - dotenv::from_filename(envfile).ok(); - } -} diff --git a/lib/u_lib/src/utils/mod.rs b/lib/u_lib/src/utils/mod.rs index 139fd5a..3cddc6b 100644 --- a/lib/u_lib/src/utils/mod.rs +++ b/lib/u_lib/src/utils/mod.rs @@ -1,5 +1,6 @@ mod combined_result; mod conv; +mod env; mod fmt; mod misc; mod proc_output; @@ -12,6 +13,7 @@ mod vec_display; pub use combined_result::*; pub use conv::*; +pub use env::Env; pub use fmt::*; pub use misc::*; pub use proc_output::*; From c60890fd679dce494094beb5e56bbe2addb81851 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sun, 29 May 2022 20:28:32 +0500 Subject: [PATCH 19/30] revert docker image caching, improved env gathering --- Makefile.toml | 11 +- bin/u_agent/src/lib.rs | 9 +- bin/u_panel/Cargo.toml | 1 - bin/u_panel/src/argparse.rs | 35 ++---- bin/u_panel/src/main.rs | 27 ++--- bin/u_server/Cargo.toml | 23 +--- bin/u_server/src/db.rs | 9 +- bin/u_server/src/handlers.rs | 1 - bin/u_server/src/main.rs | 4 +- bin/u_server/src/u_server.rs | 40 +++---- .../integration-tests/tests_base.Dockerfile | 11 -- .../integration-tests/tests_runner.Dockerfile | 12 +- integration/Cargo.toml | 5 +- integration/docker-compose.yml | 1 + integration/docker.py | 13 +-- integration/integration_tests.sh | 4 +- integration/tests/fixtures/agent.rs | 5 +- integration/tests/helpers/mod.rs | 5 + integration/tests/lib.rs | 5 +- lib/u_lib/Cargo.toml | 4 +- lib/u_lib/src/builder.rs | 108 ++++++++---------- lib/u_lib/src/lib.rs | 3 +- lib/u_lib/src/utils/env.rs | 35 +++--- lib/u_lib/src/utils/mod.rs | 22 ++-- lib/u_lib/src/utils/proc_output.rs | 22 ++-- 25 files changed, 173 insertions(+), 242 deletions(-) delete mode 100644 images/integration-tests/tests_base.Dockerfile diff --git a/Makefile.toml b/Makefile.toml index 4c11b71..32533ec 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -30,11 +30,14 @@ args = ["build", "--target", "${TARGET}", "${@}"] [tasks.release_tasks] script = ''' -if [[ "${@}" =~ "release" ]]; then - echo "Stripping binaries..." - strip $(ls ./target/${TARGET}/release/u_* -1 | grep -v ".d") +if [[ "${@}" =~ "--release" ]]; then echo "Creating symlink to release dir..." ln -s ./target/${TARGET}/release ./release || true + BINS=$(ls ./release/u_* -1 | grep -v ".d") + echo "Stripping..." + strip $BINS + echo "Packing..." + upx -9 $BINS fi ''' @@ -56,7 +59,7 @@ args = ["test", "--target", "${TARGET}", "--lib", "--", "${@}"] [tasks.integration] script = ''' cd ./integration -bash integration_tests.sh +bash integration_tests.sh ${@} ''' [tasks.gen_schema] diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 9403868..581fd7f 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -12,7 +12,8 @@ use std::sync::Arc; use tokio::time::{sleep, Duration}; use u_lib::{ api::ClientHandler, builder::JobBuilder, cache::JobCache, errors::ErrChan, - executor::pop_completed, messaging::Reportable, models::AssignedJob, utils::Env, UError, UID, + executor::pop_completed, messaging::Reportable, models::AssignedJob, utils::load_env_default, + UError, UID, }; const ITERATION_LATENCY: u64 = 5; @@ -86,14 +87,14 @@ async fn do_stuff(client: Arc) -> ! { } } -pub async fn run_forever() { +pub async fn run_forever() -> ! { //daemonize(); env_logger::init(); - let env = Env::init_default().unwrap(); + let env = load_env_default().unwrap(); let client = Arc::new(ClientHandler::new(&env.u_server)); panic::set_hook(Box::new(|panic_info| { ErrChan::send(UError::Panic(panic_info.to_string())) })); tokio::spawn(error_reporting(client.clone())); - do_stuff(client).await; + do_stuff(client).await } diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 2bccf84..6facac5 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -21,7 +21,6 @@ tui = { version = "0.16", default-features = false, features = ['crossterm'] } crossterm = "0.22.1" anyhow = "1.0.44" strum = { version = "0.22.0", features = ["derive"] } -async-trait = "0.1.51" once_cell = "1.8.0" crossbeam = "0.8.1" async-channel = "1.6.1" diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index c8f9b38..cfe9050 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -1,7 +1,8 @@ -use crate::CLIENT; use std::fmt; use structopt::StructOpt; -use u_lib::{datatypes::DataResult, messaging::AsMsg, models::JobMeta, UError, UResult}; +use u_lib::{ + api::ClientHandler, datatypes::DataResult, messaging::AsMsg, models::JobMeta, UError, UResult, +}; use uuid::Uuid; #[derive(StructOpt, Debug)] @@ -82,7 +83,7 @@ fn parse_uuid(src: &str) -> Result { Uuid::parse_str(src).map_err(|e| e.to_string()) } -pub async fn process_cmd(args: Args) -> UResult<()> { +pub async fn process_cmd(client: ClientHandler, args: Args) -> UResult<()> { struct Printer { json: bool, } @@ -107,8 +108,8 @@ pub async fn process_cmd(args: Args) -> UResult<()> { let printer = Printer { json: args.json }; match args.cmd { Cmd::Agents(action) => match action { - LD::List { uid } => printer.print(CLIENT.get().unwrap().get_agents(uid).await), - LD::Delete { uid } => printer.print(CLIENT.get().unwrap().del(Some(uid)).await), + LD::List { uid } => printer.print(client.get_agents(uid).await), + LD::Delete { uid } => printer.print(client.del(Some(uid)).await), }, Cmd::Jobs(action) => match action { JobALD::Add { @@ -120,30 +121,18 @@ pub async fn process_cmd(args: Args) -> UResult<()> { .with_shell(cmd.join(" ")) .with_alias(alias) .build()?; - printer.print(CLIENT.get().unwrap().upload_jobs(&[job]).await); - } - JobALD::LD(LD::List { uid }) => { - printer.print(CLIENT.get().unwrap().get_jobs(uid).await) - } - JobALD::LD(LD::Delete { uid }) => { - printer.print(CLIENT.get().unwrap().del(Some(uid)).await) + printer.print(client.upload_jobs(&[job]).await); } + JobALD::LD(LD::List { uid }) => printer.print(client.get_jobs(uid).await), + JobALD::LD(LD::Delete { uid }) => printer.print(client.del(Some(uid)).await), }, Cmd::Map(action) => match action { JobMapALD::Add { agent_uid, job_idents, - } => printer.print( - CLIENT - .get() - .unwrap() - .set_jobs(Some(agent_uid), &job_idents) - .await, - ), - JobMapALD::List { uid } => { - printer.print(CLIENT.get().unwrap().get_agent_jobs(uid).await) - } - JobMapALD::Delete { uid } => printer.print(CLIENT.get().unwrap().del(Some(uid)).await), + } => printer.print(client.set_jobs(Some(agent_uid), &job_idents).await), + JobMapALD::List { uid } => printer.print(client.get_agent_jobs(uid).await), + JobMapALD::Delete { uid } => printer.print(client.del(Some(uid)).await), }, /*Cmd::TUI(args) => crate::tui::init_tui(&args) .await diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index 3296915..2f0d685 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -2,36 +2,29 @@ mod argparse; mod server; //mod tui; -#[macro_use] -extern crate async_trait; - #[macro_use] extern crate tracing; +use anyhow::Result as AnyResult; use argparse::{process_cmd, Args}; -use once_cell::sync::OnceCell; use serde::Deserialize; -use std::process; use structopt::StructOpt; use u_lib::api::ClientHandler; -use u_lib::utils::Env; - -pub static CLIENT: OnceCell = OnceCell::new(); +use u_lib::utils::{env::default_host, load_env}; #[derive(Deserialize)] struct AccessEnv { admin_auth_token: String, + #[serde(default = "default_host")] + u_server: String, } #[tokio::main] -async fn main() { - let env = Env::::init().unwrap(); - - CLIENT.get_or_init(|| ClientHandler::new(&env.u_server).password(env.inner.admin_auth_token)); +async fn main() -> AnyResult<()> { + let env = load_env::()?; + let client = ClientHandler::new(&env.u_server).password(env.admin_auth_token); + let args = Args::from_args(); - let args: Args = Args::from_args(); - if let Err(e) = process_cmd(args).await { - eprintln!("Error: {}", e); - process::exit(1) - } + process_cmd(client, args).await?; + Ok(()) } diff --git a/bin/u_server/Cargo.toml b/bin/u_server/Cargo.toml index d848850..13d12ca 100644 --- a/bin/u_server/Cargo.toml +++ b/bin/u_server/Cargo.toml @@ -12,28 +12,15 @@ warp = { version = "0.3.1", features = ["tls"] } uuid = { version = "0.6.5", features = ["serde", "v4"] } once_cell = "1.7.2" hyper = "0.14" -mockall = "0.9.1" -mockall_double = "0.2" openssl = "*" +diesel = { version = "1.4.5", features = ["postgres", "uuid"] } +serde = { version = "1.0", features = ["derive"] } +tokio = { version = "1.9", features = ["macros"] } +u_lib = { path = "../../lib/u_lib", version = "*" } -[dependencies.diesel] -features = ["postgres", "uuid"] -version = "1.4.5" - -[dependencies.serde] -features = ["derive"] -version = "1.0.114" - -[dependencies.tokio] -features = ["macros"] -version = "1.9" - -[dependencies.u_lib] -path = "../../lib/u_lib" -version = "*" [dev-dependencies] -test-case = "1.1.0" +rstest = "0.12" [lib] name = "u_server_lib" diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index a5b3796..cff590f 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -4,7 +4,7 @@ use serde::Deserialize; use std::sync::{Arc, Mutex, MutexGuard}; use u_lib::{ models::{schema, Agent, AgentError, AssignedJob, JobMeta, JobState}, - utils::Env, + utils::load_env, ULocalError, ULocalResult, }; use uuid::Uuid; @@ -23,14 +23,13 @@ struct DBEnv { db_password: String, } -#[cfg_attr(test, automock)] impl UDB { - pub fn lock_db() -> MutexGuard<'static, UDB> { + pub fn lock_db() -> MutexGuard<'static, Self> { DB.get_or_init(|| { - let env = Env::::init().unwrap(); + let env = load_env::().unwrap(); let db_url = format!( "postgres://{}:{}@{}/{}", - env.inner.db_user, env.inner.db_password, env.inner.db_host, env.inner.db_name + env.db_user, env.db_password, env.db_host, env.db_name ); let conn = PgConnection::establish(&db_url).unwrap(); let instance = UDB { conn }; diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index 3158bcf..9a23f17 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -32,7 +32,6 @@ pub fn build_message(m: M) -> Response { pub struct Endpoints; -#[cfg_attr(test, automock)] impl Endpoints { pub async fn add_agent(msg: Agent) -> Result, Rejection> { info!("hnd: add_agent"); diff --git a/bin/u_server/src/main.rs b/bin/u_server/src/main.rs index 05ec4e0..e402a55 100644 --- a/bin/u_server/src/main.rs +++ b/bin/u_server/src/main.rs @@ -1,6 +1,6 @@ use u_server_lib::serve; #[tokio::main] -async fn main() { - serve().await; +async fn main() -> Result<(), String> { + serve().await } diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index d9bbd03..bf9d8ed 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -1,10 +1,9 @@ #[macro_use] extern crate log; -#[cfg_attr(test, macro_use)] -extern crate mockall; -#[cfg_attr(test, macro_use)] -extern crate mockall_double; +#[cfg(test)] +#[macro_use] +extern crate rstest; // due to linking errors extern crate openssl; @@ -19,7 +18,7 @@ mod init; use init::*; use serde::Deserialize; use std::path::PathBuf; -use u_lib::{config::MASTER_PORT, utils::Env}; +use u_lib::{config::MASTER_PORT, utils::load_env}; use warp::Filter; #[derive(Deserialize)] @@ -32,8 +31,8 @@ pub async fn serve() -> Result<(), String> { init_logger(); prefill_jobs(); - let env = Env::::init().map_err(|e| e.to_string())?; - let routes = init_filters(&env.inner.admin_auth_token); + let env = load_env::().map_err(|e| e.to_string())?; + let routes = init_filters(&env.admin_auth_token); let certs_dir = PathBuf::from("certs"); warp::serve(routes.with(warp::log("warp"))) .tls() @@ -45,25 +44,26 @@ pub async fn serve() -> Result<(), String> { Ok(()) } +/* #[cfg(test)] mod tests { use super::*; - #[double] use crate::handlers::Endpoints; use handlers::build_ok; - use mockall::predicate::*; - use test_case::test_case; use u_lib::messaging::{AsMsg, BaseMessage, Reportable}; use uuid::Uuid; - use warp::test::request; + use warp::test; - #[test_case(Some(Uuid::new_v4()))] - #[test_case(None => panics)] + #[rstest] + #[case(Some(Uuid::new_v4()))] + #[should_panic] + #[case(None)] #[tokio::test] - async fn test_get_agent_jobs_unauthorized(uid: Option) { - let mock = Endpoints::get_agent_jobs_context(); - mock.expect().with(eq(uid)).returning(|_| Ok(build_ok(""))); - request() + async fn test_get_agent_jobs_unauthorized(#[case] uid: Option) { + let mock = Endpoints::faux(); + when!(mock.get_agent_jobs).then_return(Ok(build_ok(""))); + //mock.expect().with(eq(uid)).returning(|_| Ok(build_ok(""))); + test::request() .path(&format!( "/get_agent_jobs/{}", uid.map(|u| u.simple().to_string()).unwrap_or(String::new()) @@ -72,16 +72,15 @@ mod tests { .filter(&init_filters("")) .await .unwrap(); - mock.checkpoint(); } #[tokio::test] async fn test_report_unauth_successful() { - let mock = Endpoints::report_context(); + let mock = Endpoints::report(); mock.expect() .withf(|msg: &BaseMessage<'_, Vec>| msg.inner_ref()[0] == Reportable::Dummy) .returning(|_| Ok(build_ok(""))); - request() + test::request() .path("/report/") .method("POST") .json(&vec![Reportable::Dummy].as_message()) @@ -91,3 +90,4 @@ mod tests { mock.checkpoint(); } } +*/ diff --git a/images/integration-tests/tests_base.Dockerfile b/images/integration-tests/tests_base.Dockerfile deleted file mode 100644 index 81fee07..0000000 --- a/images/integration-tests/tests_base.Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# build from lib/ -FROM rust:1.60 as chef - -RUN rustup target add x86_64-unknown-linux-musl -RUN cargo install cargo-chef - -COPY u_lib /lib/u_lib -COPY u_api_proc_macro /lib/u_api_proc_macro -COPY certs /certs - -WORKDIR /app \ No newline at end of file diff --git a/images/integration-tests/tests_runner.Dockerfile b/images/integration-tests/tests_runner.Dockerfile index ae60226..0a60d7a 100644 --- a/images/integration-tests/tests_runner.Dockerfile +++ b/images/integration-tests/tests_runner.Dockerfile @@ -1,12 +1,4 @@ -# build from integration/ -FROM unki/tests_base as chef - -FROM chef as planner -COPY . /app -RUN cargo chef prepare --recipe-path recipe.json - -FROM chef as builder -COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --release --recipe-path recipe.json +FROM rust:1.60 +RUN rustup target add x86_64-unknown-linux-musl CMD ["sleep", "3600"] \ No newline at end of file diff --git a/integration/Cargo.toml b/integration/Cargo.toml index 14fb0f6..d0c9819 100644 --- a/integration/Cargo.toml +++ b/integration/Cargo.toml @@ -16,7 +16,8 @@ serde_json = "1.0" serde = { version = "1.0.114", features = ["derive"] } futures = "0.3.5" shlex = "1.0.0" -rstest = "0.11" +rstest = "0.12" +once_cell = "1.10.0" [dependencies.u_lib] path = "../lib/u_lib" @@ -25,4 +26,4 @@ version = "*" [[test]] name = "integration" -path = "tests/lib.rs" \ No newline at end of file +path = "tests/lib.rs" diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml index 9d2e218..2186a8e 100644 --- a/integration/docker-compose.yml +++ b/integration/docker-compose.yml @@ -67,6 +67,7 @@ services: condition: service_healthy tests_runner: + user: "${DOCKER_UID}:${DOCKER_GID}" image: unki/tests_runner networks: - u_net diff --git a/integration/docker.py b/integration/docker.py index 2d47517..05182bd 100644 --- a/integration/docker.py +++ b/integration/docker.py @@ -1,9 +1,9 @@ import subprocess + from utils import * BASE_IMAGE_DIR = '../images/integration-tests' -# do not reorder DOCKERFILES = [ { 'name': 'u_agent', @@ -17,13 +17,9 @@ DOCKERFILES = [ 'name': 'u_db', 'ctx': BASE_IMAGE_DIR, }, - { - 'name': 'tests_base', - 'ctx': '../lib', - }, { 'name': 'tests_runner', - 'ctx': '../integration', + 'ctx': BASE_IMAGE_DIR, }, ] @@ -75,11 +71,10 @@ def rebuild_images_if_needed(force_rebuild=False): log(f'Building docker image {img_name}') cmd = [ 'build', - '-t', - img_name, + '-t', img_name, + '-f', f'{BASE_IMAGE_DIR}/{name}.{df_suffix}', ctx, ] - cmd += ['-f', f'{BASE_IMAGE_DIR}/{name}.{df_suffix}'] if force_rebuild: cmd += ['--no-cache'] docker(cmd) diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 374299b..97d06e8 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -1,5 +1,5 @@ #!/bin/bash set -e -cp -r ../certs ../lib/certs +export DOCKER_UID=$(id -u) +export DOCKER_GID=$(id -g) python integration_tests.py $@ -rm -rf ../lib/certs diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs index 596f9a8..5188830 100644 --- a/integration/tests/fixtures/agent.rs +++ b/integration/tests/fixtures/agent.rs @@ -1,3 +1,4 @@ +use crate::helpers::ENV; use u_lib::{api::ClientHandler, messaging::Reportable, models::*}; use uuid::Uuid; @@ -7,14 +8,14 @@ pub struct RegisteredAgent { impl RegisteredAgent { pub async fn unregister(self) { - let cli = ClientHandler::new(None); + let cli = ClientHandler::new(&ENV.u_server); cli.del(Some(self.uid)).await.unwrap(); } } #[fixture] pub async fn register_agent() -> RegisteredAgent { - let cli = ClientHandler::new(None); + let cli = ClientHandler::new(&ENV.u_server); let agent_uid = Uuid::new_v4(); let resp = cli .get_personal_jobs(Some(agent_uid)) diff --git a/integration/tests/helpers/mod.rs b/integration/tests/helpers/mod.rs index 783b365..91d7eb7 100644 --- a/integration/tests/helpers/mod.rs +++ b/integration/tests/helpers/mod.rs @@ -1,3 +1,8 @@ pub mod panel; pub use panel::Panel; + +use once_cell::sync::Lazy; +use u_lib::utils::{env::DefaultEnv, load_env_default}; + +pub static ENV: Lazy = Lazy::new(|| load_env_default().unwrap()); diff --git a/integration/tests/lib.rs b/integration/tests/lib.rs index 70019bb..aac47fb 100644 --- a/integration/tests/lib.rs +++ b/integration/tests/lib.rs @@ -2,21 +2,20 @@ mod fixtures; mod helpers; mod integration; +use crate::helpers::ENV; use u_lib::config::MASTER_PORT; -use u_lib::utils::Env; #[macro_use] extern crate rstest; #[tokio::test] async fn test_non_auth_connection_dropped() { - let env = Env::init_default().unwrap(); let client = reqwest::ClientBuilder::new() .danger_accept_invalid_certs(true) .build() .unwrap(); match client - .get(format!("https://{}:{}", &env.u_server, MASTER_PORT)) + .get(format!("https://{}:{}", &ENV.u_server, MASTER_PORT)) .send() .await { diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index adfc2c4..da445e7 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -16,7 +16,6 @@ lazy_static = "1.4.0" futures = "0.3.5" thiserror = "*" log = "*" -mockall = "0.9.1" env_logger = "0.8.3" diesel-derive-enum = { version = "1", features = ["postgres"] } chrono = "0.4.19" @@ -36,5 +35,4 @@ guess_host_triple = "0.1.2" openssl = "*" [dev-dependencies] -test-case = "1.1.0" -rstest = "0.11" +rstest = "0.12" diff --git a/lib/u_lib/src/builder.rs b/lib/u_lib/src/builder.rs index 746cc12..29aeef1 100644 --- a/lib/u_lib/src/builder.rs +++ b/lib/u_lib/src/builder.rs @@ -1,12 +1,10 @@ use crate::{ - UError, - UErrorBt, - UResult, - cache::JobCache, - executor::{Waiter, DynFut}, - messaging::Reportable, - models::{Agent, AssignedJob, JobMeta, JobType}, - utils::{CombinedResult, OneOrVec} + cache::JobCache, + executor::{DynFut, Waiter}, + messaging::Reportable, + models::{Agent, AssignedJob, JobMeta, JobType}, + utils::{CombinedResult, OneOrVec}, + UError, UErrorBt, UResult, }; use guess_host_triple::guess_host_triple; use std::collections::HashMap; @@ -15,7 +13,7 @@ pub struct JobBuilder { waiter: Waiter, } -impl JobBuilder { +impl JobBuilder { pub fn from_request(job_requests: impl OneOrVec) -> CombinedResult { let job_requests = job_requests.into_vec(); let mut prepared: Vec = vec![]; @@ -38,7 +36,8 @@ impl JobBuilder { return Err(UError::InsuitablePlatform( meta.platform.clone(), curr_platform, - ).into()); + ) + .into()); } let job = AssignedJob::new(req.job_id, Some(&req)); prepared.push(Box::pin(job.run())) @@ -152,19 +151,13 @@ impl NamedJobBuilder { #[cfg(test)] mod tests { - use super::*; - use test_case::test_case; - use std::{time::SystemTime}; use crate::{ - errors::UError, - models::{ - JobMeta, - misc::JobType - }, builder::{JobBuilder, NamedJobBuilder}, + models::{misc::JobType, JobMeta}, unwrap_enum, }; + use std::time::SystemTime; type TestResult = Result>; @@ -178,19 +171,14 @@ mod tests { assert!(now.elapsed().unwrap().as_secs() < SLEEP_SECS + 2) } - #[test_case( - "/bin/sh {}", - Some(b"echo test01 > /tmp/asd; cat /tmp/asd"), + #[rstest] + #[case::sh_payload( + "/bin/sh {}", + Some(b"echo test01 > /tmp/asd; cat /tmp/asd".as_slice()), "test01" - ;"sh payload" - )] - #[test_case( - r#"/usr/bin/python3 -c 'print("test02")'"#, - None, - "test02" - ;"python cmd" )] - #[test_case( + #[case::python_cmd(r#"/usr/bin/python3 -c 'print("test02")'"#, None, "test02")] + #[case::sh_multiline_payload( "/{}", Some( br#"#!/bin/sh @@ -198,20 +186,22 @@ mod tests { mkdir -p $TMPPATH echo test03 > $TMPPATH/t cat $TMPPATH/t - rm -rf $TMPPATH"# + rm -rf $TMPPATH"#.as_slice() ), "test03" - ;"sh multiline payload" )] - #[test_case( + #[case::standalone_binary_with_args( "/{} 'some msg as arg'", - Some(include_bytes!("../tests/fixtures/echoer")), + Some(include_bytes!("../tests/fixtures/echoer").as_slice()), "some msg as arg" - ;"standalone binary with args" )] #[tokio::test] - async fn test_shell_job(cmd: &str, payload: Option<&[u8]>, expected_result: &str) -> TestResult { - let mut job = JobMeta::builder().with_shell(cmd); + async fn test_shell_job( + #[case] cmd: &str, + #[case] payload: Option<&[u8]>, + #[case] expected_result: &str, + ) -> TestResult { + let mut job = JobMeta::builder().with_shell(cmd); if let Some(p) = payload { job = job.with_payload(p); } @@ -228,8 +218,12 @@ mod tests { const SLEEP_SECS: u64 = 1; let now = SystemTime::now(); let longest_job = JobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); - let longest_job = JobBuilder::from_meta(longest_job).unwrap_one().spawn().await; - let ls = JobBuilder::from_meta(JobMeta::from_shell("ls")?).unwrap_one() + let longest_job = JobBuilder::from_meta(longest_job) + .unwrap_one() + .spawn() + .await; + let ls = JobBuilder::from_meta(JobMeta::from_shell("ls")?) + .unwrap_one() .wait_one() .await; let ls = unwrap_enum!(ls, Reportable::Assigned); @@ -272,10 +266,7 @@ mod tests { #[tokio::test] async fn test_failing_shell_job() -> TestResult { let job = JobMeta::from_shell("lol_kek_puk")?; - let job_result = JobBuilder::from_meta(job) - .unwrap_one() - .wait_one() - .await; + let job_result = JobBuilder::from_meta(job).unwrap_one().wait_one().await; let job_result = unwrap_enum!(job_result, Reportable::Assigned); let output = job_result.to_string_result(); assert!(output.contains("No such file")); @@ -283,20 +274,15 @@ mod tests { Ok(()) } - #[test_case( - "/bin/bash {}", - None, - "contains executable" - ; "no binary" - )] - #[test_case( - "/bin/bash", - Some(b"whoami"), - "contains no executable" - ; "no path to binary" - )] + #[rstest] + #[case::no_binary("/bin/bash {}", None, "contains executable")] + #[case::no_path_to_binary("/bin/bash", Some(b"whoami".as_slice()), "contains no executable")] #[tokio::test] - async fn test_job_building_failed(cmd: &str, payload: Option<&[u8]>, err_str: &str) -> TestResult { + async fn test_job_building_failed( + #[case] cmd: &str, + #[case] payload: Option<&[u8]>, + #[case] err_str: &str, + ) -> TestResult { let mut job = JobMeta::builder().with_shell(cmd); if let Some(p) = payload { job = job.with_payload(p); @@ -311,11 +297,15 @@ mod tests { async fn test_different_job_types() -> TestResult { let mut jobs = NamedJobBuilder::from_meta(vec![ ("sleeper", JobMeta::from_shell("sleep 3")?), - ("gatherer", JobMeta::builder().with_type(JobType::Manage).build()?) - ]).wait().await; + ( + "gatherer", + JobMeta::builder().with_type(JobType::Manage).build()?, + ), + ]) + .wait() + .await; let gathered = jobs.pop("gatherer"); assert_eq!(unwrap_enum!(gathered, Reportable::Agent).alias, None); Ok(()) } - -} \ No newline at end of file +} diff --git a/lib/u_lib/src/lib.rs b/lib/u_lib/src/lib.rs index 1f37843..b49f71d 100644 --- a/lib/u_lib/src/lib.rs +++ b/lib/u_lib/src/lib.rs @@ -40,5 +40,6 @@ extern crate diesel; extern crate log; extern crate env_logger; +#[cfg(test)] #[macro_use] -extern crate mockall; +extern crate rstest; diff --git a/lib/u_lib/src/utils/env.rs b/lib/u_lib/src/utils/env.rs index a5a7352..a3c9345 100644 --- a/lib/u_lib/src/utils/env.rs +++ b/lib/u_lib/src/utils/env.rs @@ -2,35 +2,28 @@ use envy::{from_env, Result as EnvResult}; use serde::{de::DeserializeOwned, Deserialize}; #[derive(Deserialize)] -pub struct NoneEnv; - -#[derive(Deserialize)] -pub struct Env { +pub struct DefaultEnv { #[serde(default = "default_host")] pub u_server: String, - pub inner: E, } -impl Env { - pub fn init_default() -> EnvResult { - let envs = [".env", ".env.private"]; - for envfile in &envs { - dotenv::from_filename(envfile).ok(); - } - from_env() - } +pub fn load_env() -> EnvResult { + dot(); + from_env() +} + +pub fn load_env_default() -> EnvResult { + dot(); + from_env() } -impl Env { - pub fn init() -> EnvResult { - let envs = [".env", ".env.private"]; - for envfile in &envs { - dotenv::from_filename(envfile).ok(); - } - from_env() +fn dot() { + let envs = [".env", ".env.private"]; + for envfile in &envs { + dotenv::from_filename(envfile).ok(); } } -fn default_host() -> String { +pub fn default_host() -> String { "ortem.xyz".to_string() } diff --git a/lib/u_lib/src/utils/mod.rs b/lib/u_lib/src/utils/mod.rs index 3cddc6b..df527db 100644 --- a/lib/u_lib/src/utils/mod.rs +++ b/lib/u_lib/src/utils/mod.rs @@ -1,19 +1,19 @@ -mod combined_result; -mod conv; -mod env; -mod fmt; -mod misc; -mod proc_output; -mod storage; +pub mod combined_result; +pub mod conv; +pub mod env; +pub mod fmt; +pub mod misc; +pub mod proc_output; +pub mod storage; #[cfg(not(target_arch = "wasm32"))] -mod tempfile; +pub mod tempfile; #[cfg(unix)] -mod unix; -mod vec_display; +pub mod unix; +pub mod vec_display; pub use combined_result::*; pub use conv::*; -pub use env::Env; +pub use env::{load_env, load_env_default}; pub use fmt::*; pub use misc::*; pub use proc_output::*; diff --git a/lib/u_lib/src/utils/proc_output.rs b/lib/u_lib/src/utils/proc_output.rs index 5236e6b..a9c4135 100644 --- a/lib/u_lib/src/utils/proc_output.rs +++ b/lib/u_lib/src/utils/proc_output.rs @@ -117,46 +117,42 @@ impl ProcOutput { #[cfg(test)] mod tests { use crate::utils::{bytes_to_string, ProcOutput}; - use test_case::test_case; const STDOUT: &str = "<***STDOUT***>"; const STDERR: &str = "<***STDERR***>"; - #[test_case( + #[rstest] + #[case::stdout_stderr( "lol", "kek", &format!("{}lol{}kek", STDOUT, STDERR) - ;"stdout stderr" )] - #[test_case( + #[case::stderr( "", "kek", &format!("{}kek", STDERR) - ;"stderr" )] - fn test_to_combined(stdout: &str, stderr: &str, result: &str) { + fn test_to_combined(#[case] stdout: &str, #[case] stderr: &str, #[case] result: &str) { let output = ProcOutput::new() .stdout(stdout.as_bytes().to_vec()) .stderr(stderr.as_bytes().to_vec()); assert_eq!(&bytes_to_string(&output.into_combined()), result) } - #[test_case( + #[rstest] + #[case::stdout_stderr( &format!("{}lal{}kik", STDOUT, STDERR), "lal\nkik" - ;"stdout stderr" )] - #[test_case( + #[case::stdout( &format!("{}qeq", STDOUT), "qeq" - ;"stdout" )] - #[test_case( + #[case::stderr( &format!("{}vev", STDERR), "vev" - ;"stderr" )] - fn test_from_combined(src: &str, result: &str) { + fn test_from_combined(#[case] src: &str, #[case] result: &str) { let output = ProcOutput::from_combined(src.as_bytes()).unwrap(); assert_eq!(bytes_to_string(&output.to_appropriate()).trim(), result); } From 83252b6f95273c5309b190beb6f838857725fc77 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Mon, 30 May 2022 02:44:47 +0500 Subject: [PATCH 20/30] cosmetics, remove backtrace --- .env | 3 +- Makefile.toml | 9 +-- bin/u_panel/src/argparse.rs | 6 +- bin/u_server/src/db.rs | 38 ++++------ bin/u_server/src/errors.rs | 22 ++++++ bin/u_server/src/handlers.rs | 99 +++++++++----------------- bin/u_server/src/init.rs | 41 +++++++---- bin/u_server/src/main.rs | 2 +- bin/u_server/src/u_server.rs | 8 ++- integration/Cargo.toml | 1 - integration/docker-compose.yml | 6 +- integration/tests/helpers/panel.rs | 50 +++++++------ integration/tests/lib.rs | 4 +- lib/u_api_proc_macro/src/lib.rs | 22 +++--- lib/u_lib/Cargo.toml | 3 +- lib/u_lib/src/builder.rs | 8 +-- lib/u_lib/src/datatypes.rs | 6 +- lib/u_lib/src/errors/chan.rs | 8 +-- lib/u_lib/src/errors/variants.rs | 52 +------------- lib/u_lib/src/lib.rs | 2 +- lib/u_lib/src/messaging/base.rs | 5 +- lib/u_lib/src/models/jobs/assigned.rs | 6 +- lib/u_lib/src/utils/combined_result.rs | 4 +- 23 files changed, 180 insertions(+), 225 deletions(-) create mode 100644 bin/u_server/src/errors.rs diff --git a/.env b/.env index 60ff916..70e3dfb 100644 --- a/.env +++ b/.env @@ -1,4 +1,5 @@ DB_HOST=u_db DB_NAME=u_db DB_USER=postgres -RUST_BACKTRACE=1 \ No newline at end of file +RUST_BACKTRACE=1 +U_SERVER=u_server \ No newline at end of file diff --git a/Makefile.toml b/Makefile.toml index 32533ec..e70a9dc 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -4,11 +4,12 @@ default_to_workspace = false [env] TARGET = "x86_64-unknown-linux-musl" CARGO = "cargo" -PREFIX = "${CARGO_MAKE_WORKING_DIRECTORY}/static" +ROOTDIR = "${CARGO_MAKE_WORKING_DIRECTORY}" +STATIC_PREFIX = "${ROOTDIR}/static" PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true" -PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = "${PREFIX}/bin/pg_config" +PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = "${STATIC_PREFIX}/bin/pg_config" OPENSSL_STATIC = "true" -OPENSSL_DIR = "${PREFIX}" +OPENSSL_DIR = "${STATIC_PREFIX}" [tasks.build_static_libs] script = "./scripts/build_musl_libs.sh" @@ -32,7 +33,7 @@ args = ["build", "--target", "${TARGET}", "${@}"] script = ''' if [[ "${@}" =~ "--release" ]]; then echo "Creating symlink to release dir..." - ln -s ./target/${TARGET}/release ./release || true + ln -s ${ROOTDIR}/target/${TARGET}/release ${ROOTDIR}/release || true BINS=$(ls ./release/u_* -1 | grep -v ".d") echo "Stripping..." strip $BINS diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index cfe9050..c9f8c0f 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -1,7 +1,7 @@ use std::fmt; use structopt::StructOpt; use u_lib::{ - api::ClientHandler, datatypes::DataResult, messaging::AsMsg, models::JobMeta, UError, UResult, + api::ClientHandler, datatypes::PanelResult, messaging::AsMsg, models::JobMeta, UError, UResult, }; use uuid::Uuid; @@ -92,8 +92,8 @@ pub async fn process_cmd(client: ClientHandler, args: Args) -> UResult<()> { pub fn print(&self, data: UResult) { if self.json { let data = match data { - Ok(r) => DataResult::Ok(r), - Err(e) => DataResult::Err(e), + Ok(r) => PanelResult::Ok(r), + Err(e) => PanelResult::Err(e), }; println!("{}", serde_json::to_string_pretty(&data).unwrap()); } else { diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index cff590f..db064ec 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -1,3 +1,4 @@ +use crate::errors::{Error, SResult}; use diesel::{pg::PgConnection, prelude::*, result::Error as DslError}; use once_cell::sync::OnceCell; use serde::Deserialize; @@ -5,7 +6,6 @@ use std::sync::{Arc, Mutex, MutexGuard}; use u_lib::{ models::{schema, Agent, AgentError, AssignedJob, JobMeta, JobState}, utils::load_env, - ULocalError, ULocalResult, }; use uuid::Uuid; @@ -39,7 +39,7 @@ impl UDB { .unwrap() } - pub fn report_error(&self, error: &AgentError) -> ULocalResult<()> { + pub fn report_error(&self, error: &AgentError) -> SResult<()> { use schema::errors; diesel::insert_into(errors::table) .values(error) @@ -47,7 +47,7 @@ impl UDB { Ok(()) } - pub fn insert_jobs(&self, job_metas: &[JobMeta]) -> ULocalResult<()> { + pub fn insert_jobs(&self, job_metas: &[JobMeta]) -> SResult<()> { use schema::jobs; diesel::insert_into(jobs::table) .values(job_metas) @@ -55,7 +55,7 @@ impl UDB { Ok(()) } - pub fn get_jobs(&self, uid: Option) -> ULocalResult> { + pub fn get_jobs(&self, uid: Option) -> SResult> { use schema::jobs; let result = if uid.is_some() { jobs::table @@ -67,7 +67,7 @@ impl UDB { Ok(result) } - pub fn find_job_by_alias(&self, alias: &str) -> ULocalResult { + pub fn find_job_by_alias(&self, alias: &str) -> SResult { use schema::jobs; let result = jobs::table .filter(jobs::alias.eq(alias)) @@ -75,7 +75,7 @@ impl UDB { Ok(result) } - pub fn insert_agent(&self, agent: &Agent) -> ULocalResult<()> { + pub fn insert_agent(&self, agent: &Agent) -> SResult<()> { use schema::agents; diesel::insert_into(agents::table) .values(agent) @@ -86,7 +86,7 @@ impl UDB { Ok(()) } - pub fn get_agents(&self, uid: Option) -> ULocalResult> { + pub fn get_agents(&self, uid: Option) -> SResult> { use schema::agents; let result = if uid.is_some() { agents::table @@ -98,7 +98,7 @@ impl UDB { Ok(result) } - pub fn update_job_status(&self, uid: Uuid, status: JobState) -> ULocalResult<()> { + pub fn update_job_status(&self, uid: Uuid, status: JobState) -> SResult<()> { use schema::results; diesel::update(results::table) .filter(results::id.eq(uid)) @@ -108,11 +108,7 @@ impl UDB { } //TODO: filters possibly could work in a wrong way, check - pub fn get_exact_jobs( - &self, - uid: Option, - personal: bool, - ) -> ULocalResult> { + pub fn get_exact_jobs(&self, uid: Option, personal: bool) -> SResult> { use schema::results; let mut q = results::table.into_boxed(); /*if uid.is_some() { @@ -134,14 +130,10 @@ impl UDB { Ok(result) } - pub fn set_jobs_for_agent( - &self, - agent_uid: &Uuid, - job_uids: &[Uuid], - ) -> ULocalResult> { + pub fn set_jobs_for_agent(&self, agent_uid: &Uuid, job_uids: &[Uuid]) -> SResult> { use schema::{agents::dsl::agents, jobs::dsl::jobs, results}; if let Err(DslError::NotFound) = agents.find(agent_uid).first::(&self.conn) { - return Err(ULocalError::NotFound(agent_uid.to_string())); + return Err(Error::NotFound(agent_uid.to_string())); } let not_found_jobs = job_uids .iter() @@ -154,7 +146,7 @@ impl UDB { }) .collect::>(); if !not_found_jobs.is_empty() { - return Err(ULocalError::NotFound(not_found_jobs.join(", "))); + return Err(Error::NotFound(not_found_jobs.join(", "))); } let job_requests = job_uids .iter() @@ -174,7 +166,7 @@ impl UDB { Ok(assigned_uids) } - pub fn del_jobs(&self, uids: &[Uuid]) -> ULocalResult { + pub fn del_jobs(&self, uids: &[Uuid]) -> SResult { use schema::jobs; let mut affected = 0; for &uid in uids { @@ -186,7 +178,7 @@ impl UDB { Ok(affected) } - pub fn del_results(&self, uids: &[Uuid]) -> ULocalResult { + pub fn del_results(&self, uids: &[Uuid]) -> SResult { use schema::results; let mut affected = 0; for &uid in uids { @@ -198,7 +190,7 @@ impl UDB { Ok(affected) } - pub fn del_agents(&self, uids: &[Uuid]) -> ULocalResult { + pub fn del_agents(&self, uids: &[Uuid]) -> SResult { use schema::agents; let mut affected = 0; for &uid in uids { diff --git a/bin/u_server/src/errors.rs b/bin/u_server/src/errors.rs new file mode 100644 index 0000000..c77472a --- /dev/null +++ b/bin/u_server/src/errors.rs @@ -0,0 +1,22 @@ +use diesel::result::Error as DslError; +use thiserror::Error; +use warp::reject::Reject; + +pub type SResult = Result; + +#[derive(Error, Debug)] +pub enum Error { + #[error("{0} is not found")] + NotFound(String), + + #[error("Error processing {0}")] + ProcessingError(String), + + #[error(transparent)] + DBError(#[from] DslError), + + #[error("General error: {0}")] + Other(String), +} + +impl Reject for Error {} diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index 9a23f17..d1573cc 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -1,19 +1,17 @@ use crate::db::UDB; +use crate::errors::Error; use diesel::SaveChangesDsl; -use hyper::Body; -use serde::Serialize; use u_lib::{ messaging::{AsMsg, BaseMessage, Reportable}, models::*, utils::{OneOrVec, Stripped}, - ULocalError, }; use uuid::Uuid; use warp::{ http::{Response, StatusCode}, Rejection, Reply, }; - +/* pub fn build_response(code: StatusCode, body: impl Into) -> Response { Response::builder().status(code).body(body.into()).unwrap() } @@ -29,116 +27,86 @@ pub fn build_err(body: impl ToString) -> Response { pub fn build_message(m: M) -> Response { warp::reply::json(&m.as_message()).into_response() } +*/ pub struct Endpoints; impl Endpoints { - pub async fn add_agent(msg: Agent) -> Result, Rejection> { - info!("hnd: add_agent"); - UDB::lock_db() - .insert_agent(&msg) - .map(|_| build_ok("")) - .or_else(|e| Ok(build_err(e))) + pub async fn add_agent(msg: Agent) -> Result<(), Rejection> { + UDB::lock_db().insert_agent(&msg).map_err(From::from) } - pub async fn get_agents(uid: Option) -> Result, Rejection> { - info!("hnd: get_agents"); - UDB::lock_db() - .get_agents(uid) - .map(build_message) - .or_else(|e| Ok(build_err(e))) + pub async fn get_agents(uid: Option) -> Result, Rejection> { + UDB::lock_db().get_agents(uid).map_err(From::from) } - pub async fn get_jobs(uid: Option) -> Result, Rejection> { - info!("hnd: get_jobs"); - UDB::lock_db() - .get_jobs(uid) - .map(build_message) - .or_else(|e| Ok(build_err(e))) + pub async fn get_jobs(uid: Option) -> Result, Rejection> { + UDB::lock_db().get_jobs(uid).map_err(From::from) } - pub async fn get_agent_jobs(uid: Option) -> Result, Rejection> { - info!("hnd: get_agent_jobs"); + pub async fn get_agent_jobs(uid: Option) -> Result, Rejection> { UDB::lock_db() .get_exact_jobs(uid, false) - .map(build_message) - .or_else(|e| Ok(build_err(e))) + .map_err(From::from) } - pub async fn get_personal_jobs(uid: Option) -> Result, Rejection> { - info!("hnd: get_personal_jobs"); - let agents = UDB::lock_db().get_agents(uid).unwrap(); + pub async fn get_personal_jobs(uid: Option) -> Result, Rejection> { + let agents = UDB::lock_db().get_agents(uid)?; if agents.is_empty() { let db = UDB::lock_db(); - db.insert_agent(&Agent::with_id(uid.unwrap())).unwrap(); - let job = db.find_job_by_alias("agent_hello").unwrap(); - if let Err(e) = db.set_jobs_for_agent(&uid.unwrap(), &[job.id]) { - return Ok(build_err(e)); - } + db.insert_agent(&Agent::with_id(uid.unwrap()))?; + let job = db.find_job_by_alias("agent_hello")?; + db.set_jobs_for_agent(&uid.unwrap(), &[job.id])?; } let result = UDB::lock_db().get_exact_jobs(uid, true); match result { Ok(r) => { let db = UDB::lock_db(); for j in r.iter() { - db.update_job_status(j.id, JobState::Running).unwrap(); + db.update_job_status(j.id, JobState::Running)?; } - Ok(build_message(r)) + Ok(r) } - Err(e) => Ok(build_err(e)), + Err(e) => Err(e.into()), } } - pub async fn upload_jobs( - msg: BaseMessage<'static, Vec>, - ) -> Result, Rejection> { - info!("hnd: upload_jobs"); + pub async fn upload_jobs(msg: BaseMessage<'static, Vec>) -> Result<(), Rejection> { UDB::lock_db() .insert_jobs(&msg.into_inner()) - .map(|_| build_ok("")) - .or_else(|e| Ok(build_err(e))) + .map_err(From::from) } - pub async fn del(uid: Uuid) -> Result, Rejection> { - info!("hnd: del"); + pub async fn del(uid: Uuid) -> Result { let db = UDB::lock_db(); let del_fns = &[UDB::del_agents, UDB::del_jobs, UDB::del_results]; for del_fn in del_fns { let affected = del_fn(&db, &[uid]).unwrap(); if affected > 0 { - return Ok(build_message(affected as i32)); + return Ok(affected); } } - Ok(build_message(0)) + Ok(0) } pub async fn set_jobs( agent_uid: Uuid, msg: BaseMessage<'static, Vec>, - ) -> Result, Rejection> { - info!("hnd: set_jobs_by_alias, agent: {}", agent_uid); - let jobs: Result, ULocalError> = msg - .into_inner() + ) -> Result, Rejection> { + msg.into_inner() .into_iter() .map(|ident| { - info!("hnd: set_jobs_by_alias, job: {}", ident); Uuid::parse_str(&ident) .or_else(|_| UDB::lock_db().find_job_by_alias(&ident).map(|j| j.id)) }) - .collect(); - match jobs { - Ok(j) => UDB::lock_db() - .set_jobs_for_agent(&agent_uid, &j) - .map(build_message) - .or_else(|e| Ok(build_err(e))), - Err(e) => Ok(build_err(e)), - } + .collect::, Error>>() + .and_then(|j| UDB::lock_db().set_jobs_for_agent(&agent_uid, &j)) + .map_err(From::from) } pub async fn report + AsMsg + 'static>( msg: BaseMessage<'static, Data>, - ) -> Result, Rejection> { - info!("hnd: report"); + ) -> Result<(), Rejection> { let id = msg.id; let mut failed = vec![]; for entry in msg.into_inner().into_vec() { @@ -150,7 +118,7 @@ impl Endpoints { let db = UDB::lock_db(); if let Err(e) = res .save_changes::(&db.conn) - .map_err(ULocalError::from) + .map_err(Error::from) { failed.push(e.to_string()) } @@ -172,9 +140,8 @@ impl Endpoints { } } if !failed.is_empty() { - let err_msg = ULocalError::ProcessingError(failed.join(", ")); - return Ok(build_err(err_msg)); + return Err(Error::ProcessingError(failed.join(", ")).into()); } - Ok(build_ok("")) + Ok(()) } } diff --git a/bin/u_server/src/init.rs b/bin/u_server/src/init.rs index 60b3396..8127227 100644 --- a/bin/u_server/src/init.rs +++ b/bin/u_server/src/init.rs @@ -1,5 +1,5 @@ -use crate::db::UDB; use crate::handlers::Endpoints; +use crate::{db::UDB, errors::SResult}; use serde::de::DeserializeOwned; use std::path::PathBuf; use u_lib::{ @@ -7,7 +7,11 @@ use u_lib::{ models::*, }; use uuid::Uuid; -use warp::{body, Filter, Rejection, Reply}; +use warp::{ + body, + reply::{json, reply, Json}, + Filter, Rejection, Reply, +}; fn get_content() -> impl Filter,), Error = Rejection> + Clone where @@ -16,6 +20,10 @@ where body::content_length_limit(1024 * 64).and(body::json::>()) } +fn into_message(msg: M) -> Json { + json(&msg.as_message()) +} + pub fn init_filters( auth_token: &str, ) -> impl Filter + Clone { @@ -28,12 +36,14 @@ pub fn init_filters( .map(Some) .or_else(infallible_none), ) - .and_then(Endpoints::get_agents); + .and_then(Endpoints::get_agents) + .map(into_message); let upload_jobs = warp::post() .and(warp::path("upload_jobs")) .and(get_content::>()) - .and_then(Endpoints::upload_jobs); + .and_then(Endpoints::upload_jobs) + .map(|_| reply()); let get_jobs = warp::get() .and(warp::path("get_jobs")) @@ -42,7 +52,8 @@ pub fn init_filters( .map(Some) .or_else(infallible_none), ) - .and_then(Endpoints::get_jobs); + .and_then(Endpoints::get_jobs) + .map(into_message); let get_agent_jobs = warp::get() .and(warp::path("get_agent_jobs")) @@ -51,27 +62,33 @@ pub fn init_filters( .map(Some) .or_else(infallible_none), ) - .and_then(Endpoints::get_agent_jobs); + .and_then(Endpoints::get_agent_jobs) + .map(into_message); let get_personal_jobs = warp::get() .and(warp::path("get_personal_jobs")) .and(warp::path::param::().map(Some)) - .and_then(Endpoints::get_personal_jobs); + .and_then(Endpoints::get_personal_jobs) + .map(into_message); let del = warp::get() .and(warp::path("del")) .and(warp::path::param::()) - .and_then(Endpoints::del); + .and_then(Endpoints::del) + .map(|_| reply()); let set_jobs = warp::post() .and(warp::path("set_jobs")) .and(warp::path::param::()) .and(get_content::>()) - .and_then(Endpoints::set_jobs); + .and_then(Endpoints::set_jobs) + .map(into_message); let report = warp::post() .and(warp::path("report")) - .and(get_content::>().and_then(Endpoints::report)); + .and(get_content::>()) + .and_then(Endpoints::report) + .map(|_| reply()); let auth_token = format!("Bearer {auth_token}",).into_boxed_str(); let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); @@ -89,13 +106,13 @@ pub fn init_filters( auth_zone.or(agent_zone) } -pub fn prefill_jobs() { +pub fn prefill_jobs() -> SResult<()> { let agent_hello = JobMeta::builder() .with_type(misc::JobType::Manage) .with_alias("agent_hello") .build() .unwrap(); - UDB::lock_db().insert_jobs(&[agent_hello]).unwrap(); + UDB::lock_db().insert_jobs(&[agent_hello]) } pub fn init_logger() { diff --git a/bin/u_server/src/main.rs b/bin/u_server/src/main.rs index e402a55..84d2139 100644 --- a/bin/u_server/src/main.rs +++ b/bin/u_server/src/main.rs @@ -2,5 +2,5 @@ use u_server_lib::serve; #[tokio::main] async fn main() -> Result<(), String> { - serve().await + serve().await.map_err(|e| e.to_string()) } diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index bf9d8ed..1483acd 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -12,9 +12,11 @@ extern crate diesel; // in this block mod db; +mod errors; mod handlers; mod init; +use errors::{Error, SResult}; use init::*; use serde::Deserialize; use std::path::PathBuf; @@ -27,11 +29,11 @@ struct ServEnv { } //TODO: tracing-subscriber -pub async fn serve() -> Result<(), String> { +pub async fn serve() -> SResult<()> { init_logger(); - prefill_jobs(); + prefill_jobs()?; - let env = load_env::().map_err(|e| e.to_string())?; + let env = load_env::().map_err(|e| Error::Other(e.to_string()))?; let routes = init_filters(&env.admin_auth_token); let certs_dir = PathBuf::from("certs"); warp::serve(routes.with(warp::log("warp"))) diff --git a/integration/Cargo.toml b/integration/Cargo.toml index d0c9819..a4ae852 100644 --- a/integration/Cargo.toml +++ b/integration/Cargo.toml @@ -14,7 +14,6 @@ uuid = { version = "0.6.5", features = ["serde", "v4"] } reqwest = { version = "0.11", features = ["json"] } serde_json = "1.0" serde = { version = "1.0.114", features = ["derive"] } -futures = "0.3.5" shlex = "1.0.0" rstest = "0.12" once_cell = "1.10.0" diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml index 2186a8e..40122f3 100644 --- a/integration/docker-compose.yml +++ b/integration/docker-compose.yml @@ -6,13 +6,14 @@ networks: services: u_server: + user: "${DOCKER_UID:-1000}:${DOCKER_GID:-1000}" image: unki/u_server networks: - u_net volumes: - ../release/u_server:/unki/u_server - ../certs:/unki/certs - - ../logs:/unki/logs + - ../logs:/unki/logs:rw working_dir: /unki command: /unki/u_server depends_on: @@ -52,6 +53,7 @@ services: retries: 3 u_agent: + user: "${DOCKER_UID:-1000}:${DOCKER_GID:-1000}" image: unki/u_agent networks: - u_net @@ -67,7 +69,7 @@ services: condition: service_healthy tests_runner: - user: "${DOCKER_UID}:${DOCKER_GID}" + user: "${DOCKER_UID:-1000}:${DOCKER_GID:-1000}" image: unki/tests_runner networks: - u_net diff --git a/integration/tests/helpers/panel.rs b/integration/tests/helpers/panel.rs index 2ddf4f0..27d2639 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration/tests/helpers/panel.rs @@ -1,17 +1,14 @@ use serde::de::DeserializeOwned; use serde_json::{from_slice, Value}; -use shlex::split; -use std::fmt::Display; +use std::fmt::{Debug, Display}; use std::process::{Command, Output}; use u_lib::{ - datatypes::DataResult, + datatypes::PanelResult, utils::{bytes_to_string, ProcOutput}, }; const PANEL_BINARY: &str = "/u_panel"; -type PanelResult = Result, String>; - pub struct Panel; impl Panel { @@ -26,31 +23,40 @@ impl Panel { pub fn output_argv(argv: &[&str]) -> PanelResult { let result = Self::run(argv); let output = ProcOutput::from_output(&result).to_appropriate(); - from_slice(&output).map_err(|e| { - eprintln!( - "Failed to decode panel response: '{}'", - bytes_to_string(&output) - ); - e.to_string() - }) + from_slice(&output) + .map_err(|e| { + eprintln!( + "Failed to decode panel response: '{}'", + bytes_to_string(&output) + ); + e.to_string() + }) + .unwrap() } - pub fn output(args: impl Into + Display) -> PanelResult { - println!("Executing '{PANEL_BINARY} {}'", &args); - let splitted = split(args.into().as_ref()).unwrap(); - Self::output_argv( + pub fn output( + args: impl Into + Display, + ) -> PanelResult { + eprintln!("EXEC >>> {PANEL_BINARY} {}", &args); + let splitted = shlex::split(args.into().as_ref()).unwrap(); + let result = Self::output_argv( splitted .iter() .map(|s| s.as_ref()) .collect::>() .as_ref(), - ) + ); + match &result { + PanelResult::Ok(r) => eprintln!("<<<+ {r:?}"), + PanelResult::Err(e) => eprintln!("<<(data: PanelResult) -> T { - match data.unwrap() { - DataResult::Ok(r) => r, - DataResult::Err(e) => panic!("Panel failed: {}", e), + fn status_is_ok(data: PanelResult) -> T { + match data { + PanelResult::Ok(r) => r, + PanelResult::Err(e) => panic!("Panel failed: {}", e), } } @@ -59,7 +65,7 @@ impl Panel { Self::status_is_ok(result); } - pub fn check_output(args: impl Into + Display) -> T { + pub fn check_output(args: impl Into + Display) -> T { let result = Self::output(args); Self::status_is_ok(result) } diff --git a/integration/tests/lib.rs b/integration/tests/lib.rs index aac47fb..8009650 100644 --- a/integration/tests/lib.rs +++ b/integration/tests/lib.rs @@ -20,7 +20,9 @@ async fn test_non_auth_connection_dropped() { .await { Err(e) => { - assert!(e.to_string().contains("channel closed")) + let err = e.to_string(); + println!("{err}"); + assert!(err.contains("channel closed")) } _ => panic!("no error occured on foreign client connection"), } diff --git a/lib/u_api_proc_macro/src/lib.rs b/lib/u_api_proc_macro/src/lib.rs index 1a400ab..a6d0e89 100644 --- a/lib/u_api_proc_macro/src/lib.rs +++ b/lib/u_api_proc_macro/src/lib.rs @@ -69,20 +69,22 @@ pub fn api_route(args: TokenStream, item: TokenStream) -> TokenStream { Ok(_) => Ok(()), Err(e) => Err(UError::from(e)) }; + let resp = response.text().await?; let result = match is_success { - Ok(_) => response.json::>() - .await - .map(|msg| msg.into_inner()) - .or_else(|e| { - match content_len { - Some(0) => Ok(Default::default()), - _ => Err(UError::from(e)) - } - }), + Ok(_) => { + serde_json::from_str::>(&resp) + .map(|msg| msg.into_inner()) + .or_else(|e| { + match content_len { + Some(0) => Ok(Default::default()), + _ => Err(UError::NetError(e.to_string(), resp.clone())) + } + }) + }, Err(UError::NetError(err_src, _)) => Err( UError::NetError( err_src, - response.text().await? + resp ) ), _ => unreachable!() diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index da445e7..972645e 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -24,15 +24,14 @@ once_cell = "1.7.2" shlex = "1.0.0" u_api_proc_macro = { version = "*", path = "../u_api_proc_macro" } crossbeam = "0.8.1" -backtrace = "0.3.61" diesel = { version = "1.4.5", features = ["postgres", "uuid"] } envy = "0.4.2" +serde_json = "1.0.81" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] reqwest = { version = "0.11", features = ["json", "native-tls"] } tokio = { version = "1.2.0", features = ["rt-multi-thread", "sync", "macros", "process", "time"] } guess_host_triple = "0.1.2" -openssl = "*" [dev-dependencies] rstest = "0.12" diff --git a/lib/u_lib/src/builder.rs b/lib/u_lib/src/builder.rs index 29aeef1..c9efcb0 100644 --- a/lib/u_lib/src/builder.rs +++ b/lib/u_lib/src/builder.rs @@ -4,7 +4,7 @@ use crate::{ messaging::Reportable, models::{Agent, AssignedJob, JobMeta, JobType}, utils::{CombinedResult, OneOrVec}, - UError, UErrorBt, UResult, + UError, UResult, }; use guess_host_triple::guess_host_triple; use std::collections::HashMap; @@ -17,11 +17,11 @@ impl JobBuilder { pub fn from_request(job_requests: impl OneOrVec) -> CombinedResult { let job_requests = job_requests.into_vec(); let mut prepared: Vec = vec![]; - let mut result = CombinedResult::::new(); + let mut result = CombinedResult::::new(); for req in job_requests { let job_meta = JobCache::get(req.job_id); if job_meta.is_none() { - result.err(UError::NoJob(req.job_id).into_bt()); + result.err(UError::NoJob(req.job_id)); continue; } let job_meta = job_meta.unwrap(); @@ -288,7 +288,7 @@ mod tests { job = job.with_payload(p); } let err = job.build().unwrap_err(); - let err_msg = unwrap_enum!(err.err, UError::JobArgsError); + let err_msg = unwrap_enum!(err, UError::JobArgsError); assert!(err_msg.contains(err_str)); Ok(()) } diff --git a/lib/u_lib/src/datatypes.rs b/lib/u_lib/src/datatypes.rs index 983747f..f7d09a6 100644 --- a/lib/u_lib/src/datatypes.rs +++ b/lib/u_lib/src/datatypes.rs @@ -1,10 +1,10 @@ -use crate::UErrorBt; +use crate::UError; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] #[serde(rename_all = "lowercase")] #[serde(tag = "status", content = "data")] -pub enum DataResult { +pub enum PanelResult { Ok(M), - Err(UErrorBt), + Err(UError), } diff --git a/lib/u_lib/src/errors/chan.rs b/lib/u_lib/src/errors/chan.rs index cf78fd0..1c2b995 100644 --- a/lib/u_lib/src/errors/chan.rs +++ b/lib/u_lib/src/errors/chan.rs @@ -1,8 +1,8 @@ -use crate::UErrorBt; +use crate::UError; use crossbeam::channel::{self, Receiver, Sender}; use once_cell::sync::OnceCell; -type ChanError = UErrorBt; +type ChanError = UError; static ERR_CHAN: OnceCell = OnceCell::new(); pub struct ErrChan { @@ -18,8 +18,8 @@ impl ErrChan { }) } - pub fn send(msg: impl Into) { - Self::get().tx.send(msg.into()).unwrap() + pub fn send(msg: ChanError) { + Self::get().tx.send(msg).unwrap() } pub fn recv() -> ChanError { diff --git a/lib/u_lib/src/errors/variants.rs b/lib/u_lib/src/errors/variants.rs index 103afe1..829d7ab 100644 --- a/lib/u_lib/src/errors/variants.rs +++ b/lib/u_lib/src/errors/variants.rs @@ -1,35 +1,10 @@ -use backtrace::Backtrace as CrateBacktrace; -use diesel::result::Error as DslError; #[cfg(not(target_arch = "wasm32"))] use reqwest::Error as ReqError; use serde::{Deserialize, Serialize}; -use std::fmt; use thiserror::Error; use uuid::Uuid; -pub type UResult = std::result::Result; -pub type ULocalResult = std::result::Result; - -#[derive(Error, Debug, Serialize, Deserialize, Clone)] -pub struct UErrorBt { - pub err: UError, - pub backtrace: String, -} - -impl fmt::Display for UErrorBt { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}\nBACKTRACE: \n{:?}", self.err, self.backtrace) - } -} - -impl From for UErrorBt { - fn from(err: UError) -> UErrorBt { - UErrorBt { - err, - backtrace: format!("{:?}", CrateBacktrace::new()), - } - } -} +pub type UResult = std::result::Result; #[derive(Error, Debug, Serialize, Deserialize, Clone)] pub enum UError { @@ -70,34 +45,9 @@ pub enum UError { PanelError(String), } -impl UError { - pub fn into_bt(self) -> UErrorBt { - UErrorBt::from(self) - } -} - -#[cfg(not(target_arch = "wasm32"))] -impl From for UErrorBt { - fn from(e: ReqError) -> Self { - UError::from(e).into_bt() - } -} - #[cfg(not(target_arch = "wasm32"))] impl From for UError { fn from(e: ReqError) -> Self { UError::NetError(e.to_string(), String::new()) } } - -#[derive(Error, Debug)] -pub enum ULocalError { - #[error("{0} is not found")] - NotFound(String), - - #[error("Error processing {0}")] - ProcessingError(String), - - #[error(transparent)] - DBError(#[from] DslError), -} diff --git a/lib/u_lib/src/lib.rs b/lib/u_lib/src/lib.rs index b49f71d..2b6fa07 100644 --- a/lib/u_lib/src/lib.rs +++ b/lib/u_lib/src/lib.rs @@ -25,7 +25,7 @@ pub mod exports { } pub use config::UID; -pub use errors::{UError, UErrorBt, ULocalError, ULocalResult, UResult}; +pub use errors::{UError, UResult}; pub use exports::*; pub mod schema_exports { diff --git a/lib/u_lib/src/messaging/base.rs b/lib/u_lib/src/messaging/base.rs index 8ac1158..3cef82d 100644 --- a/lib/u_lib/src/messaging/base.rs +++ b/lib/u_lib/src/messaging/base.rs @@ -8,10 +8,7 @@ use uuid::Uuid; pub struct Moo<'cow, T: AsMsg + Clone>(pub Cow<'cow, T>); pub trait AsMsg: Clone + Serialize { - fn as_message<'m>(&'m self) -> BaseMessage<'m, Self> - where - Moo<'m, Self>: From<&'m Self>, - { + fn as_message(&self) -> BaseMessage<'_, Self> { BaseMessage::new(self) } } diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index 8893977..052f02b 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -84,11 +84,7 @@ impl AssignedJob { if let Some(ref payload) = meta.payload { let extracted_payload = match TempFile::write_exec(payload) { Ok(p) => p, - Err(e) => { - return Reportable::Error( - UError::Runtime(e.to_string()).into_bt().to_string(), - ) - } + Err(e) => return Reportable::Error(UError::Runtime(e.to_string()).to_string()), }; ( meta.argv.replace("{}", &extracted_payload.get_path()), diff --git a/lib/u_lib/src/utils/combined_result.rs b/lib/u_lib/src/utils/combined_result.rs index 4464c39..05b92d1 100644 --- a/lib/u_lib/src/utils/combined_result.rs +++ b/lib/u_lib/src/utils/combined_result.rs @@ -1,7 +1,7 @@ use crate::utils::OneOrVec; -use crate::UErrorBt; +use crate::UError; -pub struct CombinedResult { +pub struct CombinedResult { ok: Vec, err: Vec, } From c70cdbd26212d49b286aabfc29cf140b8464b180 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Sat, 16 Jul 2022 18:11:51 +0500 Subject: [PATCH 21/30] big improvements (as usual) * removed api client macro * fixed passing --release mode in cargo make * optimized integration tests * added logging module (tracing) * allow u_panel to alter entries * reworked u_panel args (CRUD) * improved db hooks * started implementing web-interface ** incapsulated all frontend in binary ** setup workflow ** make u_panel accept commands from interface --- Cargo.toml | 1 - Makefile.toml | 45 +++-- bin/u_agent/Cargo.toml | 4 +- bin/u_agent/src/lib.rs | 34 ++-- bin/u_panel/Cargo.toml | 7 +- bin/u_panel/src/argparse.rs | 130 ++++++------- bin/u_panel/src/main.rs | 2 +- bin/u_panel/src/server/errors.rs | 17 ++ bin/u_panel/src/server/fe/.gitignore | 48 +++++ bin/u_panel/src/server/fe/angular.json | 2 + bin/u_panel/src/server/fe/package.json | 4 + .../src/server/fe/src/app/app.component.html | 50 ++++- .../src/server/fe/src/app/app.component.ts | 78 +++++++- .../src/server/fe/src/app/app.module.ts | 12 +- bin/u_panel/src/server/fe/src/index.html | 5 +- bin/u_panel/src/server/fe/src/styles.less | 3 + bin/u_panel/src/server/mod.rs | 46 ++++- bin/u_panel/src/tui/mod.rs | 2 +- bin/u_server/Cargo.toml | 3 +- bin/u_server/src/db.rs | 5 +- bin/u_server/src/handlers.rs | 89 +++++---- bin/u_server/src/init.rs | 136 ------------- bin/u_server/src/u_server.rs | 151 ++++++++++++++- .../integration-tests/tests_runner.Dockerfile | 2 +- integration/Cargo.toml | 3 +- integration/docker-compose.yml | 7 +- integration/integration_tests.sh | 1 + integration/tests/fixtures/agent.rs | 6 +- integration/tests/helpers/panel.rs | 2 +- lib/u_api_proc_macro/Cargo.toml | 16 -- lib/u_api_proc_macro/src/lib.rs | 181 ------------------ lib/u_api_proc_macro/tests/tests.rs | 15 -- lib/u_lib/Cargo.toml | 9 +- lib/u_lib/src/api.rs | 164 +++++++++------- lib/u_lib/src/builder.rs | 22 +-- lib/u_lib/src/config.rs | 6 +- lib/u_lib/src/errors/variants.rs | 11 +- lib/u_lib/src/lib.rs | 3 +- lib/u_lib/src/logging.rs | 16 ++ lib/u_lib/src/messaging/base.rs | 7 +- lib/u_lib/src/models/agent.rs | 7 +- lib/u_lib/src/models/jobs/assigned.rs | 4 +- lib/u_lib/src/models/jobs/meta.rs | 89 ++++++--- lib/u_lib/src/models/jobs/misc.rs | 3 +- lib/u_lib/src/utils/combined_result.rs | 8 +- scripts/deploy.sh | 2 +- scripts/gen_certs.sh | 1 + 47 files changed, 808 insertions(+), 651 deletions(-) create mode 100644 bin/u_panel/src/server/errors.rs create mode 100644 bin/u_panel/src/server/fe/.gitignore delete mode 100644 bin/u_server/src/init.rs delete mode 100644 lib/u_api_proc_macro/Cargo.toml delete mode 100644 lib/u_api_proc_macro/src/lib.rs delete mode 100644 lib/u_api_proc_macro/tests/tests.rs create mode 100644 lib/u_lib/src/logging.rs diff --git a/Cargo.toml b/Cargo.toml index a9f8f02..02c9d71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,6 @@ members = [ "bin/u_run", "bin/u_server", "lib/u_lib", - "lib/u_api_proc_macro", "integration" ] diff --git a/Makefile.toml b/Makefile.toml index e70a9dc..c5fb8e0 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -1,3 +1,21 @@ +# i need to preserve --release in args, not to pass cargo make -p release +# due to cargo failing to parse "" argument +env_scripts = [''' +#!@duckscript +args = array ${1} ${2} ${3} ${4} ${5} ${6} ${7} +set_env PROFILE_OVERRIDE debug + +for arg in ${args} + e = eq ${arg} "--release" + if ${e} + set_env PROFILE_OVERRIDE release + end +end + +profile = get_env PROFILE_OVERRIDE +echo PROFILE_OVERRIDE=${profile} +'''] + [config] default_to_workspace = false @@ -11,6 +29,7 @@ PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = "${STATIC_PREFIX}/bin/pg_config" OPENSSL_STATIC = "true" OPENSSL_DIR = "${STATIC_PREFIX}" + [tasks.build_static_libs] script = "./scripts/build_musl_libs.sh" @@ -30,28 +49,21 @@ command = "${CARGO}" args = ["build", "--target", "${TARGET}", "${@}"] [tasks.release_tasks] +condition = { env = { "PROFILE_OVERRIDE" = "release"} } script = ''' -if [[ "${@}" =~ "--release" ]]; then - echo "Creating symlink to release dir..." - ln -s ${ROOTDIR}/target/${TARGET}/release ${ROOTDIR}/release || true - BINS=$(ls ./release/u_* -1 | grep -v ".d") - echo "Stripping..." - strip $BINS - echo "Packing..." - upx -9 $BINS -fi +BINS=$(ls ./target/${TARGET}/${PROFILE_OVERRIDE}/u_* -1 | grep -v ".d") +echo "Stripping..." +strip $BINS +echo "Packing..." +upx -9 $BINS ''' [tasks.build] dependencies = ["cargo_build", "release_tasks"] -command = "true" -args = [] +clear = true [tasks.run] -script = ''' -echo "wtf are you running? run binaries dud!" -exit 1 -''' +disabled = true [tasks.unit] command = "${CARGO}" @@ -59,6 +71,7 @@ args = ["test", "--target", "${TARGET}", "--lib", "--", "${@}"] [tasks.integration] script = ''' +echo "!!! This task doesn't perform project rebuild, trigger it manually if need" cd ./integration bash integration_tests.sh ${@} ''' @@ -78,4 +91,4 @@ docker run --rm \ dependencies = ["unit", "integration"] [tasks.deploy] -script = './scripts/deploy.sh' +script = './scripts/deploy.sh' \ No newline at end of file diff --git a/bin/u_agent/Cargo.toml b/bin/u_agent/Cargo.toml index 500fdb0..e06bc51 100644 --- a/bin/u_agent/Cargo.toml +++ b/bin/u_agent/Cargo.toml @@ -2,7 +2,7 @@ name = "u_agent" version = "0.1.0" authors = ["plazmoid "] -edition = "2018" +edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -10,11 +10,11 @@ edition = "2018" tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process", "time"] } sysinfo = "0.10.5" log = "^0.4" -env_logger = "0.8.3" uuid = "0.6.5" reqwest = { version = "0.11", features = ["json"] } openssl = "*" u_lib = { version = "*", path = "../../lib/u_lib" } +daemonize = "0.4.1" [build-dependencies] openssl = "*" diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 581fd7f..286b3ac 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -5,15 +5,15 @@ #[macro_use] extern crate log; -extern crate env_logger; +use daemonize::Daemonize; use std::panic; use std::sync::Arc; use tokio::time::{sleep, Duration}; use u_lib::{ - api::ClientHandler, builder::JobBuilder, cache::JobCache, errors::ErrChan, - executor::pop_completed, messaging::Reportable, models::AssignedJob, utils::load_env_default, - UError, UID, + api::ClientHandler, builder::JobBuilder, cache::JobCache, config::get_self_uid, + errors::ErrChan, executor::pop_completed, logging::init_logger, messaging::Reportable, + models::AssignedJob, utils::load_env_default, UError, }; const ITERATION_LATENCY: u64 = 5; @@ -70,10 +70,9 @@ async fn error_reporting(client: Arc) -> ! { async fn do_stuff(client: Arc) -> ! { loop { - match client.get_personal_jobs(Some(*UID)).await { + match client.get_personal_jobs(Some(get_self_uid())).await { Ok(resp) => { - let job_requests = resp.into_builtin_vec(); - process_request(job_requests, &client).await; + process_request(resp, &client).await; } Err(err) => ErrChan::send(err), } @@ -88,13 +87,26 @@ async fn do_stuff(client: Arc) -> ! { } pub async fn run_forever() -> ! { - //daemonize(); - env_logger::init(); - let env = load_env_default().unwrap(); - let client = Arc::new(ClientHandler::new(&env.u_server)); panic::set_hook(Box::new(|panic_info| { ErrChan::send(UError::Panic(panic_info.to_string())) })); + if cfg!(debug_assertions) { + init_logger(format!( + "u_agent-{}", + get_self_uid() + .hyphenated() + .to_string() + .split("-") + .next() + .unwrap() + )); + } else { + if let Err(e) = Daemonize::new().start() { + ErrChan::send(UError::Runtime(e.to_string())) + } + } + let env = load_env_default().unwrap(); + let client = Arc::new(ClientHandler::new(&env.u_server, None)); tokio::spawn(error_reporting(client.clone())); do_stuff(client).await } diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 6facac5..21456d0 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -2,7 +2,7 @@ name = "u_panel" version = "0.1.0" authors = ["plazmoid "] -edition = "2018" +edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -11,12 +11,11 @@ actix-web = "3.3.2" backtrace = "0.3.61" structopt = "0.3.21" log = "^0.4" -env_logger = "0.7.1" uuid = "0.6.5" serde_json = "1.0.4" serde = { version = "1.0.114", features = ["derive"] } tokio = { version = "1.11.0", features = ["rt", "rt-multi-thread"] } -u_lib = { version = "*", path = "../../lib/u_lib" } +u_lib = { version = "*", path = "../../lib/u_lib", features = ["panel"] } tui = { version = "0.16", default-features = false, features = ['crossterm'] } crossterm = "0.22.1" anyhow = "1.0.44" @@ -30,3 +29,5 @@ signal-hook = "0.3.12" tracing-appender = "0.2.0" rust-embed = { version = "6.3.0", features = ["debug-embed", "compression"] } mime_guess = "2.0.4" +shlex = "1.1.0" +thiserror = "1.0.31" diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index c9f8c0f..371cb77 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -1,7 +1,10 @@ -use std::fmt; use structopt::StructOpt; use u_lib::{ - api::ClientHandler, datatypes::PanelResult, messaging::AsMsg, models::JobMeta, UError, UResult, + api::ClientHandler, + datatypes::PanelResult, + messaging::AsMsg, + models::{Agent, AssignedJob, JobMeta, RawJobMeta}, + UError, UResult, }; use uuid::Uuid; @@ -9,15 +12,13 @@ use uuid::Uuid; pub struct Args { #[structopt(subcommand)] cmd: Cmd, - #[structopt(long)] - json: bool, } #[derive(StructOpt, Debug)] enum Cmd { - Agents(LD), - Jobs(JobALD), - Map(JobMapALD), + Agents(RUD), + Jobs(JobCRUD), + Map(JobMapCRUD), //TUI(TUIArgs), Serve, } @@ -29,18 +30,12 @@ pub struct TUIArgs { } #[derive(StructOpt, Debug)] -enum JobALD { - Add { - //#[structopt(long, parse(try_from_str = parse_uuid))] - //agent: Option, - #[structopt(long)] - alias: String, - - #[structopt(subcommand)] - cmd: JobCmd, +enum JobCRUD { + Create { + job: String, }, #[structopt(flatten)] - LD(LD), + RUD(RUD), } #[derive(StructOpt, Debug)] @@ -50,29 +45,26 @@ enum JobCmd { } #[derive(StructOpt, Debug)] -enum JobMapALD { - Add { +enum JobMapCRUD { + Create { #[structopt(parse(try_from_str = parse_uuid))] agent_uid: Uuid, job_idents: Vec, }, - List { - #[structopt(parse(try_from_str = parse_uuid))] - uid: Option, - }, - Delete { - #[structopt(parse(try_from_str = parse_uuid))] - uid: Uuid, - }, + #[structopt(flatten)] + RUD(RUD), } #[derive(StructOpt, Debug)] -enum LD { - List { +enum RUD { + Read { #[structopt(parse(try_from_str = parse_uuid))] uid: Option, }, + Update { + item: String, + }, Delete { #[structopt(parse(try_from_str = parse_uuid))] uid: Uuid, @@ -83,61 +75,55 @@ fn parse_uuid(src: &str) -> Result { Uuid::parse_str(src).map_err(|e| e.to_string()) } -pub async fn process_cmd(client: ClientHandler, args: Args) -> UResult<()> { - struct Printer { - json: bool, - } - - impl Printer { - pub fn print(&self, data: UResult) { - if self.json { - let data = match data { - Ok(r) => PanelResult::Ok(r), - Err(e) => PanelResult::Err(e), - }; - println!("{}", serde_json::to_string_pretty(&data).unwrap()); - } else { - match data { - Ok(r) => println!("{}", r), - Err(e) => eprintln!("Error: {}", e), - } - } - } +pub async fn process_cmd(client: ClientHandler, args: Args) -> UResult { + fn to_json(data: UResult) -> String { + let data = match data { + Ok(r) => PanelResult::Ok(r), + Err(e) => PanelResult::Err(e), + }; + serde_json::to_string(&data).unwrap() } - let printer = Printer { json: args.json }; - match args.cmd { + Ok(match args.cmd { Cmd::Agents(action) => match action { - LD::List { uid } => printer.print(client.get_agents(uid).await), - LD::Delete { uid } => printer.print(client.del(Some(uid)).await), + RUD::Read { uid } => to_json(client.get_agents(uid).await), + RUD::Update { item } => { + let agent = serde_json::from_str::(&item)?; + to_json(client.update_item(agent).await) + } + RUD::Delete { uid } => to_json(client.del(uid).await), }, Cmd::Jobs(action) => match action { - JobALD::Add { - cmd: JobCmd::Cmd(cmd), - alias, - .. - } => { - let job = JobMeta::builder() - .with_shell(cmd.join(" ")) - .with_alias(alias) - .build()?; - printer.print(client.upload_jobs(&[job]).await); + JobCRUD::Create { job } => { + let raw_job = serde_json::from_str::(&job)?; + let job = raw_job.into_builder().build()?; + to_json(client.upload_jobs(&[job]).await) + } + JobCRUD::RUD(RUD::Read { uid }) => to_json(client.get_jobs(uid).await), + JobCRUD::RUD(RUD::Update { item }) => { + let job = serde_json::from_str::(&item)?; + to_json(client.update_item(job).await) } - JobALD::LD(LD::List { uid }) => printer.print(client.get_jobs(uid).await), - JobALD::LD(LD::Delete { uid }) => printer.print(client.del(Some(uid)).await), + JobCRUD::RUD(RUD::Delete { uid }) => to_json(client.del(uid).await), }, Cmd::Map(action) => match action { - JobMapALD::Add { + JobMapCRUD::Create { agent_uid, job_idents, - } => printer.print(client.set_jobs(Some(agent_uid), &job_idents).await), - JobMapALD::List { uid } => printer.print(client.get_agent_jobs(uid).await), - JobMapALD::Delete { uid } => printer.print(client.del(Some(uid)).await), + } => to_json(client.set_jobs(agent_uid, &job_idents).await), + JobMapCRUD::RUD(RUD::Read { uid }) => to_json(client.get_agent_jobs(uid).await), + JobMapCRUD::RUD(RUD::Update { item }) => { + let assigned = serde_json::from_str::(&item)?; + to_json(client.update_item(assigned).await) + } + JobMapCRUD::RUD(RUD::Delete { uid }) => to_json(client.del(uid).await), }, /*Cmd::TUI(args) => crate::tui::init_tui(&args) .await .map_err(|e| UError::PanelError(e.to_string()))?,*/ - Cmd::Serve => crate::server::serve().map_err(|e| UError::PanelError(e.to_string()))?, - } - Ok(()) + Cmd::Serve => { + crate::server::serve(client).map_err(|e| UError::PanelError(e.to_string()))?; + String::new() + } + }) } diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index 2f0d685..413a0b7 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -22,7 +22,7 @@ struct AccessEnv { #[tokio::main] async fn main() -> AnyResult<()> { let env = load_env::()?; - let client = ClientHandler::new(&env.u_server).password(env.admin_auth_token); + let client = ClientHandler::new(&env.u_server, Some(env.admin_auth_token)); let args = Args::from_args(); process_cmd(client, args).await?; diff --git a/bin/u_panel/src/server/errors.rs b/bin/u_panel/src/server/errors.rs new file mode 100644 index 0000000..003ff2f --- /dev/null +++ b/bin/u_panel/src/server/errors.rs @@ -0,0 +1,17 @@ +use actix_web::http::StatusCode; +use actix_web::ResponseError; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Arg parse error: {0}")] + ArgparseError(#[from] structopt::clap::Error), + + #[error("Just an error: {0}")] + JustError(String), +} + +impl ResponseError for Error { + fn status_code(&self) -> actix_web::http::StatusCode { + StatusCode::BAD_REQUEST + } +} diff --git a/bin/u_panel/src/server/fe/.gitignore b/bin/u_panel/src/server/fe/.gitignore new file mode 100644 index 0000000..e5c1ba1 --- /dev/null +++ b/bin/u_panel/src/server/fe/.gitignore @@ -0,0 +1,48 @@ +# See http://help.github.com/ignore-files/ for more about ignoring files. + +# compiled output +/dist +/tmp +/out-tsc +# Only exists if Bazel was run +/bazel-out + +# dependencies +/node_modules + +# profiling files +chrome-profiler-events*.json + +# IDEs and editors +/.idea +.project +.classpath +.c9/ +*.launch +.settings/ +*.sublime-workspace + +# IDE - VSCode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +.history/* + +# misc +/.angular/cache +/.sass-cache +/connect.lock +/coverage +/libpeerconnection.log +npm-debug.log +yarn-error.log +testem.log +/typings + +# System Files +.DS_Store +Thumbs.db + +package-lock.json \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/angular.json b/bin/u_panel/src/server/fe/angular.json index 5bb4463..3972326 100644 --- a/bin/u_panel/src/server/fe/angular.json +++ b/bin/u_panel/src/server/fe/angular.json @@ -31,6 +31,7 @@ "src/assets" ], "styles": [ + "./node_modules/@angular/material/prebuilt-themes/deeppurple-amber.css", "src/styles.less" ], "scripts": [] @@ -99,6 +100,7 @@ "src/assets" ], "styles": [ + "./node_modules/@angular/material/prebuilt-themes/deeppurple-amber.css", "src/styles.less" ], "scripts": [] diff --git a/bin/u_panel/src/server/fe/package.json b/bin/u_panel/src/server/fe/package.json index ae1f9de..9cc9000 100644 --- a/bin/u_panel/src/server/fe/package.json +++ b/bin/u_panel/src/server/fe/package.json @@ -11,15 +11,19 @@ "private": true, "dependencies": { "@angular/animations": "~13.1.0", + "@angular/cdk": "^13.3.9", "@angular/common": "~13.1.0", "@angular/compiler": "~13.1.0", "@angular/core": "~13.1.0", "@angular/forms": "~13.1.0", + "@angular/material": "^13.3.9", "@angular/platform-browser": "~13.1.0", "@angular/platform-browser-dynamic": "~13.1.0", "@angular/router": "~13.1.0", + "@types/uuid": "^8.3.4", "rxjs": "~7.4.0", "tslib": "^2.3.0", + "uuid": "^8.3.2", "zone.js": "~0.11.4" }, "devDependencies": { diff --git a/bin/u_panel/src/server/fe/src/app/app.component.html b/bin/u_panel/src/server/fe/src/app/app.component.html index 171ac03..a323767 100644 --- a/bin/u_panel/src/server/fe/src/app/app.component.html +++ b/bin/u_panel/src/server/fe/src/app/app.component.html @@ -1 +1,49 @@ -{{ title }} \ No newline at end of file + + +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
id{{row.id}}Alias{{row.alias}}user@hostname{{row.username}}@{{row.hostname}} + Last active + {{row.last_active}}
+ +
+ + +
+ + + + + \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/app.component.ts b/bin/u_panel/src/server/fe/src/app/app.component.ts index 25a2bce..76deda5 100644 --- a/bin/u_panel/src/server/fe/src/app/app.component.ts +++ b/bin/u_panel/src/server/fe/src/app/app.component.ts @@ -1,10 +1,82 @@ -import { Component } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { Component, ViewChild, AfterViewInit } from '@angular/core'; +import { timer, Observable, of as observableOf } from 'rxjs'; +import { catchError, map, startWith, switchMap } from 'rxjs/operators'; + +interface Agent { + alias: string | null, + hostname: string, + id: string, + is_root: boolean, + is_root_allowed: boolean, + last_active: Date, + platform: string, + regtime: Date, + state: "new" | "active" | "banned", + token: string | null, + username: string, +} @Component({ selector: 'app-root', templateUrl: './app.component.html', styleUrls: ['./app.component.less'] }) -export class AppComponent { - title = 'ты лох'; +export class AppComponent implements AfterViewInit { + displayedColumns: string[] = ['id', 'alias', 'username', 'last_active']; + exampleDatabase!: ExampleHttpDatabase | null; + + table_data: Agent[] = []; + isLoadingResults = true; + + constructor(private _httpClient: HttpClient) { } + + ngAfterViewInit() { + this.exampleDatabase = new ExampleHttpDatabase(this._httpClient); + this.fetch_agents(); + // If the user changes the sort order, reset back to the first page. + //this.sort.sortChange.subscribe(() => (this.paginator.pageIndex = 0)); + + } + + fetch_agents() { + timer(0) + .pipe( + startWith({}), + switchMap(() => { + this.isLoadingResults = true; + return this.exampleDatabase!.getAgents().pipe(catchError(() => observableOf(null))); + }), + map(data => { + // Flip flag to show that loading has finished. + this.isLoadingResults = false; + + if (data === null) { + return []; + } + + // Only refresh the result length if there is new data. In case of rate + // limit errors, we do not want to reset the paginator to zero, as that + // would prevent users from re-triggering requests. + return data.data; + }), + ) + .subscribe(data => { if (typeof data !== 'string') { this.table_data = data } else { alert(`Error: ${data}`) } }); + } +} + +interface ServerResponse { + status: "ok" | "err", + data: T | string +} + +class ExampleHttpDatabase { + constructor(private _httpClient: HttpClient) { } + + getAgents(): Observable> { + const requestUrl = "/cmd/"; + const cmd = "agents list"; + + return this._httpClient.post>(requestUrl, cmd); + } } diff --git a/bin/u_panel/src/server/fe/src/app/app.module.ts b/bin/u_panel/src/server/fe/src/app/app.module.ts index b1c6c96..56e6d5c 100644 --- a/bin/u_panel/src/server/fe/src/app/app.module.ts +++ b/bin/u_panel/src/server/fe/src/app/app.module.ts @@ -3,6 +3,11 @@ import { BrowserModule } from '@angular/platform-browser'; import { AppRoutingModule } from './app-routing.module'; import { AppComponent } from './app.component'; +import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; +import { MatTabsModule } from '@angular/material/tabs'; +import { MatTableModule } from '@angular/material/table'; +import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; +import { HttpClientModule } from '@angular/common/http'; @NgModule({ declarations: [ @@ -10,7 +15,12 @@ import { AppComponent } from './app.component'; ], imports: [ BrowserModule, - AppRoutingModule + HttpClientModule, + AppRoutingModule, + MatTabsModule, + MatTableModule, + MatProgressSpinnerModule, + BrowserAnimationsModule ], providers: [], bootstrap: [AppComponent] diff --git a/bin/u_panel/src/server/fe/src/index.html b/bin/u_panel/src/server/fe/src/index.html index 04af3e3..94a6658 100644 --- a/bin/u_panel/src/server/fe/src/index.html +++ b/bin/u_panel/src/server/fe/src/index.html @@ -6,8 +6,11 @@ + + + - + diff --git a/bin/u_panel/src/server/fe/src/styles.less b/bin/u_panel/src/server/fe/src/styles.less index 90d4ee0..7e7239a 100644 --- a/bin/u_panel/src/server/fe/src/styles.less +++ b/bin/u_panel/src/server/fe/src/styles.less @@ -1 +1,4 @@ /* You can add global styles to this file, and also import other style files */ + +html, body { height: 100%; } +body { margin: 0; font-family: Roboto, "Helvetica Neue", sans-serif; } diff --git a/bin/u_panel/src/server/mod.rs b/bin/u_panel/src/server/mod.rs index aaebf5a..f5a5c30 100644 --- a/bin/u_panel/src/server/mod.rs +++ b/bin/u_panel/src/server/mod.rs @@ -11,10 +11,15 @@ almost all fields are editable, rows are deletable */ -use actix_web::{get, web, App, HttpResponse, HttpServer, Responder}; +mod errors; + +use crate::{process_cmd, Args}; +use actix_web::{get, middleware::Logger, post, web, App, HttpResponse, HttpServer, Responder}; +use errors::Error; use rust_embed::RustEmbed; use std::borrow::Cow; -use u_lib::unwrap_enum; +use structopt::StructOpt; +use u_lib::{api::ClientHandler, logging::init_logger, unwrap_enum}; #[derive(RustEmbed)] #[folder = "./src/server/fe/dist/fe/"] @@ -45,12 +50,37 @@ async fn static_files_adapter(path: web::Path<(String,)>) -> impl Responder { } } +#[post("/cmd/")] +async fn send_cmd( + cmd: web::Json, + client: web::Data, +) -> Result { + let parsed_cmd = Args::from_iter_safe( + shlex::split(&cmd.into_inner()).ok_or(Error::JustError("shlex failed".to_string()))?, + )?; + Ok( + match process_cmd(client.as_ref().clone(), parsed_cmd).await { + Ok(r) => HttpResponse::Ok().body(r), + Err(e) => HttpResponse::BadRequest().body(e.to_string()), + }, + ) +} + #[actix_web::main] -pub async fn serve() -> std::io::Result<()> { +pub async fn serve(client: ClientHandler) -> std::io::Result<()> { + init_logger("u_panel"); + let addr = "127.0.0.1:8080"; - println!("Serving at http://{}", addr); - HttpServer::new(|| App::new().service(main_page).service(static_files_adapter)) - .bind(addr)? - .run() - .await + info!("Serving at http://{}", addr); + HttpServer::new(move || { + App::new() + .wrap(Logger::default()) + .app_data(web::Data::new(client.clone())) + .service(main_page) + .service(send_cmd) + .service(static_files_adapter) + }) + .bind(addr)? + .run() + .await } diff --git a/bin/u_panel/src/tui/mod.rs b/bin/u_panel/src/tui/mod.rs index 80b0a08..3a86c84 100644 --- a/bin/u_panel/src/tui/mod.rs +++ b/bin/u_panel/src/tui/mod.rs @@ -148,7 +148,7 @@ fn init_signal_handlers(gui: bool) { fn init_logger() { use tracing_appender::rolling::{RollingFileAppender, Rotation}; - use tracing_subscriber::EnvFilter; + use tracing_subscriber::EnvFlter; if env::var("RUST_LOG").is_err() { env::set_var("RUST_LOG", "info") diff --git a/bin/u_server/Cargo.toml b/bin/u_server/Cargo.toml index 13d12ca..41e4693 100644 --- a/bin/u_server/Cargo.toml +++ b/bin/u_server/Cargo.toml @@ -1,12 +1,11 @@ [package] authors = ["plazmoid "] -edition = "2018" +edition = "2021" name = "u_server" version = "0.1.0" [dependencies] log = "0.4.11" -simplelog = "0.10" thiserror = "*" warp = { version = "0.3.1", features = ["tls"] } uuid = { version = "0.6.5", features = ["serde", "v4"] } diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index db064ec..2c77133 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -31,8 +31,9 @@ impl UDB { "postgres://{}:{}@{}/{}", env.db_user, env.db_password, env.db_host, env.db_name ); - let conn = PgConnection::establish(&db_url).unwrap(); - let instance = UDB { conn }; + let instance = UDB { + conn: PgConnection::establish(&db_url).unwrap(), + }; Arc::new(Mutex::new(instance)) }) .lock() diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index d1573cc..efcb78a 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -7,50 +7,32 @@ use u_lib::{ utils::{OneOrVec, Stripped}, }; use uuid::Uuid; -use warp::{ - http::{Response, StatusCode}, - Rejection, Reply, -}; -/* -pub fn build_response(code: StatusCode, body: impl Into) -> Response { - Response::builder().status(code).body(body.into()).unwrap() -} - -pub fn build_ok(body: impl Into) -> Response { - build_response(StatusCode::OK, body) -} +use warp::Rejection; -pub fn build_err(body: impl ToString) -> Response { - build_response(StatusCode::BAD_REQUEST, body.to_string()) -} - -pub fn build_message(m: M) -> Response { - warp::reply::json(&m.as_message()).into_response() -} -*/ +type EndpResult = Result; pub struct Endpoints; impl Endpoints { - pub async fn add_agent(msg: Agent) -> Result<(), Rejection> { + pub async fn add_agent(msg: Agent) -> EndpResult<()> { UDB::lock_db().insert_agent(&msg).map_err(From::from) } - pub async fn get_agents(uid: Option) -> Result, Rejection> { + pub async fn get_agents(uid: Option) -> EndpResult> { UDB::lock_db().get_agents(uid).map_err(From::from) } - pub async fn get_jobs(uid: Option) -> Result, Rejection> { + pub async fn get_jobs(uid: Option) -> EndpResult> { UDB::lock_db().get_jobs(uid).map_err(From::from) } - pub async fn get_agent_jobs(uid: Option) -> Result, Rejection> { + pub async fn get_agent_jobs(uid: Option) -> EndpResult> { UDB::lock_db() .get_exact_jobs(uid, false) .map_err(From::from) } - pub async fn get_personal_jobs(uid: Option) -> Result, Rejection> { + pub async fn get_personal_jobs(uid: Option) -> EndpResult> { let agents = UDB::lock_db().get_agents(uid)?; if agents.is_empty() { let db = UDB::lock_db(); @@ -58,30 +40,26 @@ impl Endpoints { let job = db.find_job_by_alias("agent_hello")?; db.set_jobs_for_agent(&uid.unwrap(), &[job.id])?; } - let result = UDB::lock_db().get_exact_jobs(uid, true); - match result { - Ok(r) => { - let db = UDB::lock_db(); - for j in r.iter() { - db.update_job_status(j.id, JobState::Running)?; - } - Ok(r) - } - Err(e) => Err(e.into()), + let result = UDB::lock_db().get_exact_jobs(uid, true)?; + + let db = UDB::lock_db(); + for j in result.iter() { + db.update_job_status(j.id, JobState::Running)?; } + Ok(result) } - pub async fn upload_jobs(msg: BaseMessage<'static, Vec>) -> Result<(), Rejection> { + pub async fn upload_jobs(msg: BaseMessage<'static, Vec>) -> EndpResult<()> { UDB::lock_db() .insert_jobs(&msg.into_inner()) .map_err(From::from) } - pub async fn del(uid: Uuid) -> Result { + pub async fn del(uid: Uuid) -> EndpResult { let db = UDB::lock_db(); let del_fns = &[UDB::del_agents, UDB::del_jobs, UDB::del_results]; for del_fn in del_fns { - let affected = del_fn(&db, &[uid]).unwrap(); + let affected = del_fn(&db, &[uid])?; if affected > 0 { return Ok(affected); } @@ -92,7 +70,7 @@ impl Endpoints { pub async fn set_jobs( agent_uid: Uuid, msg: BaseMessage<'static, Vec>, - ) -> Result, Rejection> { + ) -> EndpResult> { msg.into_inner() .into_iter() .map(|ident| { @@ -106,7 +84,7 @@ impl Endpoints { pub async fn report + AsMsg + 'static>( msg: BaseMessage<'static, Data>, - ) -> Result<(), Rejection> { + ) -> EndpResult<()> { let id = msg.id; let mut failed = vec![]; for entry in msg.into_inner().into_vec() { @@ -134,7 +112,7 @@ impl Endpoints { err.agent_id, Stripped(&err.msg.as_str()) ); - UDB::lock_db().report_error(&err).unwrap(); + UDB::lock_db().report_error(&err)?; } Reportable::Dummy => (), } @@ -144,4 +122,33 @@ impl Endpoints { } Ok(()) } + + pub async fn update_agent(agent: BaseMessage<'static, Agent>) -> EndpResult<()> { + agent + .into_inner() + .save_changes::(&UDB::lock_db().conn) + .map_err(Error::from)?; + Ok(()) + } + + pub async fn update_job(job: BaseMessage<'static, JobMeta>) -> EndpResult<()> { + job.into_inner() + .save_changes::(&UDB::lock_db().conn) + .map_err(Error::from)?; + Ok(()) + } + + pub async fn update_assigned_job( + assigned: BaseMessage<'static, AssignedJob>, + ) -> EndpResult<()> { + assigned + .into_inner() + .save_changes::(&UDB::lock_db().conn) + .map_err(Error::from)?; + Ok(()) + } + + pub async fn download(_file_uid: String) -> EndpResult> { + todo!() + } } diff --git a/bin/u_server/src/init.rs b/bin/u_server/src/init.rs deleted file mode 100644 index 8127227..0000000 --- a/bin/u_server/src/init.rs +++ /dev/null @@ -1,136 +0,0 @@ -use crate::handlers::Endpoints; -use crate::{db::UDB, errors::SResult}; -use serde::de::DeserializeOwned; -use std::path::PathBuf; -use u_lib::{ - messaging::{AsMsg, BaseMessage, Reportable}, - models::*, -}; -use uuid::Uuid; -use warp::{ - body, - reply::{json, reply, Json}, - Filter, Rejection, Reply, -}; - -fn get_content() -> impl Filter,), Error = Rejection> + Clone -where - M: AsMsg + Sync + Send + DeserializeOwned + 'static, -{ - body::content_length_limit(1024 * 64).and(body::json::>()) -} - -fn into_message(msg: M) -> Json { - json(&msg.as_message()) -} - -pub fn init_filters( - auth_token: &str, -) -> impl Filter + Clone { - let infallible_none = |_| async { Ok::<(Option,), std::convert::Infallible>((None,)) }; - - let get_agents = warp::get() - .and(warp::path("get_agents")) - .and( - warp::path::param::() - .map(Some) - .or_else(infallible_none), - ) - .and_then(Endpoints::get_agents) - .map(into_message); - - let upload_jobs = warp::post() - .and(warp::path("upload_jobs")) - .and(get_content::>()) - .and_then(Endpoints::upload_jobs) - .map(|_| reply()); - - let get_jobs = warp::get() - .and(warp::path("get_jobs")) - .and( - warp::path::param::() - .map(Some) - .or_else(infallible_none), - ) - .and_then(Endpoints::get_jobs) - .map(into_message); - - let get_agent_jobs = warp::get() - .and(warp::path("get_agent_jobs")) - .and( - warp::path::param::() - .map(Some) - .or_else(infallible_none), - ) - .and_then(Endpoints::get_agent_jobs) - .map(into_message); - - let get_personal_jobs = warp::get() - .and(warp::path("get_personal_jobs")) - .and(warp::path::param::().map(Some)) - .and_then(Endpoints::get_personal_jobs) - .map(into_message); - - let del = warp::get() - .and(warp::path("del")) - .and(warp::path::param::()) - .and_then(Endpoints::del) - .map(|_| reply()); - - let set_jobs = warp::post() - .and(warp::path("set_jobs")) - .and(warp::path::param::()) - .and(get_content::>()) - .and_then(Endpoints::set_jobs) - .map(into_message); - - let report = warp::post() - .and(warp::path("report")) - .and(get_content::>()) - .and_then(Endpoints::report) - .map(|_| reply()); - - let auth_token = format!("Bearer {auth_token}",).into_boxed_str(); - let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); - - let auth_zone = (get_agents - .or(get_jobs) - .or(upload_jobs) - .or(del) - .or(set_jobs) - .or(get_agent_jobs)) - .and(auth_header); - - let agent_zone = get_jobs.clone().or(get_personal_jobs).or(report); - - auth_zone.or(agent_zone) -} - -pub fn prefill_jobs() -> SResult<()> { - let agent_hello = JobMeta::builder() - .with_type(misc::JobType::Manage) - .with_alias("agent_hello") - .build() - .unwrap(); - UDB::lock_db().insert_jobs(&[agent_hello]) -} - -pub fn init_logger() { - use simplelog::*; - use std::fs::OpenOptions; - let log_cfg = ConfigBuilder::new() - .set_time_format_str("%x %X") - .set_time_to_local(true) - .build(); - let logfile = OpenOptions::new() - .append(true) - .create(true) - .open(PathBuf::from("logs").join("u_server.log")) - .unwrap(); - let level = LevelFilter::Info; - let loggers = vec![ - WriteLogger::new(level, log_cfg.clone(), logfile) as Box, - TermLogger::new(level, log_cfg, TerminalMode::Stderr, ColorChoice::Auto), - ]; - CombinedLogger::init(loggers).unwrap(); -} diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 1483acd..d7ec8b3 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -14,28 +14,159 @@ extern crate diesel; mod db; mod errors; mod handlers; -mod init; use errors::{Error, SResult}; -use init::*; -use serde::Deserialize; +use serde::{de::DeserializeOwned, Deserialize}; use std::path::PathBuf; -use u_lib::{config::MASTER_PORT, utils::load_env}; -use warp::Filter; +use u_lib::{ + config::MASTER_PORT, + logging::init_logger, + messaging::{AsMsg, BaseMessage, Reportable}, + models::*, + utils::load_env, +}; +use uuid::Uuid; +use warp::{ + body, + reply::{json, reply, Json}, + Filter, Rejection, Reply, +}; + +use crate::db::UDB; +use crate::handlers::Endpoints; #[derive(Deserialize)] struct ServEnv { admin_auth_token: String, } -//TODO: tracing-subscriber +fn get_content() -> impl Filter,), Error = Rejection> + Clone +where + M: AsMsg + Sync + Send + DeserializeOwned + 'static, +{ + body::content_length_limit(1024 * 64).and(body::json::>()) +} + +fn into_message(msg: M) -> Json { + json(&msg.as_message()) +} + +pub fn init_endpoints( + auth_token: &str, +) -> impl Filter + Clone { + let path = |p: &'static str| warp::post().and(warp::path(p)); + let infallible_none = |_| async { Ok::<(Option,), std::convert::Infallible>((None,)) }; + + let get_agents = path("get_agents") + .and( + warp::path::param::() + .map(Some) + .or_else(infallible_none), + ) + .and_then(Endpoints::get_agents) + .map(into_message); + + let upload_jobs = path("upload_jobs") + .and(get_content::>()) + .and_then(Endpoints::upload_jobs) + .map(ok); + + let get_jobs = path("get_jobs") + .and( + warp::path::param::() + .map(Some) + .or_else(infallible_none), + ) + .and_then(Endpoints::get_jobs) + .map(into_message); + + let get_agent_jobs = path("get_agent_jobs") + .and( + warp::path::param::() + .map(Some) + .or_else(infallible_none), + ) + .and_then(Endpoints::get_agent_jobs) + .map(into_message); + + let get_personal_jobs = path("get_personal_jobs") + .and(warp::path::param::().map(Some)) + .and_then(Endpoints::get_personal_jobs) + .map(into_message); + + let del = path("del") + .and(warp::path::param::()) + .and_then(Endpoints::del) + .map(ok); + + let set_jobs = path("set_jobs") + .and(warp::path::param::()) + .and(get_content::>()) + .and_then(Endpoints::set_jobs) + .map(into_message); + + let report = path("report") + .and(get_content::>()) + .and_then(Endpoints::report) + .map(ok); + + let update_agent = path("update_item") + .and(get_content::()) + .and_then(Endpoints::update_agent) + .map(ok); + + let update_job = path("update_item") + .and(get_content::()) + .and_then(Endpoints::update_job) + .map(ok); + + let update_assigned_job = path("update_item") + .and(get_content::()) + .and_then(Endpoints::update_assigned_job) + .map(ok); + + let download = path("download") + .and(warp::path::param::()) + .and_then(Endpoints::download) + .map(ok); + + let auth_token = format!("Bearer {auth_token}",).into_boxed_str(); + let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); + + let auth_zone = (get_agents + .or(get_jobs) + .or(upload_jobs) + .or(del) + .or(set_jobs) + .or(get_agent_jobs) + .or(update_agent) + .or(update_job) + .or(update_assigned_job) + .or(download)) + .and(auth_header); + + let agent_zone = get_jobs.or(get_personal_jobs).or(report).or(download); + + auth_zone.or(agent_zone) +} + +pub fn prefill_jobs() -> SResult<()> { + let agent_hello = RawJobMeta::builder() + .with_type(misc::JobType::Manage) + .with_alias("agent_hello") + .build() + .unwrap(); + UDB::lock_db().insert_jobs(&[agent_hello]) +} + pub async fn serve() -> SResult<()> { - init_logger(); + init_logger("u_server"); prefill_jobs()?; let env = load_env::().map_err(|e| Error::Other(e.to_string()))?; - let routes = init_filters(&env.admin_auth_token); + let routes = init_endpoints(&env.admin_auth_token); let certs_dir = PathBuf::from("certs"); + warp::serve(routes.with(warp::log("warp"))) .tls() .cert_path(certs_dir.join("server.crt")) @@ -46,6 +177,10 @@ pub async fn serve() -> SResult<()> { Ok(()) } +fn ok(_: T) -> impl Reply { + reply() +} + /* #[cfg(test)] mod tests { diff --git a/images/integration-tests/tests_runner.Dockerfile b/images/integration-tests/tests_runner.Dockerfile index 0a60d7a..4714c7d 100644 --- a/images/integration-tests/tests_runner.Dockerfile +++ b/images/integration-tests/tests_runner.Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.60 +FROM rust:1.62 RUN rustup target add x86_64-unknown-linux-musl CMD ["sleep", "3600"] \ No newline at end of file diff --git a/integration/Cargo.toml b/integration/Cargo.toml index a4ae852..5e3f6fd 100644 --- a/integration/Cargo.toml +++ b/integration/Cargo.toml @@ -2,14 +2,13 @@ name = "integration" version = "0.1.0" authors = ["plazmoid "] -edition = "2018" +edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process", "time"] } log = "^0.4" -env_logger = "0.8.3" uuid = { version = "0.6.5", features = ["serde", "v4"] } reqwest = { version = "0.11", features = ["json"] } serde_json = "1.0" diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml index 40122f3..bca0fdd 100644 --- a/integration/docker-compose.yml +++ b/integration/docker-compose.yml @@ -11,7 +11,7 @@ services: networks: - u_net volumes: - - ../release/u_server:/unki/u_server + - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_server:/unki/u_server - ../certs:/unki/certs - ../logs:/unki/logs:rw working_dir: /unki @@ -58,7 +58,7 @@ services: networks: - u_net volumes: - - ../release/u_agent:/u_agent + - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_agent:/u_agent command: /u_agent u_server env_file: - ../.env @@ -76,9 +76,8 @@ services: volumes: - ./:/tests/ - ../certs:/certs - - ../release/u_panel:/u_panel + - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_panel:/u_panel - ../lib/u_lib:/lib/u_lib - - ../lib/u_api_proc_macro:/lib/u_api_proc_macro working_dir: /tests/ depends_on: diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 97d06e8..b8efc7e 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -2,4 +2,5 @@ set -e export DOCKER_UID=$(id -u) export DOCKER_GID=$(id -g) +[[ "$@" =~ "--release" ]] && export PROFILE=release || export PROFILE=debug python integration_tests.py $@ diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs index 5188830..4160af7 100644 --- a/integration/tests/fixtures/agent.rs +++ b/integration/tests/fixtures/agent.rs @@ -8,14 +8,14 @@ pub struct RegisteredAgent { impl RegisteredAgent { pub async fn unregister(self) { - let cli = ClientHandler::new(&ENV.u_server); - cli.del(Some(self.uid)).await.unwrap(); + let cli = ClientHandler::new(&ENV.u_server, None); + cli.del(self.uid).await.unwrap(); } } #[fixture] pub async fn register_agent() -> RegisteredAgent { - let cli = ClientHandler::new(&ENV.u_server); + let cli = ClientHandler::new(&ENV.u_server, None); let agent_uid = Uuid::new_v4(); let resp = cli .get_personal_jobs(Some(agent_uid)) diff --git a/integration/tests/helpers/panel.rs b/integration/tests/helpers/panel.rs index 27d2639..526b028 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration/tests/helpers/panel.rs @@ -37,7 +37,7 @@ impl Panel { pub fn output( args: impl Into + Display, ) -> PanelResult { - eprintln!("EXEC >>> {PANEL_BINARY} {}", &args); + eprintln!(">>> {PANEL_BINARY} {}", &args); let splitted = shlex::split(args.into().as_ref()).unwrap(); let result = Self::output_argv( splitted diff --git a/lib/u_api_proc_macro/Cargo.toml b/lib/u_api_proc_macro/Cargo.toml deleted file mode 100644 index c0e8456..0000000 --- a/lib/u_api_proc_macro/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "u_api_proc_macro" -version = "0.1.0" -authors = ["plazmoid "] -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[lib] -proc-macro = true - -[dependencies] -syn = { version = "1.0", features = ["full", "extra-traits"] } -quote = "1.0" -strum = { version = "0.20", features = ["derive"] } -proc-macro2 = "1.0" \ No newline at end of file diff --git a/lib/u_api_proc_macro/src/lib.rs b/lib/u_api_proc_macro/src/lib.rs deleted file mode 100644 index a6d0e89..0000000 --- a/lib/u_api_proc_macro/src/lib.rs +++ /dev/null @@ -1,181 +0,0 @@ -use proc_macro::TokenStream; -use proc_macro2::{Ident, TokenStream as TokenStream2}; -use quote::quote; -use std::{collections::HashMap, str::FromStr}; -use strum::EnumString; -use syn::{ - parse_macro_input, punctuated::Punctuated, AttributeArgs, FnArg, ItemFn, Lit, NestedMeta, - ReturnType, Signature, Token, Type, -}; - -#[derive(EnumString, Debug)] -enum ReqMethod { - GET, - POST, -} - -#[derive(Debug)] -struct Endpoint { - method: ReqMethod, -} - -#[derive(Debug)] -struct FnArgs { - url_param: Option, - payload: Option, -} - -#[proc_macro_attribute] -pub fn api_route(args: TokenStream, item: TokenStream) -> TokenStream { - let args: AttributeArgs = parse_macro_input!(args); - let input: ItemFn = parse_macro_input!(item); - let Signature { - ident, - inputs, - generics, - output, - .. - } = input.sig; - let (impl_generics, _, _) = generics.split_for_impl(); - let FnArgs { url_param, payload } = parse_fn_args(inputs); - let Endpoint { method } = parse_attr_args(args); - let url_path = build_url_path(&ident, &url_param); - let return_ty = match output { - ReturnType::Type(_, ty) => quote!(#ty), - ReturnType::Default => quote!(()), - }; - let request = match method { - ReqMethod::GET => build_get(url_path), - ReqMethod::POST => build_post(url_path, &payload), - }; - let url_param = match url_param { - Some(p) => quote!(, param: #p), - None => TokenStream2::new(), - }; - let payload = match payload { - Some(p) => quote!(, payload: #p), - None => TokenStream2::new(), - }; - let q = quote! { - pub async fn #ident #impl_generics( - &self #url_param #payload - ) -> UResult<#return_ty> { - let request = { - #request - }; - let response = request.send().await?; - let content_len = response.content_length(); - let is_success = match response.error_for_status_ref() { - Ok(_) => Ok(()), - Err(e) => Err(UError::from(e)) - }; - let resp = response.text().await?; - let result = match is_success { - Ok(_) => { - serde_json::from_str::>(&resp) - .map(|msg| msg.into_inner()) - .or_else(|e| { - match content_len { - Some(0) => Ok(Default::default()), - _ => Err(UError::NetError(e.to_string(), resp.clone())) - } - }) - }, - Err(UError::NetError(err_src, _)) => Err( - UError::NetError( - err_src, - resp - ) - ), - _ => unreachable!() - }; - Ok(result?) - } - }; - q.into() -} - -fn parse_fn_args(raw: Punctuated) -> FnArgs { - let mut arg: HashMap = raw - .into_iter() - .filter_map(|arg| { - if let FnArg::Typed(argt) = arg { - let mut arg_name = String::new(); - // did you think I won't overplay you? won't destroy? - |arg_ident| -> TokenStream { - let q: TokenStream = quote!(#arg_ident).into(); - arg_name = parse_macro_input!(q as Ident).to_string(); - TokenStream::new() - }(argt.pat); - if &arg_name != "url_param" && &arg_name != "payload" { - panic!("Wrong arg name: {}", &arg_name) - } - let arg_type = *argt.ty; - Some((arg_name, arg_type)) - } else { - None - } - }) - .collect(); - FnArgs { - url_param: arg.remove("url_param"), - payload: arg.remove("payload"), - } -} - -fn build_get(url: TokenStream2) -> TokenStream2 { - quote! { - let request = self.build_get(#url); - request - } -} - -fn build_post(url: TokenStream2, payload: &Option) -> TokenStream2 { - let pld = match payload { - Some(_) => quote! { - .json(&payload.as_message()) - }, - None => TokenStream2::new(), - }; - quote! { - let request = self.build_post(#url); - request #pld - } -} - -fn build_url_path(path: &Ident, url_param: &Option) -> TokenStream2 { - let url_param = match url_param { - Some(_) => quote! { - + &opt_to_string(param) - }, - None => TokenStream2::new(), - }; - quote! { - &format!( - "{}/{}", - stringify!(#path), - String::new() #url_param - ) - } -} - -fn parse_attr_args(args: AttributeArgs) -> Endpoint { - let mut args = args.into_iter(); - let method = match args.next() { - Some(method) => match method { - NestedMeta::Lit(l) => { - if let Lit::Str(s) = l { - match ReqMethod::from_str(&s.value()) { - Ok(v) => v, - Err(_) => panic!("Unknown method"), - } - } else { - panic!("Method must be a str") - } - } - _ => panic!("Method must be on the first place"), - }, - None => panic!("Method required"), - }; - Endpoint { method } -} diff --git a/lib/u_api_proc_macro/tests/tests.rs b/lib/u_api_proc_macro/tests/tests.rs deleted file mode 100644 index 7c4b404..0000000 --- a/lib/u_api_proc_macro/tests/tests.rs +++ /dev/null @@ -1,15 +0,0 @@ -/* -use std::fmt::Display; -use u_api_proc_macro::api_route; - -type UResult = Result; - -struct ClientHandler; -struct Paths; - -#[test] -fn test1() { - #[api_route("GET", Uuid)] - fn list(url_param: T) {} -} -*/ diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index 972645e..88a2cc9 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -2,7 +2,7 @@ name = "u_lib" version = "0.1.0" authors = ["plazmoid "] -edition = "2018" +edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -16,17 +16,20 @@ lazy_static = "1.4.0" futures = "0.3.5" thiserror = "*" log = "*" -env_logger = "0.8.3" diesel-derive-enum = { version = "1", features = ["postgres"] } chrono = "0.4.19" strum = { version = "0.20", features = ["derive"] } once_cell = "1.7.2" shlex = "1.0.0" -u_api_proc_macro = { version = "*", path = "../u_api_proc_macro" } crossbeam = "0.8.1" diesel = { version = "1.4.5", features = ["postgres", "uuid"] } envy = "0.4.2" serde_json = "1.0.81" +tracing-subscriber = { version = "0.3.14", features = ["env-filter"] } +tracing-appender = "0.2.2" + +[features] +panel = [] [target.'cfg(not(target_arch = "wasm32"))'.dependencies] reqwest = { version = "0.11", features = ["json", "native-tls"] } diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 35af4d3..7bc0e6d 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -1,98 +1,134 @@ +use std::collections::HashMap; + use crate::{ config::MASTER_PORT, - messaging::{self, AsMsg, BaseMessage}, - models, - utils::{opt_to_string, VecDisplay}, + messaging::{self, AsMsg, BaseMessage, Empty}, + models::{self, Agent}, + utils::opt_to_string, UError, UResult, }; -use reqwest::{Certificate, Client, Identity, RequestBuilder, Url}; -use u_api_proc_macro::api_route; +use reqwest::{header::HeaderMap, Certificate, Client, Identity, Url}; +use serde::de::DeserializeOwned; use uuid::Uuid; const AGENT_IDENTITY: &[u8] = include_bytes!("../../../certs/alice.p12"); const ROOT_CA_CERT: &[u8] = include_bytes!("../../../certs/ca.crt"); +#[derive(Clone)] pub struct ClientHandler { base_url: Url, client: Client, - password: Option, } impl ClientHandler { - pub fn new(server: &str) -> Self { + pub fn new(server: &str, password: Option) -> Self { let identity = Identity::from_pkcs12_der(AGENT_IDENTITY, "").unwrap(); - let client = Client::builder() - .identity(identity) + let mut client = Client::builder().identity(identity); + if let Some(pwd) = password { + client = client.default_headers( + HeaderMap::try_from(&HashMap::from([( + "Authorization".to_string(), + format!("Bearer {pwd}"), + )])) + .unwrap(), + ) + } + let client = client .add_root_certificate(Certificate::from_pem(ROOT_CA_CERT).unwrap()) .build() .unwrap(); Self { client, base_url: Url::parse(&format!("https://{}:{}", server, MASTER_PORT)).unwrap(), - password: None, } } - pub fn password(mut self, password: String) -> ClientHandler { - self.password = Some(password); - self - } + async fn _req( + &self, + url: impl AsRef, + payload: P, + ) -> UResult { + let request = self + .client + .post(self.base_url.join(url.as_ref()).unwrap()) + .json(&payload.as_message()); - fn set_pwd(&self, rb: RequestBuilder) -> RequestBuilder { - match &self.password { - Some(p) => rb.bearer_auth(p), - None => rb, + let response = request.send().await?; + let is_success = match response.error_for_status_ref() { + Ok(_) => Ok(()), + Err(e) => Err(UError::from(e)), + }; + let resp = response.text().await?; + match is_success { + Ok(_) => serde_json::from_str::>(&resp) + .map(|msg| msg.into_inner()) + .or_else(|e| Err(UError::NetError(e.to_string(), resp.clone()))), + Err(UError::NetError(err, _)) => Err(UError::NetError(err, resp)), + _ => unreachable!(), } } - fn build_get(&self, url: &str) -> RequestBuilder { - let rb = self.client.get(self.base_url.join(url).unwrap()); - self.set_pwd(rb) + // get jobs for client + pub async fn get_personal_jobs( + &self, + url_param: Option, + ) -> UResult> { + self._req( + format!("get_personal_jobs/{}", opt_to_string(url_param)), + Empty, + ) + .await } - fn build_post(&self, url: &str) -> RequestBuilder { - let rb = self.client.post(self.base_url.join(url).unwrap()); - self.set_pwd(rb) - } - // - // get jobs for client - #[api_route("GET")] - async fn get_personal_jobs(&self, url_param: Option) -> VecDisplay {} - // // send something to server - #[api_route("POST")] - async fn report(&self, payload: &[messaging::Reportable]) -> messaging::Empty {} - // + pub async fn report(&self, payload: &[messaging::Reportable]) -> UResult { + self._req("report", payload).await + } + // download file - #[api_route("GET")] - async fn dl(&self, url_param: Option) -> Vec {} - // - // request download - #[api_route("POST")] - async fn dlr(&self, url_param: Option) -> messaging::DownloadInfo {} - - //##########// Admin area //##########// - /// client listing - #[api_route("GET")] - async fn get_agents(&self, url_param: Option) -> VecDisplay {} - // - // get all available jobs - #[api_route("GET")] - async fn get_jobs(&self, url_param: Option) -> VecDisplay {} - // - // create and upload job - #[api_route("POST")] - async fn upload_jobs(&self, payload: &[models::JobMeta]) -> messaging::Empty {} - // - // delete something - #[api_route("GET")] - async fn del(&self, url_param: Option) -> i32 {} - // - // set jobs for any client - #[api_route("POST")] - async fn set_jobs(&self, url_param: Option, payload: &[String]) -> VecDisplay {} - // - // get jobs for any client - #[api_route("GET")] - async fn get_agent_jobs(&self, url_param: Option) -> VecDisplay {} + pub async fn dl(&self, file: String) -> UResult> { + self._req(format!("dl/{file}"), Empty).await + } +} + +//##########// Admin area //##########// +#[cfg(feature = "panel")] +impl ClientHandler { + /// agent listing + pub async fn get_agents(&self, agent: Option) -> UResult> { + self._req(format!("get_agents/{}", opt_to_string(agent)), Empty) + .await + } + + /// update something + pub async fn update_item(&self, item: impl AsMsg) -> UResult { + self._req("update_item", item).await + } + + /// get all available jobs + pub async fn get_jobs(&self, job: Option) -> UResult> { + self._req(format!("get_jobs/{}", opt_to_string(job)), Empty) + .await + } + + /// create and upload job + pub async fn upload_jobs(&self, payload: &[models::JobMeta]) -> UResult { + self._req("upload_jobs", payload).await + } + + /// delete something + pub async fn del(&self, item: Uuid) -> UResult { + self._req(format!("del/{item}"), Empty).await + } + + /// set jobs for any agent + pub async fn set_jobs(&self, agent: Uuid, job_idents: &[String]) -> UResult> { + self._req(format!("set_jobs/{agent}"), job_idents).await + } + + /// get jobs for any agent + pub async fn get_agent_jobs(&self, agent: Option) -> UResult> { + self._req(format!("set_jobs/{}", opt_to_string(agent)), Empty) + .await + } } diff --git a/lib/u_lib/src/builder.rs b/lib/u_lib/src/builder.rs index c9efcb0..1c1f6ba 100644 --- a/lib/u_lib/src/builder.rs +++ b/lib/u_lib/src/builder.rs @@ -2,7 +2,7 @@ use crate::{ cache::JobCache, executor::{DynFut, Waiter}, messaging::Reportable, - models::{Agent, AssignedJob, JobMeta, JobType}, + models::{Agent, AssignedJob, JobMeta, JobType, RawJobMeta}, utils::{CombinedResult, OneOrVec}, UError, UResult, }; @@ -102,7 +102,7 @@ impl NamedJobBuilder { .into_vec() .into_iter() .filter_map( - |(alias, cmd)| match JobMeta::builder().with_shell(cmd).build() { + |(alias, cmd)| match RawJobMeta::builder().with_shell(cmd).build() { Ok(meta) => Some((alias, meta)), Err(e) => { result.err(e); @@ -164,7 +164,7 @@ mod tests { #[tokio::test] async fn test_is_really_async() { const SLEEP_SECS: u64 = 1; - let job = JobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); + let job = RawJobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); let sleep_jobs: Vec = (0..50).map(|_| job.clone()).collect(); let now = SystemTime::now(); JobBuilder::from_meta(sleep_jobs).unwrap_one().wait().await; @@ -201,7 +201,7 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] expected_result: &str, ) -> TestResult { - let mut job = JobMeta::builder().with_shell(cmd); + let mut job = RawJobMeta::builder().with_shell(cmd); if let Some(p) = payload { job = job.with_payload(p); } @@ -217,12 +217,12 @@ mod tests { async fn test_complex_load() -> TestResult { const SLEEP_SECS: u64 = 1; let now = SystemTime::now(); - let longest_job = JobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); + let longest_job = RawJobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); let longest_job = JobBuilder::from_meta(longest_job) .unwrap_one() .spawn() .await; - let ls = JobBuilder::from_meta(JobMeta::from_shell("ls")?) + let ls = JobBuilder::from_meta(RawJobMeta::from_shell("ls")?) .unwrap_one() .wait_one() .await; @@ -231,7 +231,7 @@ mod tests { let folders = ls.to_string_result(); let subfolders_jobs: Vec = folders .lines() - .map(|f| JobMeta::from_shell(format!("ls {}", f)).unwrap()) + .map(|f| RawJobMeta::from_shell(format!("ls {}", f)).unwrap()) .collect(); let ls_subfolders = JobBuilder::from_meta(subfolders_jobs) .unwrap_one() @@ -265,7 +265,7 @@ mod tests { */ #[tokio::test] async fn test_failing_shell_job() -> TestResult { - let job = JobMeta::from_shell("lol_kek_puk")?; + let job = RawJobMeta::from_shell("lol_kek_puk")?; let job_result = JobBuilder::from_meta(job).unwrap_one().wait_one().await; let job_result = unwrap_enum!(job_result, Reportable::Assigned); let output = job_result.to_string_result(); @@ -283,7 +283,7 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] err_str: &str, ) -> TestResult { - let mut job = JobMeta::builder().with_shell(cmd); + let mut job = RawJobMeta::builder().with_shell(cmd); if let Some(p) = payload { job = job.with_payload(p); } @@ -296,10 +296,10 @@ mod tests { #[tokio::test] async fn test_different_job_types() -> TestResult { let mut jobs = NamedJobBuilder::from_meta(vec![ - ("sleeper", JobMeta::from_shell("sleep 3")?), + ("sleeper", RawJobMeta::from_shell("sleep 3")?), ( "gatherer", - JobMeta::builder().with_type(JobType::Manage).build()?, + RawJobMeta::builder().with_type(JobType::Manage).build()?, ), ]) .wait() diff --git a/lib/u_lib/src/config.rs b/lib/u_lib/src/config.rs index e9d649a..50d3408 100644 --- a/lib/u_lib/src/config.rs +++ b/lib/u_lib/src/config.rs @@ -4,5 +4,9 @@ use uuid::Uuid; pub const MASTER_PORT: u16 = 63714; lazy_static! { - pub static ref UID: Uuid = Uuid::new_v4(); + static ref UID: Uuid = Uuid::new_v4(); +} + +pub fn get_self_uid() -> Uuid { + *UID } diff --git a/lib/u_lib/src/errors/variants.rs b/lib/u_lib/src/errors/variants.rs index 829d7ab..2a9a48f 100644 --- a/lib/u_lib/src/errors/variants.rs +++ b/lib/u_lib/src/errors/variants.rs @@ -32,7 +32,7 @@ pub enum UError { #[error("Job {0} doesn't exist")] NoJob(Uuid), - #[error("Error while processing {0}: {1}")] + #[error("FS error while processing {0}: {1}")] FSError(String, String), #[error("Wrong auth token")] @@ -43,6 +43,9 @@ pub enum UError { #[error("Panel error: {0}")] PanelError(String), + + #[error("Deserialize from json error: {0}")] + DeserializeError(String), } #[cfg(not(target_arch = "wasm32"))] @@ -51,3 +54,9 @@ impl From for UError { UError::NetError(e.to_string(), String::new()) } } + +impl From for UError { + fn from(e: serde_json::Error) -> Self { + UError::DeserializeError(e.to_string()) + } +} diff --git a/lib/u_lib/src/lib.rs b/lib/u_lib/src/lib.rs index 2b6fa07..41383a4 100644 --- a/lib/u_lib/src/lib.rs +++ b/lib/u_lib/src/lib.rs @@ -9,6 +9,7 @@ pub mod exports { pub mod datatypes; pub mod errors; pub mod executor; + pub mod logging; pub mod messaging; pub mod models; pub mod utils; @@ -24,7 +25,6 @@ pub mod exports { pub mod utils; } -pub use config::UID; pub use errors::{UError, UResult}; pub use exports::*; @@ -38,7 +38,6 @@ extern crate diesel; #[macro_use] extern crate log; -extern crate env_logger; #[cfg(test)] #[macro_use] diff --git a/lib/u_lib/src/logging.rs b/lib/u_lib/src/logging.rs new file mode 100644 index 0000000..cbd7dac --- /dev/null +++ b/lib/u_lib/src/logging.rs @@ -0,0 +1,16 @@ +use std::env; +use std::path::Path; + +use tracing_appender::rolling; +use tracing_subscriber::EnvFilter; + +pub fn init_logger(logfile: impl AsRef + Send + Sync + 'static) { + if env::var("RUST_LOG").is_err() { + env::set_var("RUST_LOG", "info") + } + + tracing_subscriber::fmt::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .with_writer(move || rolling::never(".", logfile.as_ref().with_extension("log"))) + .init(); +} diff --git a/lib/u_lib/src/messaging/base.rs b/lib/u_lib/src/messaging/base.rs index 3cef82d..06b73a6 100644 --- a/lib/u_lib/src/messaging/base.rs +++ b/lib/u_lib/src/messaging/base.rs @@ -1,5 +1,5 @@ +use crate::config::get_self_uid; use crate::utils::VecDisplay; -use crate::UID; use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::fmt::Display; @@ -43,7 +43,10 @@ impl<'cow, I: AsMsg> BaseMessage<'cow, I> { C: Into>, { let Moo(inner) = inner.into(); - Self { id: *UID, inner } + Self { + id: get_self_uid(), + inner, + } } pub fn into_inner(self) -> I { diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index 7c238a8..6843e0e 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -6,7 +6,10 @@ use strum::Display; #[cfg(not(target_arch = "wasm32"))] use crate::builder::NamedJobBuilder; -use crate::{messaging::Reportable, models::schema::*, unwrap_enum, utils::systime_to_string, UID}; +use crate::{ + config::get_self_uid, messaging::Reportable, models::schema::*, unwrap_enum, + utils::systime_to_string, +}; use uuid::Uuid; @@ -107,7 +110,7 @@ impl Default for Agent { fn default() -> Self { Self { alias: None, - id: *UID, + id: get_self_uid(), hostname: String::new(), is_root: false, is_root_allowed: false, diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index 052f02b..d2465b6 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -2,11 +2,11 @@ use super::JobState; #[cfg(not(target_arch = "wasm32"))] use crate::{cache::JobCache, utils::TempFile}; use crate::{ + config::get_self_uid, errors::UError, messaging::Reportable, models::schema::*, utils::{systime_to_string, ProcOutput}, - UID, }; use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; @@ -119,7 +119,7 @@ impl AssignedJob { pub fn new(job_id: Uuid, other: Option<&Self>) -> Self { Self { - agent_id: *UID, + agent_id: get_self_uid(), job_id, ..other.unwrap_or(&Default::default()).clone() } diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 939d256..6074c31 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -2,11 +2,14 @@ use super::JobType; use crate::{models::schema::*, utils::Stripped, UError, UResult}; use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; -use std::fmt; +use std::path::PathBuf; use std::str::from_utf8; +use std::{fmt, fs}; use uuid::Uuid; -#[derive(Serialize, Deserialize, Clone, Debug, Queryable, Identifiable, Insertable)] +#[derive( + Serialize, Deserialize, Clone, Debug, Queryable, Identifiable, Insertable, AsChangeset, +)] #[table_name = "jobs"] pub struct JobMeta { pub alias: Option, @@ -20,12 +23,28 @@ pub struct JobMeta { pub payload: Option>, } -impl JobMeta { +#[derive(Deserialize)] +pub struct RawJobMeta { + pub alias: Option, + pub argv: String, + pub id: Uuid, + pub exec_type: JobType, + //pub schedule: JobSchedule, + pub platform: String, + pub payload: Option>, + pub payload_path: Option, +} + +impl RawJobMeta { pub fn builder() -> JobMetaBuilder { JobMetaBuilder::default() } - pub fn from_shell(cmd: impl Into) -> UResult { + pub fn into_builder(self) -> JobMetaBuilder { + JobMetaBuilder { inner: self } + } + + pub fn from_shell(cmd: impl Into) -> UResult { Self::builder().with_shell(cmd).build() } } @@ -60,29 +79,35 @@ impl Default for JobMeta { alias: None, argv: String::new(), exec_type: JobType::Shell, - #[cfg(not(target_arch = "wasm32"))] platform: guess_host_triple::guess_host_triple() .unwrap_or("unknown") .to_string(), - #[cfg(target_arch = "wasm32")] - platform: "unknown".to_string(), payload: None, } } } -pub struct JobMetaBuilder { - inner: JobMeta, -} - -impl Default for JobMetaBuilder { +impl Default for RawJobMeta { fn default() -> Self { Self { - inner: JobMeta::default(), + id: Uuid::new_v4(), + alias: None, + argv: String::new(), + exec_type: JobType::Shell, + platform: guess_host_triple::guess_host_triple() + .unwrap_or("unknown") + .to_string(), + payload: None, + payload_path: None, } } } +#[derive(Default)] +pub struct JobMetaBuilder { + inner: RawJobMeta, +} + impl JobMetaBuilder { pub fn with_shell(mut self, shell_cmd: impl Into) -> Self { self.inner.argv = shell_cmd.into(); @@ -94,6 +119,11 @@ impl JobMetaBuilder { self } + pub fn with_payload_src(mut self, path: impl Into) -> Self { + self.inner.payload_path = Some(path.into()); + self + } + pub fn with_alias(mut self, alias: impl Into) -> Self { self.inner.alias = Some(alias.into()); self @@ -117,6 +147,12 @@ impl JobMetaBuilder { if argv_parts.get(0).ok_or(empty_err.clone())?.is_empty() { return Err(empty_err.into()); } + if let Some(path) = &inner.payload_path { + let data = fs::read(path.clone()).map_err(|e| { + UError::FSError(path.to_string_lossy().to_string(), e.to_string()) + })?; + inner.payload = Some(data) + } match inner.payload.as_ref() { Some(_) => { if !inner.argv.contains("{}") { @@ -136,20 +172,23 @@ impl JobMetaBuilder { } } }; - Ok(inner) + Ok(inner.into()) } - JobType::Manage => Ok(inner), + JobType::Manage => Ok(inner.into()), _ => todo!(), } } - /* - pub fn from_file(path: PathBuf) -> UResult { - let data = fs::read(path) - .map_err(|e| UError::FilesystemError( - path.to_string_lossy().to_string(), - e.to_string() - ))?; - let filename = path.file_name().unwrap().to_str().unwrap(); - - }*/ +} + +impl From for JobMeta { + fn from(rjm: RawJobMeta) -> Self { + JobMeta { + alias: rjm.alias, + argv: rjm.argv, + id: rjm.id, + exec_type: rjm.exec_type, + platform: rjm.platform, + payload: rjm.payload, + } + } } diff --git a/lib/u_lib/src/models/jobs/misc.rs b/lib/u_lib/src/models/jobs/misc.rs index ebcd38d..2eb4ce4 100644 --- a/lib/u_lib/src/models/jobs/misc.rs +++ b/lib/u_lib/src/models/jobs/misc.rs @@ -27,11 +27,12 @@ pub enum JobState { Finished, } -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, DbEnum, Display)] +#[derive(Default, Serialize, Deserialize, Clone, Debug, PartialEq, DbEnum, Display)] #[PgType = "JobType"] #[DieselType = "Jobtype"] pub enum JobType { Manage, + #[default] Shell, Python, } diff --git a/lib/u_lib/src/utils/combined_result.rs b/lib/u_lib/src/utils/combined_result.rs index 05b92d1..b9a7125 100644 --- a/lib/u_lib/src/utils/combined_result.rs +++ b/lib/u_lib/src/utils/combined_result.rs @@ -1,12 +1,14 @@ +use std::fmt::Debug; + use crate::utils::OneOrVec; use crate::UError; -pub struct CombinedResult { +pub struct CombinedResult { ok: Vec, err: Vec, } -impl CombinedResult { +impl CombinedResult { pub fn new() -> Self { Self { ok: vec![], @@ -25,7 +27,7 @@ impl CombinedResult { pub fn unwrap(self) -> Vec { let err_len = self.err.len(); if err_len > 0 { - panic!("CombinedResult has {} errors", err_len); + panic!("CombinedResult has errors: {:?}", self.err); } self.ok } diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 0804965..bb019c0 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -8,7 +8,7 @@ REMOTE_PATH=$SERVER:$REMOTE_DIR RSYNC="rsync -arzh --progress" ssh $SERVER mkdir -p $REMOTE_DIR/{release,deploy} -$RSYNC $ROOTDIR/release/u_server $REMOTE_PATH/release/u_server +$RSYNC $ROOTDIR/target/x86_64-unknown-linux-musl/release/u_server $REMOTE_PATH/release/u_server $RSYNC $ROOTDIR/certs/server.{crt,key} $REMOTE_PATH/certs $RSYNC $ROOTDIR/certs/ca.crt $REMOTE_PATH/certs $RSYNC $ROOTDIR/migrations/ $REMOTE_PATH/migrations diff --git a/scripts/gen_certs.sh b/scripts/gen_certs.sh index 3be58ea..af36f2c 100755 --- a/scripts/gen_certs.sh +++ b/scripts/gen_certs.sh @@ -13,6 +13,7 @@ subjectAltName = @alt_names [alt_names] DNS.1 = ortem.xyz DNS.2 = u_server +DNS.3 = localhost EOF openssl req -x509 -newkey rsa:4096 -keyout $DIR/ca.key -out $DIR/ca.crt -nodes -days 365 -subj "/CN=root" From a50e6d242fe7c5225a1fb7921a673158a2863f58 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Mon, 18 Jul 2022 04:24:53 +0500 Subject: [PATCH 22/30] small but useful fixes * optimized error handling * more usage of tracing-subscriber * fixed some integration test bugs * fixed db constraints --- bin/u_agent/src/lib.rs | 51 ++++++++-------- bin/u_panel/Cargo.toml | 1 - bin/u_panel/src/argparse.rs | 5 +- bin/u_panel/src/main.rs | 5 +- bin/u_panel/src/server/mod.rs | 2 +- bin/u_server/Cargo.toml | 2 +- bin/u_server/src/handlers.rs | 2 +- bin/u_server/src/main.rs | 9 ++- bin/u_server/src/u_server.rs | 6 +- integration/Cargo.toml | 3 +- integration/docker-compose.yml | 15 +++-- integration/tests/fixtures/agent.rs | 2 +- integration/tests/helpers/panel.rs | 6 +- integration/tests/integration/behaviour.rs | 20 +++--- lib/u_lib/Cargo.toml | 3 +- lib/u_lib/src/api.rs | 61 ++++++++++--------- lib/u_lib/src/builder.rs | 2 +- lib/u_lib/src/errors/chan.rs | 36 +++++++---- lib/u_lib/src/logging.rs | 23 +++++-- lib/u_lib/src/messaging/mod.rs | 2 +- lib/u_lib/src/models/jobs/meta.rs | 7 +++ lib/u_lib/src/utils/combined_result.rs | 13 ++-- lib/u_lib/src/utils/fmt/stripped.rs | 2 +- lib/u_lib/src/utils/proc_output.rs | 2 +- .../2020-10-24-111622_create_all/up.sql | 8 +-- 25 files changed, 170 insertions(+), 118 deletions(-) diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 286b3ac..a7405ab 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -7,7 +7,6 @@ extern crate log; use daemonize::Daemonize; -use std::panic; use std::sync::Arc; use tokio::time::{sleep, Duration}; use u_lib::{ @@ -22,7 +21,7 @@ pub async fn process_request(job_requests: Vec, client: &ClientHand if !job_requests.is_empty() { for jr in &job_requests { if !JobCache::contains(jr.job_id) { - debug!("Fetching job: {}", &jr.job_id); + info!("Fetching job: {}", &jr.job_id); let fetched_job = loop { match client.get_jobs(Some(jr.job_id)).await { Ok(mut result) => break result.pop().unwrap(), @@ -35,7 +34,7 @@ pub async fn process_request(job_requests: Vec, client: &ClientHand JobCache::insert(fetched_job); } } - debug!( + info!( "Scheduling jobs: {}", job_requests .iter() @@ -46,7 +45,9 @@ pub async fn process_request(job_requests: Vec, client: &ClientHand let mut builder = JobBuilder::from_request(job_requests); let errors = builder.pop_errors(); if !errors.is_empty() { - errors.into_iter().for_each(ErrChan::send) + for e in errors { + ErrChan::send(e, "ebld").await; + } } builder.unwrap_one().spawn().await; } @@ -54,32 +55,35 @@ pub async fn process_request(job_requests: Vec, client: &ClientHand async fn error_reporting(client: Arc) -> ! { loop { - let err = ErrChan::recv(); - debug!("Error encountered: {:?}", err); - 'retry: for _ in 0..3 { - match client.report(&[Reportable::Error(err.to_string())]).await { - Ok(_) => break 'retry, - Err(e) => { - debug!("Reporting error: {:?}", e); - sleep(Duration::from_secs(10)).await; + match ErrChan::recv().await { + Some(err) => { + 'retry: for _ in 0..3 { + match client.report(&[Reportable::Error(err.to_string())]).await { + Ok(_) => break 'retry, + Err(e) => { + debug!("Reporting error: {:?}", e); + sleep(Duration::from_secs(10)).await; + } + } } } + None => sleep(Duration::from_secs(3)).await, } } } async fn do_stuff(client: Arc) -> ! { loop { - match client.get_personal_jobs(Some(get_self_uid())).await { + match client.get_personal_jobs(get_self_uid()).await { Ok(resp) => { process_request(resp, &client).await; } - Err(err) => ErrChan::send(err), + Err(err) => ErrChan::send(err, "personal").await, } let result: Vec = pop_completed().await.into_iter().collect(); if !result.is_empty() { if let Err(err) = client.report(&result).await { - ErrChan::send(err) + ErrChan::send(err, "report").await; } } sleep(Duration::from_secs(ITERATION_LATENCY)).await; @@ -87,11 +91,12 @@ async fn do_stuff(client: Arc) -> ! { } pub async fn run_forever() -> ! { - panic::set_hook(Box::new(|panic_info| { - ErrChan::send(UError::Panic(panic_info.to_string())) - })); + let env = load_env_default().unwrap(); + let client = Arc::new(ClientHandler::new(&env.u_server, None)); + tokio::spawn(error_reporting(client.clone())); + if cfg!(debug_assertions) { - init_logger(format!( + init_logger(Some(format!( "u_agent-{}", get_self_uid() .hyphenated() @@ -99,14 +104,12 @@ pub async fn run_forever() -> ! { .split("-") .next() .unwrap() - )); + ))); } else { if let Err(e) = Daemonize::new().start() { - ErrChan::send(UError::Runtime(e.to_string())) + ErrChan::send(UError::Runtime(e.to_string()), "deeeemon").await } } - let env = load_env_default().unwrap(); - let client = Arc::new(ClientHandler::new(&env.u_server, None)); - tokio::spawn(error_reporting(client.clone())); + info!("Startup"); do_stuff(client).await } diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 21456d0..4ba454a 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -10,7 +10,6 @@ edition = "2021" actix-web = "3.3.2" backtrace = "0.3.61" structopt = "0.3.21" -log = "^0.4" uuid = "0.6.5" serde_json = "1.0.4" serde = { version = "1.0.114", features = ["derive"] } diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 371cb77..e89710b 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -1,3 +1,4 @@ +use anyhow::Result as AnyResult; use structopt::StructOpt; use u_lib::{ api::ClientHandler, @@ -76,10 +77,10 @@ fn parse_uuid(src: &str) -> Result { } pub async fn process_cmd(client: ClientHandler, args: Args) -> UResult { - fn to_json(data: UResult) -> String { + fn to_json(data: AnyResult) -> String { let data = match data { Ok(r) => PanelResult::Ok(r), - Err(e) => PanelResult::Err(e), + Err(e) => PanelResult::Err(e.downcast().expect("unknown error type")), }; serde_json::to_string(&data).unwrap() } diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index 413a0b7..0726a88 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -10,6 +10,7 @@ use argparse::{process_cmd, Args}; use serde::Deserialize; use structopt::StructOpt; use u_lib::api::ClientHandler; +use u_lib::logging::init_logger; use u_lib::utils::{env::default_host, load_env}; #[derive(Deserialize)] @@ -25,6 +26,8 @@ async fn main() -> AnyResult<()> { let client = ClientHandler::new(&env.u_server, Some(env.admin_auth_token)); let args = Args::from_args(); - process_cmd(client, args).await?; + init_logger(None::<&str>); + let result = process_cmd(client, args).await?; + println!("{result}"); Ok(()) } diff --git a/bin/u_panel/src/server/mod.rs b/bin/u_panel/src/server/mod.rs index f5a5c30..a427567 100644 --- a/bin/u_panel/src/server/mod.rs +++ b/bin/u_panel/src/server/mod.rs @@ -68,7 +68,7 @@ async fn send_cmd( #[actix_web::main] pub async fn serve(client: ClientHandler) -> std::io::Result<()> { - init_logger("u_panel"); + init_logger(Some("u_panel")); let addr = "127.0.0.1:8080"; info!("Serving at http://{}", addr); diff --git a/bin/u_server/Cargo.toml b/bin/u_server/Cargo.toml index 41e4693..d2e5f75 100644 --- a/bin/u_server/Cargo.toml +++ b/bin/u_server/Cargo.toml @@ -5,7 +5,7 @@ name = "u_server" version = "0.1.0" [dependencies] -log = "0.4.11" +tracing = "0.1.35" thiserror = "*" warp = { version = "0.3.1", features = ["tls"] } uuid = { version = "0.6.5", features = ["serde", "v4"] } diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index efcb78a..41c2411 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -112,7 +112,7 @@ impl Endpoints { err.agent_id, Stripped(&err.msg.as_str()) ); - UDB::lock_db().report_error(&err)?; + //UDB::lock_db().report_error(&err)?; } Reportable::Dummy => (), } diff --git a/bin/u_server/src/main.rs b/bin/u_server/src/main.rs index 84d2139..77877ce 100644 --- a/bin/u_server/src/main.rs +++ b/bin/u_server/src/main.rs @@ -1,6 +1,11 @@ use u_server_lib::serve; +#[macro_use] +extern crate tracing; + #[tokio::main] -async fn main() -> Result<(), String> { - serve().await.map_err(|e| e.to_string()) +async fn main() { + if let Err(e) = serve().await { + error!("U_SERVER error: {}", e); + } } diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index d7ec8b3..406159c 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -1,5 +1,5 @@ #[macro_use] -extern crate log; +extern crate tracing; #[cfg(test)] #[macro_use] @@ -131,7 +131,7 @@ pub fn init_endpoints( .map(ok); let auth_token = format!("Bearer {auth_token}",).into_boxed_str(); - let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); + let auth_header = warp::header::exact("Authorization", Box::leak(auth_token)); let auth_zone = (get_agents .or(get_jobs) @@ -160,7 +160,7 @@ pub fn prefill_jobs() -> SResult<()> { } pub async fn serve() -> SResult<()> { - init_logger("u_server"); + init_logger(Some("u_server")); prefill_jobs()?; let env = load_env::().map_err(|e| Error::Other(e.to_string()))?; diff --git a/integration/Cargo.toml b/integration/Cargo.toml index 5e3f6fd..eda6095 100644 --- a/integration/Cargo.toml +++ b/integration/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" [dependencies] tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process", "time"] } -log = "^0.4" +tracing = "0.1.35" uuid = { version = "0.6.5", features = ["serde", "v4"] } reqwest = { version = "0.11", features = ["json"] } serde_json = "1.0" @@ -20,6 +20,7 @@ once_cell = "1.10.0" [dependencies.u_lib] path = "../lib/u_lib" version = "*" +features = ["panel"] [[test]] diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml index bca0fdd..04bc87b 100644 --- a/integration/docker-compose.yml +++ b/integration/docker-compose.yml @@ -25,7 +25,7 @@ services: - ../.env - ../.env.private environment: - RUST_LOG: trace + RUST_LOG: info healthcheck: test: ss -tlpn | grep 63714 interval: 5s @@ -36,8 +36,8 @@ services: image: unki/u_db networks: - u_net - expose: - - '5432' + ports: + - 54321:5432 env_file: - ../.env - ../.env.private @@ -58,12 +58,14 @@ services: networks: - u_net volumes: - - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_agent:/u_agent - command: /u_agent u_server + - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_agent:/unki/u_agent + - ../logs:/unki/logs:rw + working_dir: /unki + command: /unki/u_agent u_server env_file: - ../.env environment: - RUST_LOG: u_agent=debug + RUST_LOG: u_agent=debug,u_lib=debug depends_on: u_server: condition: service_healthy @@ -78,6 +80,7 @@ services: - ../certs:/certs - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_panel:/u_panel - ../lib/u_lib:/lib/u_lib + - ../logs:/tests/logs:rw working_dir: /tests/ depends_on: diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs index 4160af7..b80366d 100644 --- a/integration/tests/fixtures/agent.rs +++ b/integration/tests/fixtures/agent.rs @@ -18,7 +18,7 @@ pub async fn register_agent() -> RegisteredAgent { let cli = ClientHandler::new(&ENV.u_server, None); let agent_uid = Uuid::new_v4(); let resp = cli - .get_personal_jobs(Some(agent_uid)) + .get_personal_jobs(agent_uid) .await .unwrap() .pop() diff --git a/integration/tests/helpers/panel.rs b/integration/tests/helpers/panel.rs index 526b028..d381898 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration/tests/helpers/panel.rs @@ -13,11 +13,7 @@ pub struct Panel; impl Panel { fn run(args: &[&str]) -> Output { - Command::new(PANEL_BINARY) - .arg("--json") - .args(args) - .output() - .unwrap() + Command::new(PANEL_BINARY).args(args).output().unwrap() } pub fn output_argv(argv: &[&str]) -> PanelResult { diff --git a/integration/tests/integration/behaviour.rs b/integration/tests/integration/behaviour.rs index 6ff1bf3..6472505 100644 --- a/integration/tests/integration/behaviour.rs +++ b/integration/tests/integration/behaviour.rs @@ -2,6 +2,7 @@ use crate::fixtures::agent::*; use crate::helpers::Panel; use rstest::rstest; +use serde_json::{json, to_string}; use std::error::Error; use std::time::Duration; use tokio::time::sleep; @@ -14,7 +15,7 @@ type TestResult = Result>; #[tokio::test] async fn test_registration(#[future] register_agent: RegisteredAgent) -> TestResult { let agent = register_agent.await; - let agents: Vec = Panel::check_output("agents list"); + let agents: Vec = Panel::check_output("agents read"); let found = agents.iter().find(|v| v.id == agent.uid); assert!(found.is_some()); Panel::check_status(format!("agents delete {}", agent.uid)); @@ -23,17 +24,22 @@ async fn test_registration(#[future] register_agent: RegisteredAgent) -> TestRes #[tokio::test] async fn test_setup_tasks() -> TestResult { - //some independent agents should present - let agents: Vec = Panel::check_output("agents list"); - let agent_uid = agents[0].id; + let agents: Vec = Panel::check_output("agents read"); + let agent_uid = match agents.get(0) { + Some(a) => a.id, + None => panic!("Some independent agents should present"), + }; let job_alias = "passwd_contents"; - let cmd = format!("jobs add --alias {job_alias} 'cat /etc/passwd'"); + let job = json!( + {"alias": job_alias, "payload": b"cat /etc/passwd" } + ); + let cmd = format!("jobs create '{}'", to_string(&job).unwrap()); Panel::check_status(cmd); - let cmd = format!("map add {} {}", agent_uid, job_alias); + let cmd = format!("map create {} {}", agent_uid, job_alias); let assigned_uids: Vec = Panel::check_output(cmd); for _ in 0..3 { let result: Vec = - Panel::check_output(format!("map list {}", assigned_uids[0])); + Panel::check_output(format!("map read {}", assigned_uids[0])); if result[0].state == JobState::Finished { return Ok(()); } else { diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index 88a2cc9..54f1c42 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -15,7 +15,6 @@ libc = "^0.2" lazy_static = "1.4.0" futures = "0.3.5" thiserror = "*" -log = "*" diesel-derive-enum = { version = "1", features = ["postgres"] } chrono = "0.4.19" strum = { version = "0.20", features = ["derive"] } @@ -27,6 +26,8 @@ envy = "0.4.2" serde_json = "1.0.81" tracing-subscriber = { version = "0.3.14", features = ["env-filter"] } tracing-appender = "0.2.2" +log = "*" +anyhow = "1.0.58" [features] panel = [] diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 7bc0e6d..f5c86fd 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -1,20 +1,23 @@ use std::collections::HashMap; +use std::fmt::Debug; use crate::{ config::MASTER_PORT, messaging::{self, AsMsg, BaseMessage, Empty}, - models::{self, Agent}, + models::{self}, utils::opt_to_string, - UError, UResult, + UError, }; +use anyhow::{Context, Result}; use reqwest::{header::HeaderMap, Certificate, Client, Identity, Url}; use serde::de::DeserializeOwned; +use serde_json::from_str; use uuid::Uuid; const AGENT_IDENTITY: &[u8] = include_bytes!("../../../certs/alice.p12"); const ROOT_CA_CERT: &[u8] = include_bytes!("../../../certs/ca.crt"); -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct ClientHandler { base_url: Url, client: Client, @@ -43,50 +46,50 @@ impl ClientHandler { } } - async fn _req( + async fn _req( &self, - url: impl AsRef, + url: impl AsRef + Debug, payload: P, - ) -> UResult { + ) -> Result { let request = self .client .post(self.base_url.join(url.as_ref()).unwrap()) .json(&payload.as_message()); - let response = request.send().await?; + let response = request.send().await.context("send")?; + let content_len = response.content_length(); let is_success = match response.error_for_status_ref() { Ok(_) => Ok(()), Err(e) => Err(UError::from(e)), }; - let resp = response.text().await?; + let resp = response.text().await.context("resp")?; + debug!("url = {}, resp = {}", url.as_ref(), resp); match is_success { - Ok(_) => serde_json::from_str::>(&resp) + Ok(_) => from_str::>(&resp) .map(|msg| msg.into_inner()) - .or_else(|e| Err(UError::NetError(e.to_string(), resp.clone()))), + .or_else(|e| match content_len { + Some(0) => Ok(Default::default()), + _ => Err(UError::NetError(e.to_string(), resp)), + }), Err(UError::NetError(err, _)) => Err(UError::NetError(err, resp)), _ => unreachable!(), } + .map_err(From::from) } // get jobs for client - pub async fn get_personal_jobs( - &self, - url_param: Option, - ) -> UResult> { - self._req( - format!("get_personal_jobs/{}", opt_to_string(url_param)), - Empty, - ) - .await + pub async fn get_personal_jobs(&self, url_param: Uuid) -> Result> { + self._req(format!("get_personal_jobs/{}", url_param), Empty) + .await } // send something to server - pub async fn report(&self, payload: &[messaging::Reportable]) -> UResult { + pub async fn report(&self, payload: &[messaging::Reportable]) -> Result { self._req("report", payload).await } // download file - pub async fn dl(&self, file: String) -> UResult> { + pub async fn dl(&self, file: String) -> Result> { self._req(format!("dl/{file}"), Empty).await } } @@ -95,40 +98,40 @@ impl ClientHandler { #[cfg(feature = "panel")] impl ClientHandler { /// agent listing - pub async fn get_agents(&self, agent: Option) -> UResult> { + pub async fn get_agents(&self, agent: Option) -> Result> { self._req(format!("get_agents/{}", opt_to_string(agent)), Empty) .await } /// update something - pub async fn update_item(&self, item: impl AsMsg) -> UResult { + pub async fn update_item(&self, item: impl AsMsg + Debug) -> Result { self._req("update_item", item).await } /// get all available jobs - pub async fn get_jobs(&self, job: Option) -> UResult> { + pub async fn get_jobs(&self, job: Option) -> Result> { self._req(format!("get_jobs/{}", opt_to_string(job)), Empty) .await } /// create and upload job - pub async fn upload_jobs(&self, payload: &[models::JobMeta]) -> UResult { + pub async fn upload_jobs(&self, payload: &[models::JobMeta]) -> Result { self._req("upload_jobs", payload).await } /// delete something - pub async fn del(&self, item: Uuid) -> UResult { + pub async fn del(&self, item: Uuid) -> Result { self._req(format!("del/{item}"), Empty).await } /// set jobs for any agent - pub async fn set_jobs(&self, agent: Uuid, job_idents: &[String]) -> UResult> { + pub async fn set_jobs(&self, agent: Uuid, job_idents: &[String]) -> Result> { self._req(format!("set_jobs/{agent}"), job_idents).await } /// get jobs for any agent - pub async fn get_agent_jobs(&self, agent: Option) -> UResult> { - self._req(format!("set_jobs/{}", opt_to_string(agent)), Empty) + pub async fn get_agent_jobs(&self, agent: Option) -> Result> { + self._req(format!("get_personal_jobs/{}", opt_to_string(agent)), Empty) .await } } diff --git a/lib/u_lib/src/builder.rs b/lib/u_lib/src/builder.rs index 1c1f6ba..01b949d 100644 --- a/lib/u_lib/src/builder.rs +++ b/lib/u_lib/src/builder.rs @@ -17,7 +17,7 @@ impl JobBuilder { pub fn from_request(job_requests: impl OneOrVec) -> CombinedResult { let job_requests = job_requests.into_vec(); let mut prepared: Vec = vec![]; - let mut result = CombinedResult::::new(); + let mut result = CombinedResult::::new(); for req in job_requests { let job_meta = JobCache::get(req.job_id); if job_meta.is_none() { diff --git a/lib/u_lib/src/errors/chan.rs b/lib/u_lib/src/errors/chan.rs index 1c2b995..10668a4 100644 --- a/lib/u_lib/src/errors/chan.rs +++ b/lib/u_lib/src/errors/chan.rs @@ -1,9 +1,10 @@ -use crate::UError; -use crossbeam::channel::{self, Receiver, Sender}; +use anyhow::Error; use once_cell::sync::OnceCell; +use tokio::sync::mpsc::{channel, error::TryRecvError, Receiver, Sender}; +use tokio::sync::{Mutex, MutexGuard}; -type ChanError = UError; -static ERR_CHAN: OnceCell = OnceCell::new(); +type ChanError = Error; +static ERR_CHAN: OnceCell> = OnceCell::new(); pub struct ErrChan { tx: Sender, @@ -11,18 +12,27 @@ pub struct ErrChan { } impl ErrChan { - fn get() -> &'static Self { - ERR_CHAN.get_or_init(|| { - let (tx, rx) = channel::bounded(20); - Self { tx, rx } - }) + async fn get() -> MutexGuard<'static, Self> { + ERR_CHAN + .get_or_init(|| { + let (tx, rx) = channel(20); + Mutex::new(Self { tx, rx }) + }) + .lock() + .await } - pub fn send(msg: ChanError) { - Self::get().tx.send(msg).unwrap() + pub async fn send(err: impl Into, ctx: impl AsRef) { + let err = err.into(); + error!("Encountered an error at '{}': {:?}", ctx.as_ref(), err); + Self::get().await.tx.try_send(err).unwrap(); } - pub fn recv() -> ChanError { - Self::get().rx.recv().unwrap() + pub async fn recv() -> Option { + match Self::get().await.rx.try_recv() { + Ok(r) => Some(r), + Err(TryRecvError::Disconnected) => panic!("err chan disconnected"), + Err(TryRecvError::Empty) => None, + } } } diff --git a/lib/u_lib/src/logging.rs b/lib/u_lib/src/logging.rs index cbd7dac..7c4b991 100644 --- a/lib/u_lib/src/logging.rs +++ b/lib/u_lib/src/logging.rs @@ -2,15 +2,26 @@ use std::env; use std::path::Path; use tracing_appender::rolling; -use tracing_subscriber::EnvFilter; +use tracing_subscriber::{fmt, prelude::*, registry, EnvFilter}; -pub fn init_logger(logfile: impl AsRef + Send + Sync + 'static) { +pub fn init_logger(logfile: Option + Send + Sync + 'static>) { if env::var("RUST_LOG").is_err() { env::set_var("RUST_LOG", "info") } - tracing_subscriber::fmt::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .with_writer(move || rolling::never(".", logfile.as_ref().with_extension("log"))) - .init(); + let reg = registry() + .with(EnvFilter::from_default_env()) + .with(fmt::layer()); + match logfile { + Some(file) => reg + .with( + fmt::layer() + .with_writer(move || { + rolling::never("logs", file.as_ref().with_extension("log")) + }) + .with_ansi(false), + ) + .init(), + None => reg.init(), + }; } diff --git a/lib/u_lib/src/messaging/mod.rs b/lib/u_lib/src/messaging/mod.rs index 64485ae..e79ef34 100644 --- a/lib/u_lib/src/messaging/mod.rs +++ b/lib/u_lib/src/messaging/mod.rs @@ -28,7 +28,7 @@ impl fmt::Display for Empty { } } -#[derive(Serialize, Deserialize, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Clone, PartialEq, Debug)] pub enum Reportable { Assigned(AssignedJob), Agent(Agent), diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 6074c31..657be72 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -25,13 +25,20 @@ pub struct JobMeta { #[derive(Deserialize)] pub struct RawJobMeta { + #[serde(default)] pub alias: Option, + #[serde(default)] pub argv: String, + #[serde(default)] pub id: Uuid, + #[serde(default)] pub exec_type: JobType, //pub schedule: JobSchedule, + #[serde(default)] pub platform: String, + #[serde(default)] pub payload: Option>, + #[serde(default)] pub payload_path: Option, } diff --git a/lib/u_lib/src/utils/combined_result.rs b/lib/u_lib/src/utils/combined_result.rs index b9a7125..12dd4d6 100644 --- a/lib/u_lib/src/utils/combined_result.rs +++ b/lib/u_lib/src/utils/combined_result.rs @@ -1,9 +1,9 @@ use std::fmt::Debug; use crate::utils::OneOrVec; -use crate::UError; +use anyhow::Error; -pub struct CombinedResult { +pub struct CombinedResult { ok: Vec, err: Vec, } @@ -20,8 +20,13 @@ impl CombinedResult { self.ok.extend(result.into_vec()); } - pub fn err(&mut self, err: impl OneOrVec) { - self.err.extend(err.into_vec()); + pub fn err>(&mut self, err: impl OneOrVec) { + self.err.extend( + err.into_vec() + .into_iter() + .map(Into::into) + .collect::>(), + ); } pub fn unwrap(self) -> Vec { diff --git a/lib/u_lib/src/utils/fmt/stripped.rs b/lib/u_lib/src/utils/fmt/stripped.rs index b1ed949..21d0032 100644 --- a/lib/u_lib/src/utils/fmt/stripped.rs +++ b/lib/u_lib/src/utils/fmt/stripped.rs @@ -3,7 +3,7 @@ use std::iter::Iterator; use std::slice::Iter as SliceIter; use std::str::Chars; -const MAX_DATA_LEN: usize = 200; +const MAX_DATA_LEN: usize = 2000; pub trait Strippable { type Item: fmt::Display; diff --git a/lib/u_lib/src/utils/proc_output.rs b/lib/u_lib/src/utils/proc_output.rs index a9c4135..d060e89 100644 --- a/lib/u_lib/src/utils/proc_output.rs +++ b/lib/u_lib/src/utils/proc_output.rs @@ -108,7 +108,7 @@ impl ProcOutput { altered = true; } if !altered { - result.extend(b"No data"); + result.extend(b""); } result } diff --git a/migrations/2020-10-24-111622_create_all/up.sql b/migrations/2020-10-24-111622_create_all/up.sql index ec2949f..d319d65 100644 --- a/migrations/2020-10-24-111622_create_all/up.sql +++ b/migrations/2020-10-24-111622_create_all/up.sql @@ -4,7 +4,7 @@ CREATE TYPE JobState AS ENUM ('queued', 'running', 'finished'); CREATE TYPE AgentState AS ENUM ('new', 'active', 'banned'); CREATE TABLE IF NOT EXISTS agents ( - alias TEXT UNIQUE + alias TEXT , hostname TEXT NOT NULL , id UUID NOT NULL DEFAULT uuid_generate_v4() , is_root BOOLEAN NOT NULL DEFAULT false @@ -34,11 +34,9 @@ CREATE TABLE IF NOT EXISTS ip_addrs ( ); CREATE TABLE IF NOT EXISTS jobs ( - alias TEXT UNIQUE + alias TEXT , argv TEXT NOT NULL , id UUID NOT NULL DEFAULT uuid_generate_v4() - -- Shell, Binary (with program download), - -- Python (with program and python download if not exist), Management , exec_type JobType NOT NULL DEFAULT 'shell' , platform TEXT NOT NULL , payload BYTEA @@ -47,7 +45,7 @@ CREATE TABLE IF NOT EXISTS jobs ( CREATE TABLE IF NOT EXISTS results ( agent_id UUID NOT NULL - , alias TEXT UNIQUE + , alias TEXT , created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , id UUID NOT NULL DEFAULT uuid_generate_v4() , job_id UUID NOT NULL From c25fa780bfaece7198119f59c221edbd04244073 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Tue, 26 Jul 2022 23:41:06 +0500 Subject: [PATCH 23/30] wow such web * temporarily disabled u_run * clean unused deps a bit * disable daemonizing in release mode because wtf * use one async runtime in u_panel * fix cors issues * fix passing cmd from frontend * initial pretty web interface * remove storing errors in db * check target and payload platform * prepare for cross compile to windows binary --- Cargo.toml | 2 +- bin/u_agent/Cargo.toml | 4 - bin/u_agent/src/lib.rs | 12 +-- bin/u_panel/Cargo.toml | 4 +- bin/u_panel/src/argparse.rs | 4 +- bin/u_panel/src/main.rs | 2 +- .../src/server/fe/src/app/app.component.html | 52 ++----------- .../src/server/fe/src/app/app.component.ts | 75 +------------------ .../src/server/fe/src/app/app.module.ts | 13 +++- .../src/server/fe/src/app/core/index.ts | 1 + .../fe/src/app/core/models/agent.model.ts | 15 ++++ .../server/fe/src/app/core/models/index.ts | 8 ++ .../fe/src/app/core/models/job.model.ts | 8 ++ .../fe/src/app/core/models/result.model.ts | 13 ++++ .../fe/src/app/core/services/api.service.ts | 43 +++++++++++ .../server/fe/src/app/core/services/index.ts | 1 + .../fe/src/app/core/tables/agent.component.ts | 41 ++++++++++ .../server/fe/src/app/core/tables/index.ts | 3 + .../fe/src/app/core/tables/job.component.ts | 46 ++++++++++++ .../src/app/core/tables/result.component.ts | 46 ++++++++++++ .../src/app/core/tables/table.component.html | 36 +++++++++ .../src/app/core/tables/table.component.less | 24 ++++++ .../fe/src/app/core/tables/table.component.ts | 66 ++++++++++++++++ .../fe/src/environments/environment.prod.ts | 3 +- .../server/fe/src/environments/environment.ts | 3 +- bin/u_panel/src/server/mod.rs | 26 +++++-- bin/u_server/src/db.rs | 42 +---------- bin/u_server/src/handlers.rs | 18 ++--- bin/u_server/src/models.rs | 0 bin/u_server/src/u_server.rs | 21 ++++-- integration/tests/integration/connection.rs | 22 ++++++ integration/tests/integration/mod.rs | 1 + integration/tests/lib.rs | 23 ------ lib/u_lib/Cargo.toml | 1 + lib/u_lib/src/api.rs | 14 ++-- lib/u_lib/src/builder.rs | 10 +-- lib/u_lib/src/models/agent.rs | 16 ++-- lib/u_lib/src/models/error.rs | 35 --------- lib/u_lib/src/models/jobs/assigned.rs | 2 - lib/u_lib/src/models/jobs/meta.rs | 36 ++++++--- lib/u_lib/src/models/mod.rs | 3 +- lib/u_lib/src/utils/mod.rs | 2 + lib/u_lib/src/utils/platform.rs | 38 ++++++++++ lib/u_lib/src/utils/tempfile.rs | 12 ++- .../2020-10-24-111622_create_all/down.sql | 1 - .../2020-10-24-111622_create_all/up.sql | 10 --- 46 files changed, 550 insertions(+), 308 deletions(-) create mode 100644 bin/u_panel/src/server/fe/src/app/core/index.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/models/index.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/models/job.model.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/models/result.model.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/services/api.service.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/services/index.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/index.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/table.component.html create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/table.component.less create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts create mode 100644 bin/u_server/src/models.rs create mode 100644 integration/tests/integration/connection.rs delete mode 100644 lib/u_lib/src/models/error.rs create mode 100644 lib/u_lib/src/utils/platform.rs diff --git a/Cargo.toml b/Cargo.toml index 02c9d71..228808c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ members = [ "bin/u_agent", "bin/u_panel", - "bin/u_run", + #"bin/u_run", "bin/u_server", "lib/u_lib", "integration" diff --git a/bin/u_agent/Cargo.toml b/bin/u_agent/Cargo.toml index e06bc51..a7cfb91 100644 --- a/bin/u_agent/Cargo.toml +++ b/bin/u_agent/Cargo.toml @@ -12,9 +12,5 @@ sysinfo = "0.10.5" log = "^0.4" uuid = "0.6.5" reqwest = { version = "0.11", features = ["json"] } -openssl = "*" u_lib = { version = "*", path = "../../lib/u_lib" } -daemonize = "0.4.1" -[build-dependencies] -openssl = "*" diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index a7405ab..09e016f 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -6,7 +6,7 @@ #[macro_use] extern crate log; -use daemonize::Daemonize; +//use daemonize::Daemonize; use std::sync::Arc; use tokio::time::{sleep, Duration}; use u_lib::{ @@ -78,7 +78,7 @@ async fn do_stuff(client: Arc) -> ! { Ok(resp) => { process_request(resp, &client).await; } - Err(err) => ErrChan::send(err, "personal").await, + Err(err) => ErrChan::send(err, "processing").await, } let result: Vec = pop_completed().await.into_iter().collect(); if !result.is_empty() { @@ -105,10 +105,10 @@ pub async fn run_forever() -> ! { .next() .unwrap() ))); - } else { - if let Err(e) = Daemonize::new().start() { - ErrChan::send(UError::Runtime(e.to_string()), "deeeemon").await - } + // } else { + // if let Err(e) = Daemonize::new().start() { + // ErrChan::send(UError::Runtime(e.to_string()), "deeeemon").await + // } } info!("Startup"); do_stuff(client).await diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 4ba454a..1ea2be9 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -actix-web = "3.3.2" +actix-web = "4.1" backtrace = "0.3.61" structopt = "0.3.21" uuid = "0.6.5" @@ -30,3 +30,5 @@ rust-embed = { version = "6.3.0", features = ["debug-embed", "compression"] } mime_guess = "2.0.4" shlex = "1.1.0" thiserror = "1.0.31" +futures-util = "0.3.21" +actix-cors = "0.6.1" diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index e89710b..b414fe7 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -123,7 +123,9 @@ pub async fn process_cmd(client: ClientHandler, args: Args) -> UResult { .await .map_err(|e| UError::PanelError(e.to_string()))?,*/ Cmd::Serve => { - crate::server::serve(client).map_err(|e| UError::PanelError(e.to_string()))?; + crate::server::serve(client) + .await + .map_err(|e| UError::PanelError(e.to_string()))?; String::new() } }) diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index 0726a88..ea263b8 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -20,7 +20,7 @@ struct AccessEnv { u_server: String, } -#[tokio::main] +#[actix_web::main] async fn main() -> AnyResult<()> { let env = load_env::()?; let client = ClientHandler::new(&env.u_server, Some(env.admin_auth_token)); diff --git a/bin/u_panel/src/server/fe/src/app/app.component.html b/bin/u_panel/src/server/fe/src/app/app.component.html index a323767..6aedf2e 100644 --- a/bin/u_panel/src/server/fe/src/app/app.component.html +++ b/bin/u_panel/src/server/fe/src/app/app.component.html @@ -1,49 +1,11 @@ -
-
- -
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - -
id{{row.id}}Alias{{row.alias}}user@hostname{{row.username}}@{{row.hostname}} - Last active - {{row.last_active}}
- -
- - -
- + + + + + + + - - \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/app.component.ts b/bin/u_panel/src/server/fe/src/app/app.component.ts index 76deda5..a76961e 100644 --- a/bin/u_panel/src/server/fe/src/app/app.component.ts +++ b/bin/u_panel/src/server/fe/src/app/app.component.ts @@ -1,82 +1,9 @@ -import { HttpClient } from '@angular/common/http'; import { Component, ViewChild, AfterViewInit } from '@angular/core'; -import { timer, Observable, of as observableOf } from 'rxjs'; -import { catchError, map, startWith, switchMap } from 'rxjs/operators'; - -interface Agent { - alias: string | null, - hostname: string, - id: string, - is_root: boolean, - is_root_allowed: boolean, - last_active: Date, - platform: string, - regtime: Date, - state: "new" | "active" | "banned", - token: string | null, - username: string, -} @Component({ selector: 'app-root', templateUrl: './app.component.html', styleUrls: ['./app.component.less'] }) -export class AppComponent implements AfterViewInit { - displayedColumns: string[] = ['id', 'alias', 'username', 'last_active']; - exampleDatabase!: ExampleHttpDatabase | null; - - table_data: Agent[] = []; - isLoadingResults = true; - - constructor(private _httpClient: HttpClient) { } - - ngAfterViewInit() { - this.exampleDatabase = new ExampleHttpDatabase(this._httpClient); - this.fetch_agents(); - // If the user changes the sort order, reset back to the first page. - //this.sort.sortChange.subscribe(() => (this.paginator.pageIndex = 0)); - - } - - fetch_agents() { - timer(0) - .pipe( - startWith({}), - switchMap(() => { - this.isLoadingResults = true; - return this.exampleDatabase!.getAgents().pipe(catchError(() => observableOf(null))); - }), - map(data => { - // Flip flag to show that loading has finished. - this.isLoadingResults = false; - - if (data === null) { - return []; - } - - // Only refresh the result length if there is new data. In case of rate - // limit errors, we do not want to reset the paginator to zero, as that - // would prevent users from re-triggering requests. - return data.data; - }), - ) - .subscribe(data => { if (typeof data !== 'string') { this.table_data = data } else { alert(`Error: ${data}`) } }); - } -} - -interface ServerResponse { - status: "ok" | "err", - data: T | string -} - -class ExampleHttpDatabase { - constructor(private _httpClient: HttpClient) { } - - getAgents(): Observable> { - const requestUrl = "/cmd/"; - const cmd = "agents list"; - - return this._httpClient.post>(requestUrl, cmd); - } +export class AppComponent { } diff --git a/bin/u_panel/src/server/fe/src/app/app.module.ts b/bin/u_panel/src/server/fe/src/app/app.module.ts index 56e6d5c..b251fa0 100644 --- a/bin/u_panel/src/server/fe/src/app/app.module.ts +++ b/bin/u_panel/src/server/fe/src/app/app.module.ts @@ -1,17 +1,23 @@ import { NgModule } from '@angular/core'; import { BrowserModule } from '@angular/platform-browser'; - import { AppRoutingModule } from './app-routing.module'; import { AppComponent } from './app.component'; import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; import { MatTabsModule } from '@angular/material/tabs'; import { MatTableModule } from '@angular/material/table'; +import { MatFormFieldModule } from '@angular/material/form-field'; +import { MatButtonModule } from '@angular/material/button' +import { MatInputModule } from '@angular/material/input'; import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; import { HttpClientModule } from '@angular/common/http'; +import { AgentComponent, JobComponent, ResultComponent } from './core/tables'; @NgModule({ declarations: [ - AppComponent + AppComponent, + AgentComponent, + JobComponent, + ResultComponent ], imports: [ BrowserModule, @@ -19,6 +25,9 @@ import { HttpClientModule } from '@angular/common/http'; AppRoutingModule, MatTabsModule, MatTableModule, + MatButtonModule, + MatFormFieldModule, + MatInputModule, MatProgressSpinnerModule, BrowserAnimationsModule ], diff --git a/bin/u_panel/src/server/fe/src/app/core/index.ts b/bin/u_panel/src/server/fe/src/app/core/index.ts new file mode 100644 index 0000000..b248abc --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/index.ts @@ -0,0 +1 @@ +export * from './services'; \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts b/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts new file mode 100644 index 0000000..8015ee2 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts @@ -0,0 +1,15 @@ +import { UTCDate } from "."; + +export interface AgentModel { + alias: string | null, + hostname: string, + id: string, + is_root: boolean, + is_root_allowed: boolean, + last_active: UTCDate, + platform: string, + regtime: UTCDate, + state: "new" | "active" | "banned", + token: string | null, + username: string, +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/models/index.ts b/bin/u_panel/src/server/fe/src/app/core/models/index.ts new file mode 100644 index 0000000..38623ef --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/models/index.ts @@ -0,0 +1,8 @@ +export * from './agent.model'; +export * from './result.model'; +export * from './job.model'; + +export interface UTCDate { + secs_since_epoch: number, + nanos_since_epoch: number +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/models/job.model.ts b/bin/u_panel/src/server/fe/src/app/core/models/job.model.ts new file mode 100644 index 0000000..66f1012 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/models/job.model.ts @@ -0,0 +1,8 @@ +export interface JobModel { + alias: string, + argv: string, + id: string, + exec_type: string, + platform: string, + payload: Uint8Array | null, +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/models/result.model.ts b/bin/u_panel/src/server/fe/src/app/core/models/result.model.ts new file mode 100644 index 0000000..825323b --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/models/result.model.ts @@ -0,0 +1,13 @@ +import { UTCDate } from "."; + +export interface ResultModel { + agent_id: string, + alias: string, + created: UTCDate, + id: string, + job_id: string, + result: Uint8Array, + state: "Queued" | "Running" | "Finished", + retcode: number | null, + updated: UTCDate, +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts b/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts new file mode 100644 index 0000000..cc04493 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts @@ -0,0 +1,43 @@ +import { Injectable } from '@angular/core'; +import { environment } from 'src/environments/environment'; +import { HttpClient } from '@angular/common/http'; +import { Observable } from 'rxjs'; + +interface ServerResponse { + status: "ok" | "err", + data: T | string +} + +export class ApiTableService { + area: string; + + constructor(private http: HttpClient, area: string) { + this.area = area; + } + + requestUrl = `${environment.server}/cmd/`; + + req(cmd: string): Observable> { + return this.http.post>(this.requestUrl, cmd); + } + + getOne(id: string): Observable> { + return this.req(`${this.area} read ${id}`) + } + + getMany(): Observable> { + return this.req(`${this.area} read`) + } + + update(item: T): Observable> { + return this.req(`${this.area} update '${JSON.stringify(item)}'`) + } + + delete(id: string): Observable> { + return this.req(`${this.area} delete ${id}`) + } + + create(item: string): Observable> { + return this.req(`${this.area} create ${item}`) + } +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/services/index.ts b/bin/u_panel/src/server/fe/src/app/core/services/index.ts new file mode 100644 index 0000000..aeef56f --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/services/index.ts @@ -0,0 +1 @@ +export * from './api.service' \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts new file mode 100644 index 0000000..8b1b5cb --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts @@ -0,0 +1,41 @@ +import { Component, OnInit } from '@angular/core'; +import { TablesComponent } from './table.component'; +import { AgentModel } from '../models'; + +@Component({ + selector: 'agent-table', + templateUrl: './table.component.html', + styleUrls: ['./table.component.less'] +}) +export class AgentComponent extends TablesComponent { + area = 'agents' as const; + + columns = [ + { + def: "id", + name: "ID", + cell: (cell: AgentModel) => `${cell.id}` + }, + { + def: "alias", + name: "Alias", + cell: (cell: AgentModel) => `${cell.alias}` + }, + { + def: "username", + name: "User", + cell: (cell: AgentModel) => `${cell.username}` + }, + { + def: "hostname", + name: "Host", + cell: (cell: AgentModel) => `${cell.hostname}` + }, + { + def: "last_active", + name: "Last active", + cell: (cell: AgentModel) => `${cell.last_active.secs_since_epoch}` + }, + ] + displayedColumns = this.columns.map((c) => c.def); +} diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/index.ts b/bin/u_panel/src/server/fe/src/app/core/tables/index.ts new file mode 100644 index 0000000..11dfaf2 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/index.ts @@ -0,0 +1,3 @@ +export * from './agent.component'; +export * from './job.component'; +export * from './result.component'; \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts new file mode 100644 index 0000000..27d8d42 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts @@ -0,0 +1,46 @@ +import { Component, OnInit } from '@angular/core'; +import { TablesComponent } from './table.component'; +import { JobModel } from '../models'; + +@Component({ + selector: 'job-table', + templateUrl: './table.component.html', + styleUrls: ['./table.component.less'] +}) +export class JobComponent extends TablesComponent { + area = 'jobs' as const; + + columns = [ + { + def: "id", + name: "ID", + cell: (cell: JobModel) => `${cell.id}` + }, + { + def: "alias", + name: "Alias", + cell: (cell: JobModel) => `${cell.alias}` + }, + { + def: "argv", + name: "Cmd-line args", + cell: (cell: JobModel) => `${cell.argv}` + }, + { + def: "platform", + name: "Platform", + cell: (cell: JobModel) => `${cell.platform}` + }, + { + def: "payload", + name: "Payload", + cell: (cell: JobModel) => `${cell.payload}` + }, + { + def: "etype", + name: "Type", + cell: (cell: JobModel) => `${cell.exec_type}` + }, + ] + displayedColumns = this.columns.map((c) => c.def); +} diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts new file mode 100644 index 0000000..964c8cd --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts @@ -0,0 +1,46 @@ +import { Component, OnInit } from '@angular/core'; +import { TablesComponent } from './table.component'; +import { ResultModel } from '../models'; + +@Component({ + selector: 'result-table', + templateUrl: './table.component.html', + styleUrls: ['./table.component.less'] +}) +export class ResultComponent extends TablesComponent { + area = 'map' as const; + + columns = [ + { + def: "id", + name: "ID", + cell: (cell: ResultModel) => `${cell.id}` + }, + { + def: "alias", + name: "Alias", + cell: (cell: ResultModel) => `${cell.alias}` + }, + { + def: "agent_id", + name: "Agent ID", + cell: (cell: ResultModel) => `${cell.agent_id}` + }, + { + def: "job_id", + name: "Job ID", + cell: (cell: ResultModel) => `${cell.job_id}` + }, + { + def: "state", + name: "State", + cell: (cell: ResultModel) => `${cell.state} `.concat((cell.state === "Finished") ? `(${cell.retcode})` : '') + }, + { + def: "last_updated", + name: "Last updated", + cell: (cell: ResultModel) => `${cell.updated.secs_since_epoch}` + }, + ] + displayedColumns = this.columns.map((c) => c.def); +} diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.html new file mode 100644 index 0000000..777d508 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.html @@ -0,0 +1,36 @@ +
+ +
+
+ +
+ + Filter + + + + + + + + + + + + + + + + + +
+ {{column.name}} + + {{column.cell(row)}} +
No data
+
+ + +
\ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less new file mode 100644 index 0000000..6a4475a --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less @@ -0,0 +1,24 @@ +.data-table { + width: 100%; +} + +.table-container { + margin: 50px; +} + +.loading-shade { + position: absolute; + top: 0; + left: 0; + bottom: 56px; + right: 0; + background: rgba(0, 0, 0, 0.15); + z-index: 1; + display: flex; + align-items: center; + justify-content: center; +} + +#refresh_btn { + margin-left: 10px; +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts new file mode 100644 index 0000000..5581de6 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts @@ -0,0 +1,66 @@ +import { Component, OnInit, Directive } from '@angular/core'; +import { timer, of as observableOf } from 'rxjs'; +import { catchError, map, startWith, switchMap } from 'rxjs/operators'; +import { HttpClient } from '@angular/common/http'; +import { ApiTableService } from '../'; +import { MatTableDataSource } from '@angular/material/table'; + +@Directive() +export abstract class TablesComponent implements OnInit { + abstract area: "agents" | "jobs" | "map"; + data_source!: ApiTableService | null; + table_data!: MatTableDataSource; + + isLoadingResults = true; + + constructor(private _httpClient: HttpClient) { + this.table_data = new MatTableDataSource; + } + + ngOnInit() { + this.data_source = new ApiTableService(this._httpClient, this.area); + this.fetch_many(); + // If the user changes the sort order, reset back to the first page. + //this.sort.sortChange.subscribe(() => (this.paginator.pageIndex = 0)); + + } + + fetch_many() { + timer(0) + .pipe( + startWith({}), + switchMap(() => { + this.isLoadingResults = true; + return this.data_source!.getMany().pipe(catchError(() => observableOf(null))); + }), + map(data => { + this.isLoadingResults = false; + + if (data === null) { + return []; + } + + // Only refresh the result length if there is new data. In case of rate + // limit errors, we do not want to reset the paginator to zero, as that + // would prevent users from re-triggering requests. + return data.data; + }), + ) + .subscribe(data => { if (typeof data !== 'string') { this.table_data.data = data } else { alert(`Error: ${data}`) } }); + } + + apply_filter(event: Event) { + const filterValue = (event.target as HTMLInputElement).value; + this.table_data.filter = filterValue.trim().toLowerCase(); + } + + + abstract columns: ColumnDef[]; + abstract displayedColumns: string[]; +} + +type ColumnDef = { + def: string, + name: string, + cell: (cell: C) => string +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/environments/environment.prod.ts b/bin/u_panel/src/server/fe/src/environments/environment.prod.ts index 3612073..95d0187 100644 --- a/bin/u_panel/src/server/fe/src/environments/environment.prod.ts +++ b/bin/u_panel/src/server/fe/src/environments/environment.prod.ts @@ -1,3 +1,4 @@ export const environment = { - production: true + production: true, + server: "" }; diff --git a/bin/u_panel/src/server/fe/src/environments/environment.ts b/bin/u_panel/src/server/fe/src/environments/environment.ts index f56ff47..c016fed 100644 --- a/bin/u_panel/src/server/fe/src/environments/environment.ts +++ b/bin/u_panel/src/server/fe/src/environments/environment.ts @@ -3,7 +3,8 @@ // The list of file replacements can be found in `angular.json`. export const environment = { - production: false + production: false, + server: "http://127.0.0.1:8080" }; /* diff --git a/bin/u_panel/src/server/mod.rs b/bin/u_panel/src/server/mod.rs index a427567..49b08f7 100644 --- a/bin/u_panel/src/server/mod.rs +++ b/bin/u_panel/src/server/mod.rs @@ -14,12 +14,14 @@ almost all fields are editable, rows are deletable mod errors; use crate::{process_cmd, Args}; +use actix_cors::Cors; use actix_web::{get, middleware::Logger, post, web, App, HttpResponse, HttpServer, Responder}; use errors::Error; +use futures_util::StreamExt; use rust_embed::RustEmbed; use std::borrow::Cow; use structopt::StructOpt; -use u_lib::{api::ClientHandler, logging::init_logger, unwrap_enum}; +use u_lib::{api::ClientHandler, unwrap_enum}; #[derive(RustEmbed)] #[folder = "./src/server/fe/dist/fe/"] @@ -52,12 +54,21 @@ async fn static_files_adapter(path: web::Path<(String,)>) -> impl Responder { #[post("/cmd/")] async fn send_cmd( - cmd: web::Json, + mut body: web::Payload, client: web::Data, ) -> Result { - let parsed_cmd = Args::from_iter_safe( - shlex::split(&cmd.into_inner()).ok_or(Error::JustError("shlex failed".to_string()))?, - )?; + let mut bytes = web::BytesMut::new(); + while let Some(item) = body.next().await { + bytes.extend_from_slice( + &item.map_err(|e| Error::JustError(format!("payload loading failure: {e}")))?, + ); + } + let cmd = String::from_utf8(bytes.to_vec()) + .map_err(|_| Error::JustError("cmd contains non-utf8 data".to_string()))?; + let mut cmd = shlex::split(&cmd).ok_or(Error::JustError("argparse failed".to_string()))?; + info!("cmd: {:?}", cmd); + cmd.insert(0, String::from("u_panel")); + let parsed_cmd = Args::from_iter_safe(cmd)?; Ok( match process_cmd(client.as_ref().clone(), parsed_cmd).await { Ok(r) => HttpResponse::Ok().body(r), @@ -66,15 +77,14 @@ async fn send_cmd( ) } -#[actix_web::main] pub async fn serve(client: ClientHandler) -> std::io::Result<()> { - init_logger(Some("u_panel")); - let addr = "127.0.0.1:8080"; info!("Serving at http://{}", addr); + HttpServer::new(move || { App::new() .wrap(Logger::default()) + .wrap(Cors::permissive()) .app_data(web::Data::new(client.clone())) .service(main_page) .service(send_cmd) diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index 2c77133..d0929ca 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -4,7 +4,7 @@ use once_cell::sync::OnceCell; use serde::Deserialize; use std::sync::{Arc, Mutex, MutexGuard}; use u_lib::{ - models::{schema, Agent, AgentError, AssignedJob, JobMeta, JobState}, + models::{schema, Agent, AssignedJob, JobMeta, JobState}, utils::load_env, }; use uuid::Uuid; @@ -40,14 +40,6 @@ impl UDB { .unwrap() } - pub fn report_error(&self, error: &AgentError) -> SResult<()> { - use schema::errors; - diesel::insert_into(errors::table) - .values(error) - .execute(&self.conn)?; - Ok(()) - } - pub fn insert_jobs(&self, job_metas: &[JobMeta]) -> SResult<()> { use schema::jobs; diesel::insert_into(jobs::table) @@ -203,35 +195,3 @@ impl UDB { Ok(affected) } } -/* -#[cfg(test)] -mod tests { - use super::*; - - fn setup_db() -> Storage { - return UDB::new().unwrap(); - } - - #[tokio::test] - async fn test_add_agent() { - let db = setup_db(); - let agent = IAgent { - alias: None, - id: "000-000".to_string(), - hostname: "test".to_string(), - is_root: false, - is_root_allowed: false, - platform: "linux".to_string(), - status: None, - token: None, - username: "test".to_string() - }; - db.lock().unwrap().new_agent(agent).unwrap(); - let result = db.lock().unwrap().get_agents().unwrap(); - assert_eq!( - result[0].username, - "test".to_string() - ) - } -} -*/ diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index 41c2411..4e1eebe 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -1,3 +1,5 @@ +use std::time::SystemTime; + use crate::db::UDB; use crate::errors::Error; use diesel::SaveChangesDsl; @@ -89,12 +91,14 @@ impl Endpoints { let mut failed = vec![]; for entry in msg.into_inner().into_vec() { match entry { - Reportable::Assigned(res) => { - if id != res.agent_id { + Reportable::Assigned(mut result) => { + if id != result.agent_id { continue; } + result.state = JobState::Finished; + result.updated = SystemTime::now(); let db = UDB::lock_db(); - if let Err(e) = res + if let Err(e) = result .save_changes::(&db.conn) .map_err(Error::from) { @@ -106,13 +110,7 @@ impl Endpoints { Self::add_agent(a).await?; } Reportable::Error(e) => { - let err = AgentError::from_msg(e, id); - warn!( - "{} reported an error: {}", - err.agent_id, - Stripped(&err.msg.as_str()) - ); - //UDB::lock_db().report_error(&err)?; + warn!("{} reported an error: {}", id, Stripped(&e.as_str())); } Reportable::Dummy => (), } diff --git a/bin/u_server/src/models.rs b/bin/u_server/src/models.rs new file mode 100644 index 0000000..e69de29 diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 406159c..9c6a2d7 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -14,6 +14,7 @@ extern crate diesel; mod db; mod errors; mod handlers; +mod models; use errors::{Error, SResult}; use serde::{de::DeserializeOwned, Deserialize}; @@ -151,12 +152,20 @@ pub fn init_endpoints( } pub fn prefill_jobs() -> SResult<()> { - let agent_hello = RawJobMeta::builder() - .with_type(misc::JobType::Manage) - .with_alias("agent_hello") - .build() - .unwrap(); - UDB::lock_db().insert_jobs(&[agent_hello]) + let job_alias = "agent_hello"; + let if_job_exists = UDB::lock_db().find_job_by_alias(job_alias); + match if_job_exists { + Ok(_) => Ok(()), + Err(Error::DBError(diesel::result::Error::NotFound)) => { + let agent_hello = RawJobMeta::builder() + .with_type(misc::JobType::Manage) + .with_alias(job_alias) + .build() + .unwrap(); + UDB::lock_db().insert_jobs(&[agent_hello]) + } + Err(e) => Err(e), + } } pub async fn serve() -> SResult<()> { diff --git a/integration/tests/integration/connection.rs b/integration/tests/integration/connection.rs new file mode 100644 index 0000000..78a4c2c --- /dev/null +++ b/integration/tests/integration/connection.rs @@ -0,0 +1,22 @@ +use crate::helpers::ENV; +use u_lib::config::MASTER_PORT; + +#[tokio::test] +async fn test_non_auth_connection_dropped() { + let client = reqwest::ClientBuilder::new() + .danger_accept_invalid_certs(true) + .build() + .unwrap(); + match client + .get(format!("https://{}:{}", &ENV.u_server, MASTER_PORT)) + .send() + .await + { + Err(e) => { + let err = e.to_string(); + println!("captured err: {err}"); + assert!(err.contains("certificate required")); + } + _ => panic!("no error occured on foreign client connection"), + } +} diff --git a/integration/tests/integration/mod.rs b/integration/tests/integration/mod.rs index b3a413f..0b76512 100644 --- a/integration/tests/integration/mod.rs +++ b/integration/tests/integration/mod.rs @@ -1 +1,2 @@ mod behaviour; +mod connection; diff --git a/integration/tests/lib.rs b/integration/tests/lib.rs index 8009650..826b82d 100644 --- a/integration/tests/lib.rs +++ b/integration/tests/lib.rs @@ -2,28 +2,5 @@ mod fixtures; mod helpers; mod integration; -use crate::helpers::ENV; -use u_lib::config::MASTER_PORT; - #[macro_use] extern crate rstest; - -#[tokio::test] -async fn test_non_auth_connection_dropped() { - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); - match client - .get(format!("https://{}:{}", &ENV.u_server, MASTER_PORT)) - .send() - .await - { - Err(e) => { - let err = e.to_string(); - println!("{err}"); - assert!(err.contains("channel closed")) - } - _ => panic!("no error occured on foreign client connection"), - } -} diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index 54f1c42..70644bb 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -28,6 +28,7 @@ tracing-subscriber = { version = "0.3.14", features = ["env-filter"] } tracing-appender = "0.2.2" log = "*" anyhow = "1.0.58" +platforms = "3.0.1" [features] panel = [] diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index f5c86fd..7669da2 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -92,6 +92,12 @@ impl ClientHandler { pub async fn dl(&self, file: String) -> Result> { self._req(format!("dl/{file}"), Empty).await } + + /// get all available jobs + pub async fn get_jobs(&self, job: Option) -> Result> { + self._req(format!("get_jobs/{}", opt_to_string(job)), Empty) + .await + } } //##########// Admin area //##########// @@ -108,12 +114,6 @@ impl ClientHandler { self._req("update_item", item).await } - /// get all available jobs - pub async fn get_jobs(&self, job: Option) -> Result> { - self._req(format!("get_jobs/{}", opt_to_string(job)), Empty) - .await - } - /// create and upload job pub async fn upload_jobs(&self, payload: &[models::JobMeta]) -> Result { self._req("upload_jobs", payload).await @@ -131,7 +131,7 @@ impl ClientHandler { /// get jobs for any agent pub async fn get_agent_jobs(&self, agent: Option) -> Result> { - self._req(format!("get_personal_jobs/{}", opt_to_string(agent)), Empty) + self._req(format!("get_agent_jobs/{}", opt_to_string(agent)), Empty) .await } } diff --git a/lib/u_lib/src/builder.rs b/lib/u_lib/src/builder.rs index 01b949d..84e46c5 100644 --- a/lib/u_lib/src/builder.rs +++ b/lib/u_lib/src/builder.rs @@ -3,10 +3,9 @@ use crate::{ executor::{DynFut, Waiter}, messaging::Reportable, models::{Agent, AssignedJob, JobMeta, JobType, RawJobMeta}, - utils::{CombinedResult, OneOrVec}, + utils::{CombinedResult, OneOrVec, Platform}, UError, UResult, }; -use guess_host_triple::guess_host_triple; use std::collections::HashMap; pub struct JobBuilder { @@ -30,12 +29,11 @@ impl JobBuilder { Ok(match job_meta.exec_type { JobType::Shell => { let meta = JobCache::get(req.job_id).ok_or(UError::NoJob(req.job_id))?; - let curr_platform = guess_host_triple().unwrap_or("unknown").to_string(); - //TODO: extend platform checking (partial check) - if meta.platform != curr_platform { + let curr_platform = Platform::current(); + if !curr_platform.matches(&meta.platform) { return Err(UError::InsuitablePlatform( meta.platform.clone(), - curr_platform, + curr_platform.into_string(), ) .into()); } diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index 6843e0e..869be5e 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -7,8 +7,11 @@ use strum::Display; #[cfg(not(target_arch = "wasm32"))] use crate::builder::NamedJobBuilder; use crate::{ - config::get_self_uid, messaging::Reportable, models::schema::*, unwrap_enum, - utils::systime_to_string, + config::get_self_uid, + messaging::Reportable, + models::schema::*, + unwrap_enum, + utils::{systime_to_string, Platform}, }; use uuid::Uuid; @@ -94,13 +97,16 @@ impl Agent { hostname: decoder(builder.pop("hostname")), is_root: &decoder(builder.pop("is_root")) == "0", username: decoder(builder.pop("username")), - platform: guess_host_triple::guess_host_triple() - .unwrap_or("unknown") - .to_string(), + platform: Platform::current().into_string(), ..Default::default() } } + #[cfg(not(unix))] + pub async fn gather() -> Self { + Self::default() + } + pub async fn run() -> Reportable { Reportable::Agent(Agent::gather().await) } diff --git a/lib/u_lib/src/models/error.rs b/lib/u_lib/src/models/error.rs deleted file mode 100644 index a916dd0..0000000 --- a/lib/u_lib/src/models/error.rs +++ /dev/null @@ -1,35 +0,0 @@ -use crate::models::schema::*; -use diesel::{Insertable, Queryable}; -use serde::{Deserialize, Serialize}; -use std::time::SystemTime; -use uuid::Uuid; - -#[derive(Serialize, Deserialize, Clone, Debug, Queryable, Insertable, PartialEq)] -#[table_name = "errors"] -pub struct AgentError { - pub agent_id: Uuid, - pub created: SystemTime, - pub id: Uuid, - pub msg: String, -} - -impl AgentError { - pub fn from_msg(msg: impl Into, agent_id: Uuid) -> Self { - AgentError { - agent_id, - msg: msg.into(), - ..Default::default() - } - } -} - -impl Default for AgentError { - fn default() -> Self { - Self { - agent_id: Uuid::new_v4(), - created: SystemTime::now(), - id: Uuid::new_v4(), - msg: String::new(), - } - } -} diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index d2465b6..2854e0f 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -112,8 +112,6 @@ impl AssignedJob { }; self.result = Some(data); self.retcode = retcode; - self.updated = SystemTime::now(); - self.state = JobState::Finished; Reportable::Assigned(self) } diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 657be72..ccb88d5 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -1,4 +1,5 @@ use super::JobType; +use crate::utils::Platform; use crate::{models::schema::*, utils::Stripped, UError, UResult}; use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; @@ -23,19 +24,19 @@ pub struct JobMeta { pub payload: Option>, } -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] pub struct RawJobMeta { #[serde(default)] pub alias: Option, #[serde(default)] pub argv: String, #[serde(default)] - pub id: Uuid, + pub id: UuidDefaultUnique, #[serde(default)] pub exec_type: JobType, //pub schedule: JobSchedule, #[serde(default)] - pub platform: String, + pub platform: Platform, #[serde(default)] pub payload: Option>, #[serde(default)] @@ -86,9 +87,7 @@ impl Default for JobMeta { alias: None, argv: String::new(), exec_type: JobType::Shell, - platform: guess_host_triple::guess_host_triple() - .unwrap_or("unknown") - .to_string(), + platform: Platform::current().into_string(), payload: None, } } @@ -97,13 +96,11 @@ impl Default for JobMeta { impl Default for RawJobMeta { fn default() -> Self { Self { - id: Uuid::new_v4(), + id: UuidDefaultUnique::default(), alias: None, argv: String::new(), exec_type: JobType::Shell, - platform: guess_host_triple::guess_host_triple() - .unwrap_or("unknown") - .to_string(), + platform: Platform::current(), payload: None, payload_path: None, } @@ -179,6 +176,12 @@ impl JobMetaBuilder { } } }; + if !inner.platform.check() { + return Err(UError::JobArgsError(format!( + "Unknown platform {}", + inner.platform.into_string() + ))); + } Ok(inner.into()) } JobType::Manage => Ok(inner.into()), @@ -192,10 +195,19 @@ impl From for JobMeta { JobMeta { alias: rjm.alias, argv: rjm.argv, - id: rjm.id, + id: rjm.id.0, exec_type: rjm.exec_type, - platform: rjm.platform, + platform: rjm.platform.into_string(), payload: rjm.payload, } } } + +#[derive(Deserialize, Debug)] +pub struct UuidDefaultUnique(Uuid); + +impl Default for UuidDefaultUnique { + fn default() -> Self { + Self(Uuid::new_v4()) + } +} diff --git a/lib/u_lib/src/models/mod.rs b/lib/u_lib/src/models/mod.rs index 3178567..b0b30a7 100644 --- a/lib/u_lib/src/models/mod.rs +++ b/lib/u_lib/src/models/mod.rs @@ -1,6 +1,5 @@ mod agent; -mod error; mod jobs; pub mod schema; -pub use crate::models::{agent::*, error::*, jobs::*}; +pub use crate::models::{agent::*, jobs::*}; diff --git a/lib/u_lib/src/utils/mod.rs b/lib/u_lib/src/utils/mod.rs index df527db..4d6e004 100644 --- a/lib/u_lib/src/utils/mod.rs +++ b/lib/u_lib/src/utils/mod.rs @@ -3,6 +3,7 @@ pub mod conv; pub mod env; pub mod fmt; pub mod misc; +pub mod platform; pub mod proc_output; pub mod storage; #[cfg(not(target_arch = "wasm32"))] @@ -16,6 +17,7 @@ pub use conv::*; pub use env::{load_env, load_env_default}; pub use fmt::*; pub use misc::*; +pub use platform::*; pub use proc_output::*; pub use storage::*; #[cfg(not(target_arch = "wasm32"))] diff --git a/lib/u_lib/src/utils/platform.rs b/lib/u_lib/src/utils/platform.rs new file mode 100644 index 0000000..7e13301 --- /dev/null +++ b/lib/u_lib/src/utils/platform.rs @@ -0,0 +1,38 @@ +use guess_host_triple::guess_host_triple; +use platforms::{Platform as _Platform, PlatformReq}; +use serde::Deserialize; +use std::str::FromStr; + +#[derive(Debug, Deserialize)] +pub struct Platform(String); + +impl Platform { + pub fn current() -> Platform { + Self(guess_host_triple().unwrap_or("unknown").to_string()) + } + + pub fn matches(&self, pf: impl AsRef) -> bool { + match PlatformReq::from_str(pf.as_ref()) { + Ok(p) => p.matches(&_Platform::find(&self.0).unwrap()), + Err(_) => false, + } + } + + pub fn check(&self) -> bool { + PlatformReq::from_str(&self.0).is_ok() + } + + pub fn into_string(self) -> String { + self.0 + } + + pub fn any() -> Platform { + Self(String::from("*")) + } +} + +impl Default for Platform { + fn default() -> Self { + Self::any() + } +} diff --git a/lib/u_lib/src/utils/tempfile.rs b/lib/u_lib/src/utils/tempfile.rs index a5a510e..d2fb58f 100644 --- a/lib/u_lib/src/utils/tempfile.rs +++ b/lib/u_lib/src/utils/tempfile.rs @@ -1,5 +1,7 @@ use crate::{UError, UResult}; -use std::{env::temp_dir, fs, ops::Drop, os::unix::fs::PermissionsExt, path::PathBuf}; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::{env::temp_dir, fs, ops::Drop, path::PathBuf}; use uuid::Uuid; pub struct TempFile { @@ -28,8 +30,12 @@ impl TempFile { let path = this.get_path(); dbg!(&path); this.write_all(data)?; - let perms = fs::Permissions::from_mode(0o555); - fs::set_permissions(&path, perms).map_err(|e| UError::FSError(path, e.to_string()))?; + + #[cfg(unix)] + { + let perms = fs::Permissions::from_mode(0o555); + fs::set_permissions(&path, perms).map_err(|e| UError::FSError(path, e.to_string()))?; + } Ok(this) } } diff --git a/migrations/2020-10-24-111622_create_all/down.sql b/migrations/2020-10-24-111622_create_all/down.sql index c3bfd8c..3ded775 100644 --- a/migrations/2020-10-24-111622_create_all/down.sql +++ b/migrations/2020-10-24-111622_create_all/down.sql @@ -3,7 +3,6 @@ DROP TABLE results; DROP TABLE certificates; DROP TABLE jobs; DROP TABLE agents; -DROP TABLE errors; DROP TYPE IF EXISTS JobState; DROP TYPE IF EXISTS JobType; diff --git a/migrations/2020-10-24-111622_create_all/up.sql b/migrations/2020-10-24-111622_create_all/up.sql index d319d65..a26d094 100644 --- a/migrations/2020-10-24-111622_create_all/up.sql +++ b/migrations/2020-10-24-111622_create_all/up.sql @@ -64,14 +64,4 @@ CREATE TABLE IF NOT EXISTS certificates ( , is_revoked BOOLEAN NOT NULL DEFAULT FALSE , PRIMARY KEY(id) , FOREIGN KEY(agent_id) REFERENCES agents(id) -); - - -CREATE TABLE IF NOT EXISTS errors ( - agent_id UUID NOT NULL - , created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - , id UUID NOT NULL DEFAULT uuid_generate_v4() - , msg TEXT - , FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE - , PRIMARY KEY(id) ); \ No newline at end of file From 0a077af9363f0f9e2378949a6459ed490d9a1c0b Mon Sep 17 00:00:00 2001 From: plazmoid Date: Wed, 27 Jul 2022 00:34:25 +0500 Subject: [PATCH 24/30] make diesel optional in u_lib and cross-compile u_agent to windows --- Cargo.toml | 2 +- bin/u_server/Cargo.toml | 2 +- bin/u_server/src/models.rs | 0 bin/u_server/src/u_server.rs | 1 - lib/u_lib/Cargo.toml | 5 +-- lib/u_lib/src/lib.rs | 2 ++ lib/u_lib/src/models/agent.rs | 49 ++++++++++++++------------- lib/u_lib/src/models/jobs/assigned.rs | 21 +++++------- lib/u_lib/src/models/jobs/meta.rs | 13 ++++--- lib/u_lib/src/models/jobs/misc.rs | 21 ++++++++---- lib/u_lib/src/models/mod.rs | 1 + 11 files changed, 66 insertions(+), 51 deletions(-) delete mode 100644 bin/u_server/src/models.rs diff --git a/Cargo.toml b/Cargo.toml index 228808c..02c9d71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ members = [ "bin/u_agent", "bin/u_panel", - #"bin/u_run", + "bin/u_run", "bin/u_server", "lib/u_lib", "integration" diff --git a/bin/u_server/Cargo.toml b/bin/u_server/Cargo.toml index d2e5f75..5b07891 100644 --- a/bin/u_server/Cargo.toml +++ b/bin/u_server/Cargo.toml @@ -15,7 +15,7 @@ openssl = "*" diesel = { version = "1.4.5", features = ["postgres", "uuid"] } serde = { version = "1.0", features = ["derive"] } tokio = { version = "1.9", features = ["macros"] } -u_lib = { path = "../../lib/u_lib", version = "*" } +u_lib = { path = "../../lib/u_lib", version = "*", features = ["server"] } [dev-dependencies] diff --git a/bin/u_server/src/models.rs b/bin/u_server/src/models.rs deleted file mode 100644 index e69de29..0000000 diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 9c6a2d7..d2e84d5 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -14,7 +14,6 @@ extern crate diesel; mod db; mod errors; mod handlers; -mod models; use errors::{Error, SResult}; use serde::{de::DeserializeOwned, Deserialize}; diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index 70644bb..63de9f6 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -15,13 +15,13 @@ libc = "^0.2" lazy_static = "1.4.0" futures = "0.3.5" thiserror = "*" -diesel-derive-enum = { version = "1", features = ["postgres"] } +diesel-derive-enum = { version = "1", features = ["postgres"], optional = true } chrono = "0.4.19" strum = { version = "0.20", features = ["derive"] } once_cell = "1.7.2" shlex = "1.0.0" crossbeam = "0.8.1" -diesel = { version = "1.4.5", features = ["postgres", "uuid"] } +diesel = { version = "1.4.5", features = ["postgres", "uuid"], optional = true } envy = "0.4.2" serde_json = "1.0.81" tracing-subscriber = { version = "0.3.14", features = ["env-filter"] } @@ -32,6 +32,7 @@ platforms = "3.0.1" [features] panel = [] +server = ["dep:diesel", "dep:diesel-derive-enum"] [target.'cfg(not(target_arch = "wasm32"))'.dependencies] reqwest = { version = "0.11", features = ["json", "native-tls"] } diff --git a/lib/u_lib/src/lib.rs b/lib/u_lib/src/lib.rs index 41383a4..b6a94f8 100644 --- a/lib/u_lib/src/lib.rs +++ b/lib/u_lib/src/lib.rs @@ -28,11 +28,13 @@ pub mod exports { pub use errors::{UError, UResult}; pub use exports::*; +#[cfg(feature = "server")] pub mod schema_exports { pub use crate::models::{Agentstate, Jobstate, Jobtype}; pub use diesel::sql_types::*; } +#[cfg(feature = "server")] #[macro_use] extern crate diesel; diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index 869be5e..71483ac 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -1,4 +1,6 @@ +#[cfg(feature = "server")] use diesel::{AsChangeset, Identifiable, Insertable, Queryable}; +#[cfg(feature = "server")] use diesel_derive_enum::DbEnum; use serde::{Deserialize, Serialize}; use std::{fmt, time::SystemTime}; @@ -6,19 +8,24 @@ use strum::Display; #[cfg(not(target_arch = "wasm32"))] use crate::builder::NamedJobBuilder; +#[cfg(feature = "server")] +use crate::models::schema::*; use crate::{ config::get_self_uid, messaging::Reportable, - models::schema::*, unwrap_enum, utils::{systime_to_string, Platform}, }; use uuid::Uuid; -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, DbEnum, Display)] -#[PgType = "AgentState"] -#[DieselType = "Agentstate"] +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Display)] +#[cfg_attr( + feature = "server", + derive(DbEnum), + PgType = "AgentState", + DieselType = "Agentstate" +)] pub enum AgentState { New, Active, @@ -26,18 +33,12 @@ pub enum AgentState { } //belongs_to -#[derive( - Clone, - Debug, - Serialize, - Deserialize, - Identifiable, - Queryable, - Insertable, - AsChangeset, - PartialEq, +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[cfg_attr( + feature = "server", + derive(Identifiable, Queryable, Insertable, AsChangeset), + table_name = "agents" )] -#[table_name = "agents"] pub struct Agent { pub alias: Option, pub hostname: String, @@ -130,13 +131,13 @@ impl Default for Agent { } } -#[cfg(test)] -mod tests { - use super::*; +// #[cfg(test)] +// mod tests { +// use super::*; - #[tokio::test] - async fn test_gather() { - let cli_info = Agent::gather().await; - assert_eq!(cli_info.alias, None) - } -} +// #[tokio::test] +// async fn test_gather() { +// let cli_info = Agent::gather().await; +// assert_eq!(cli_info.alias, None) +// } +// } diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index 2854e0f..1122031 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -1,30 +1,27 @@ use super::JobState; +#[cfg(feature = "server")] +use crate::models::schema::*; #[cfg(not(target_arch = "wasm32"))] use crate::{cache::JobCache, utils::TempFile}; use crate::{ config::get_self_uid, errors::UError, messaging::Reportable, - models::schema::*, utils::{systime_to_string, ProcOutput}, }; +#[cfg(feature = "server")] use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; use std::{fmt, time::SystemTime}; use uuid::Uuid; -#[derive( - Serialize, - Deserialize, - Clone, - Debug, - Queryable, - Identifiable, - Insertable, - AsChangeset, - PartialEq, +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] +#[cfg_attr( + feature = "server", + derive(Queryable, Identifiable, Insertable, AsChangeset), + table_name = "results" )] -#[table_name = "results"] + pub struct AssignedJob { pub agent_id: Uuid, pub alias: Option, diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index ccb88d5..2b76941 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -1,6 +1,9 @@ use super::JobType; +#[cfg(feature = "server")] +use crate::models::schema::*; use crate::utils::Platform; -use crate::{models::schema::*, utils::Stripped, UError, UResult}; +use crate::{utils::Stripped, UError, UResult}; +#[cfg(feature = "server")] use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -8,10 +11,12 @@ use std::str::from_utf8; use std::{fmt, fs}; use uuid::Uuid; -#[derive( - Serialize, Deserialize, Clone, Debug, Queryable, Identifiable, Insertable, AsChangeset, +#[derive(Serialize, Deserialize, Clone, Debug)] +#[cfg_attr( + feature = "server", + derive(Queryable, Identifiable, Insertable, AsChangeset), + table_name = "jobs" )] -#[table_name = "jobs"] pub struct JobMeta { pub alias: Option, /// string like `bash -c {} -a 1 --arg2`, diff --git a/lib/u_lib/src/models/jobs/misc.rs b/lib/u_lib/src/models/jobs/misc.rs index 2eb4ce4..df91f0d 100644 --- a/lib/u_lib/src/models/jobs/misc.rs +++ b/lib/u_lib/src/models/jobs/misc.rs @@ -1,3 +1,4 @@ +#[cfg(feature = "server")] use diesel_derive_enum::DbEnum; use serde::{Deserialize, Serialize}; use strum::Display; @@ -17,9 +18,13 @@ pub enum JobSchedule { //Scheduled } -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, DbEnum, Display)] -#[PgType = "JobState"] -#[DieselType = "Jobstate"] +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Display)] +#[cfg_attr( + feature = "server", + derive(DbEnum), + PgType = "JobState", + DieselType = "Jobstate" +)] pub enum JobState { Queued, // server created a job, but client didn't get it yet //Pending, // client got a job, but not running yet @@ -27,9 +32,13 @@ pub enum JobState { Finished, } -#[derive(Default, Serialize, Deserialize, Clone, Debug, PartialEq, DbEnum, Display)] -#[PgType = "JobType"] -#[DieselType = "Jobtype"] +#[derive(Default, Serialize, Deserialize, Clone, Debug, PartialEq, Display)] +#[cfg_attr( + feature = "server", + derive(DbEnum), + PgType = "JobType", + DieselType = "Jobtype" +)] pub enum JobType { Manage, #[default] diff --git a/lib/u_lib/src/models/mod.rs b/lib/u_lib/src/models/mod.rs index b0b30a7..bb6e50e 100644 --- a/lib/u_lib/src/models/mod.rs +++ b/lib/u_lib/src/models/mod.rs @@ -1,5 +1,6 @@ mod agent; mod jobs; +#[cfg(feature = "server")] pub mod schema; pub use crate::models::{agent::*, jobs::*}; From a594348a303c5d190404ea05adb6a9801748f296 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Mon, 12 Sep 2022 02:28:20 +0500 Subject: [PATCH 25/30] smol updates --- Makefile.toml | 2 +- bin/u_agent/src/lib.rs | 1 - bin/u_panel/Cargo.toml | 5 ---- bin/u_panel/src/server/fe/package.json | 2 +- .../src/server/fe/src/app/app.module.ts | 6 ++++- .../fe/src/app/core/tables/agent.component.ts | 26 +++++++++++++++---- .../tables/dialogs/agent-info-dialog.html | 12 +++++++++ .../tables/dialogs/agent_info.component.ts | 11 ++++++++ .../fe/src/app/core/tables/dialogs/index.ts | 1 + .../fe/src/app/core/tables/job.component.ts | 12 +++++---- .../src/app/core/tables/result.component.ts | 15 +++++++---- .../src/app/core/tables/table.component.html | 3 ++- .../src/app/core/tables/table.component.less | 5 ++++ .../fe/src/app/core/tables/table.component.ts | 6 +++-- .../src/server/fe/src/app/core/utils.ts | 3 +++ bin/u_server/src/db.rs | 6 ++--- images/integration-tests/u_db.Dockerfile | 5 ++-- integration/docker_compose.py | 5 ++-- integration/integration_tests.py | 22 +++++++++++++--- lib/u_lib/src/api.rs | 4 +-- todos.txt | 7 +++++ 21 files changed, 119 insertions(+), 40 deletions(-) create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/dialogs/index.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/utils.ts create mode 100644 todos.txt diff --git a/Makefile.toml b/Makefile.toml index c5fb8e0..63e14e2 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -71,7 +71,7 @@ args = ["test", "--target", "${TARGET}", "--lib", "--", "${@}"] [tasks.integration] script = ''' -echo "!!! This task doesn't perform project rebuild, trigger it manually if need" +[[ ! -d "./target/${TARGET}/${PROFILE_OVERRIDE}" ]] && echo 'No target folder. Build project first' && exit 1 cd ./integration bash integration_tests.sh ${@} ''' diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 09e016f..597df48 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -1,7 +1,6 @@ // TODO: // поддержка питона // резолв адреса управляющего сервера через DoT -// кроссплатформенность (реализовать интерфейс для винды и никсов) #[macro_use] extern crate log; diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 1ea2be9..1e69c0c 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -8,20 +8,15 @@ edition = "2021" [dependencies] actix-web = "4.1" -backtrace = "0.3.61" structopt = "0.3.21" uuid = "0.6.5" serde_json = "1.0.4" serde = { version = "1.0.114", features = ["derive"] } tokio = { version = "1.11.0", features = ["rt", "rt-multi-thread"] } u_lib = { version = "*", path = "../../lib/u_lib", features = ["panel"] } -tui = { version = "0.16", default-features = false, features = ['crossterm'] } -crossterm = "0.22.1" anyhow = "1.0.44" strum = { version = "0.22.0", features = ["derive"] } once_cell = "1.8.0" -crossbeam = "0.8.1" -async-channel = "1.6.1" tracing = "0.1.29" tracing-subscriber = { version = "0.3.3", features = ["env-filter"]} signal-hook = "0.3.12" diff --git a/bin/u_panel/src/server/fe/package.json b/bin/u_panel/src/server/fe/package.json index 9cc9000..fb86167 100644 --- a/bin/u_panel/src/server/fe/package.json +++ b/bin/u_panel/src/server/fe/package.json @@ -27,7 +27,7 @@ "zone.js": "~0.11.4" }, "devDependencies": { - "@angular-devkit/build-angular": "~13.1.2", + "@angular-devkit/build-angular": "^13.3.9", "@angular/cli": "~13.1.2", "@angular/compiler-cli": "~13.1.0", "@types/jasmine": "~3.10.0", diff --git a/bin/u_panel/src/server/fe/src/app/app.module.ts b/bin/u_panel/src/server/fe/src/app/app.module.ts index b251fa0..a78705c 100644 --- a/bin/u_panel/src/server/fe/src/app/app.module.ts +++ b/bin/u_panel/src/server/fe/src/app/app.module.ts @@ -10,14 +10,17 @@ import { MatButtonModule } from '@angular/material/button' import { MatInputModule } from '@angular/material/input'; import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; import { HttpClientModule } from '@angular/common/http'; +import { MatDialogModule } from '@angular/material/dialog'; import { AgentComponent, JobComponent, ResultComponent } from './core/tables'; +import { AgentInfoDialogComponent } from './core/tables/dialogs'; @NgModule({ declarations: [ AppComponent, AgentComponent, JobComponent, - ResultComponent + ResultComponent, + AgentInfoDialogComponent ], imports: [ BrowserModule, @@ -28,6 +31,7 @@ import { AgentComponent, JobComponent, ResultComponent } from './core/tables'; MatButtonModule, MatFormFieldModule, MatInputModule, + MatDialogModule, MatProgressSpinnerModule, BrowserAnimationsModule ], diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts index 8b1b5cb..dea2e61 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts @@ -1,6 +1,10 @@ import { Component, OnInit } from '@angular/core'; import { TablesComponent } from './table.component'; import { AgentModel } from '../models'; +import { AgentInfoDialogComponent } from './dialogs/agent_info.component'; +import { HttpClient } from '@angular/common/http'; +import { MatDialog } from '@angular/material/dialog'; +import { epochToStr } from '../utils'; @Component({ selector: 'agent-table', @@ -8,34 +12,46 @@ import { AgentModel } from '../models'; styleUrls: ['./table.component.less'] }) export class AgentComponent extends TablesComponent { + + constructor(public override _httpClient: HttpClient, public override info_dlg: MatDialog) { + super(_httpClient, info_dlg); + } + area = 'agents' as const; columns = [ { def: "id", name: "ID", - cell: (cell: AgentModel) => `${cell.id}` + cell: (cell: AgentModel) => cell.id }, { def: "alias", name: "Alias", - cell: (cell: AgentModel) => `${cell.alias}` + cell: (cell: AgentModel) => cell.alias ?? "" }, { def: "username", name: "User", - cell: (cell: AgentModel) => `${cell.username}` + cell: (cell: AgentModel) => cell.username }, { def: "hostname", name: "Host", - cell: (cell: AgentModel) => `${cell.hostname}` + cell: (cell: AgentModel) => cell.hostname }, { def: "last_active", name: "Last active", - cell: (cell: AgentModel) => `${cell.last_active.secs_since_epoch}` + cell: (cell: AgentModel) => epochToStr(cell.last_active.secs_since_epoch) }, ] + displayedColumns = this.columns.map((c) => c.def); + + show_item_dialog(obj: AgentModel) { + const dialog = this.info_dlg.open(AgentInfoDialogComponent, { + data: obj + }); + } } diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html new file mode 100644 index 0000000..eaf5e5a --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html @@ -0,0 +1,12 @@ + +
+

Alias: {{data.alias}}

+

Username: {{data.username}}

+

Hostname: {{data.hostname}}

+

Platform: {{data.platform}}

+
+ + + + + \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts new file mode 100644 index 0000000..1c318ca --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts @@ -0,0 +1,11 @@ +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { AgentModel } from '../../models/agent.model'; + +@Component({ + selector: 'agent-info-dialog', + templateUrl: 'agent-info-dialog.html', +}) +export class AgentInfoDialogComponent { + constructor(@Inject(MAT_DIALOG_DATA) public data: AgentModel) { } +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/index.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/index.ts new file mode 100644 index 0000000..1aa08ec --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/index.ts @@ -0,0 +1 @@ +export * from './agent_info.component'; \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts index 27d8d42..28f9c92 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts @@ -14,22 +14,22 @@ export class JobComponent extends TablesComponent { { def: "id", name: "ID", - cell: (cell: JobModel) => `${cell.id}` + cell: (cell: JobModel) => cell.id }, { def: "alias", name: "Alias", - cell: (cell: JobModel) => `${cell.alias}` + cell: (cell: JobModel) => cell.alias }, { def: "argv", name: "Cmd-line args", - cell: (cell: JobModel) => `${cell.argv}` + cell: (cell: JobModel) => cell.argv }, { def: "platform", name: "Platform", - cell: (cell: JobModel) => `${cell.platform}` + cell: (cell: JobModel) => cell.platform }, { def: "payload", @@ -39,8 +39,10 @@ export class JobComponent extends TablesComponent { { def: "etype", name: "Type", - cell: (cell: JobModel) => `${cell.exec_type}` + cell: (cell: JobModel) => cell.exec_type }, ] displayedColumns = this.columns.map((c) => c.def); + + show_item_dialog(obj: JobModel) { } } diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts index 964c8cd..3b7cade 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts @@ -1,6 +1,7 @@ import { Component, OnInit } from '@angular/core'; import { TablesComponent } from './table.component'; import { ResultModel } from '../models'; +import { epochToStr } from '../utils'; @Component({ selector: 'result-table', @@ -14,22 +15,22 @@ export class ResultComponent extends TablesComponent { { def: "id", name: "ID", - cell: (cell: ResultModel) => `${cell.id}` + cell: (cell: ResultModel) => cell.id }, { def: "alias", name: "Alias", - cell: (cell: ResultModel) => `${cell.alias}` + cell: (cell: ResultModel) => cell.alias }, { def: "agent_id", name: "Agent ID", - cell: (cell: ResultModel) => `${cell.agent_id}` + cell: (cell: ResultModel) => cell.agent_id }, { def: "job_id", name: "Job ID", - cell: (cell: ResultModel) => `${cell.job_id}` + cell: (cell: ResultModel) => cell.job_id }, { def: "state", @@ -39,8 +40,12 @@ export class ResultComponent extends TablesComponent { { def: "last_updated", name: "Last updated", - cell: (cell: ResultModel) => `${cell.updated.secs_since_epoch}` + cell: (cell: ResultModel) => epochToStr(cell.updated.secs_since_epoch) }, ] displayedColumns = this.columns.map((c) => c.def); + + show_item_dialog(obj: ResultModel) { + + } } diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.html index 777d508..c54abd9 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.html +++ b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.html @@ -23,7 +23,8 @@ - + No data diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less index 6a4475a..931742f 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less +++ b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less @@ -21,4 +21,9 @@ #refresh_btn { margin-left: 10px; +} + +.data-table-row:hover { + background: whitesmoke; + cursor: pointer; } \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts index 5581de6..39107d7 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts @@ -4,6 +4,7 @@ import { catchError, map, startWith, switchMap } from 'rxjs/operators'; import { HttpClient } from '@angular/common/http'; import { ApiTableService } from '../'; import { MatTableDataSource } from '@angular/material/table'; +import { MatDialog } from '@angular/material/dialog'; @Directive() export abstract class TablesComponent implements OnInit { @@ -13,7 +14,7 @@ export abstract class TablesComponent implements OnInit { isLoadingResults = true; - constructor(private _httpClient: HttpClient) { + constructor(public _httpClient: HttpClient, public info_dlg: MatDialog) { this.table_data = new MatTableDataSource; } @@ -37,7 +38,7 @@ export abstract class TablesComponent implements OnInit { this.isLoadingResults = false; if (data === null) { - return []; + return "no data returned" } // Only refresh the result length if there is new data. In case of rate @@ -54,6 +55,7 @@ export abstract class TablesComponent implements OnInit { this.table_data.filter = filterValue.trim().toLowerCase(); } + abstract show_item_dialog(obj: T): void; abstract columns: ColumnDef[]; abstract displayedColumns: string[]; diff --git a/bin/u_panel/src/server/fe/src/app/core/utils.ts b/bin/u_panel/src/server/fe/src/app/core/utils.ts new file mode 100644 index 0000000..27aadd8 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/utils.ts @@ -0,0 +1,3 @@ +export function epochToStr(epoch: number): string { + return new Date(epoch * 1000).toLocaleString('en-GB') +} \ No newline at end of file diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index d0929ca..8eb3a55 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -2,7 +2,7 @@ use crate::errors::{Error, SResult}; use diesel::{pg::PgConnection, prelude::*, result::Error as DslError}; use once_cell::sync::OnceCell; use serde::Deserialize; -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::{Mutex, MutexGuard}; use u_lib::{ models::{schema, Agent, AssignedJob, JobMeta, JobState}, utils::load_env, @@ -13,7 +13,7 @@ pub struct UDB { pub conn: PgConnection, } -static DB: OnceCell>> = OnceCell::new(); +static DB: OnceCell> = OnceCell::new(); #[derive(Deserialize)] struct DBEnv { @@ -34,7 +34,7 @@ impl UDB { let instance = UDB { conn: PgConnection::establish(&db_url).unwrap(), }; - Arc::new(Mutex::new(instance)) + Mutex::new(instance) }) .lock() .unwrap() diff --git a/images/integration-tests/u_db.Dockerfile b/images/integration-tests/u_db.Dockerfile index 783d887..8578b44 100644 --- a/images/integration-tests/u_db.Dockerfile +++ b/images/integration-tests/u_db.Dockerfile @@ -1,5 +1,6 @@ -FROM postgres:13.3 +FROM postgres:14.5 +ENV DEBIAN_FRONTEND=noninteractive RUN apt update && apt upgrade -y RUN apt install -y curl build-essential libpq-dev iproute2 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal @@ -8,4 +9,4 @@ RUN rustup target add x86_64-unknown-linux-musl RUN cargo install diesel_cli --no-default-features --features postgres RUN mkdir -p /unki -COPY u_db_entrypoint.sh /unki/ \ No newline at end of file +COPY u_db_entrypoint.sh /unki/ diff --git a/integration/docker_compose.py b/integration/docker_compose.py index 69ef302..0c445dd 100644 --- a/integration/docker_compose.py +++ b/integration/docker_compose.py @@ -15,7 +15,8 @@ class Compose: def __init__(self): self.container_tpl = 'integration_%s_%d' self.cmd_container = self.container_tpl % ('tests_runner', 1) - self.ALL_CONTAINERS = [self.container_tpl % (c, 1) for c in self.ALL_CONTAINERS] + self.ALL_CONTAINERS = [self.container_tpl % + (c, 1) for c in self.ALL_CONTAINERS] self.scaled_svc = {} self.scale("u_agent", 2) @@ -28,7 +29,7 @@ class Compose: def _call(self, *args): cmd = [ 'docker-compose', - '--no-ansi', + '--ansi=never', ] + list(args) log(f'Running docker-compose command: {cmd}') subprocess.check_call(cmd) diff --git a/integration/integration_tests.py b/integration/integration_tests.py index e4fe460..3402c9a 100644 --- a/integration/integration_tests.py +++ b/integration/integration_tests.py @@ -14,21 +14,35 @@ def abort_handler(s, _): cluster.down() +def usage_exit(): + usage = f"""Usage: + python {__file__.split('/')[-1]} [--rebuild] [--preserve] [--no-run]""" + + print(usage) + sys.exit(1) + + def run_tests(): - force_rebuild = '--rebuild' in sys.argv - preserve_containers = '--preserve' in sys.argv + allowed_args = set(["--rebuild", "--preserve", "--no-run"]) + args = sys.argv[1:] + if not set(args).issubset(allowed_args): + usage_exit() + force_rebuild = '--rebuild' in args + preserve_containers = '--preserve' in args + only_setup_cluster = '--no-run' in args for s in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): signal.signal(s, abort_handler) rebuild_images_if_needed(force_rebuild) try: cluster.up() cluster.is_alive() - cluster.run('cargo test --test integration') + if not only_setup_cluster: + cluster.run('cargo test --test integration') except Exception as e: err(e) sys.exit(1) finally: - if not preserve_containers: + if not preserve_containers and not only_setup_cluster: cluster.down() diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 7669da2..a860365 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -46,9 +46,9 @@ impl ClientHandler { } } - async fn _req( + async fn _req( &self, - url: impl AsRef + Debug, + url: impl AsRef, payload: P, ) -> Result { let request = self diff --git a/todos.txt b/todos.txt new file mode 100644 index 0000000..5be7b2a --- /dev/null +++ b/todos.txt @@ -0,0 +1,7 @@ +Upload/download files +More tests +Agent update (use more JobType's) +Erase log macros in release mode +Bump wine version to test agent on windows +Store downloaded payload on disk instead of memory +Improve web interface \ No newline at end of file From 6fe0e71959480760d30a19c64e0a2bc0bb596e70 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Mon, 26 Sep 2022 02:50:10 +0500 Subject: [PATCH 26/30] refactor again - deduplicate deps - simplify & unify Job* interfaces - unify api with OneOrVec - remove some Display impls - remove ips db table --- Cargo.toml | 10 + bin/u_agent/Cargo.toml | 10 +- bin/u_agent/src/lib.rs | 42 ++-- bin/u_panel/Cargo.toml | 27 ++- bin/u_panel/src/argparse.rs | 8 +- bin/u_run/Cargo.toml | 2 +- bin/u_server/Cargo.toml | 19 +- bin/u_server/src/db.rs | 2 + bin/u_server/src/handlers.rs | 42 ++-- bin/u_server/src/u_server.rs | 4 +- .../integration-tests/tests_runner.Dockerfile | 3 +- images/integration-tests/u_db.Dockerfile | 5 + integration/Cargo.toml | 22 +- integration/docker-compose.yml | 23 +- integration/docker.py | 79 +++++++ integration/docker_compose.py | 72 ------- integration/integration_tests.py | 55 +++-- integration/tests/fixtures/agent.rs | 11 +- lib/u_lib/Cargo.toml | 41 ++-- lib/u_lib/src/api.rs | 21 +- lib/u_lib/src/errors/chan.rs | 5 +- lib/u_lib/src/errors/variants.rs | 11 +- lib/u_lib/src/executor.rs | 51 +++-- lib/u_lib/src/lib.rs | 2 +- lib/u_lib/src/messaging/base.rs | 5 +- lib/u_lib/src/messaging/mod.rs | 12 +- lib/u_lib/src/models/agent.rs | 44 ++-- lib/u_lib/src/models/jobs/assigned.rs | 119 ++++------ lib/u_lib/src/models/jobs/meta.rs | 117 +++------- lib/u_lib/src/models/jobs/misc.rs | 32 +-- lib/u_lib/src/models/schema.rs | 42 +--- lib/u_lib/src/{builder.rs => runner.rs} | 203 +++++++++++------- lib/u_lib/src/utils/fmt/stripped.rs | 7 +- lib/u_lib/src/utils/mod.rs | 2 - lib/u_lib/src/utils/platform.rs | 4 + .../2020-10-24-111622_create_all/up.sql | 22 +- 36 files changed, 584 insertions(+), 592 deletions(-) delete mode 100644 integration/docker_compose.py rename lib/u_lib/src/{builder.rs => runner.rs} (54%) diff --git a/Cargo.toml b/Cargo.toml index 02c9d71..ece9d2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,16 @@ members = [ "integration" ] +[workspace.dependencies] +reqwest = { version = "0.11", features = ["json"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1.11", features = ["macros"] } +tracing = "0.1.35" +tracing-appender = "0.2.0" +tracing-subscriber = { version = "0.3.0", features = ["env-filter"]} +uuid = "0.6.5" + [profile.release] panic = "abort" strip = "symbols" diff --git a/bin/u_agent/Cargo.toml b/bin/u_agent/Cargo.toml index a7cfb91..77259f2 100644 --- a/bin/u_agent/Cargo.toml +++ b/bin/u_agent/Cargo.toml @@ -7,10 +7,10 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process", "time"] } -sysinfo = "0.10.5" log = "^0.4" -uuid = "0.6.5" -reqwest = { version = "0.11", features = ["json"] } -u_lib = { version = "*", path = "../../lib/u_lib" } +reqwest = { workspace = true } +sysinfo = "0.10.5" +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "time"] } +uuid = { workspace = true } +u_lib = { path = "../../lib/u_lib" } diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 597df48..1725e58 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -9,16 +9,16 @@ extern crate log; use std::sync::Arc; use tokio::time::{sleep, Duration}; use u_lib::{ - api::ClientHandler, builder::JobBuilder, cache::JobCache, config::get_self_uid, - errors::ErrChan, executor::pop_completed, logging::init_logger, messaging::Reportable, - models::AssignedJob, utils::load_env_default, UError, + api::ClientHandler, cache::JobCache, config::get_self_uid, errors::ErrChan, + executor::pop_completed, logging::init_logger, messaging::Reportable, models::AssignedJobById, + runner::JobRunner, utils::load_env_default, }; const ITERATION_LATENCY: u64 = 5; -pub async fn process_request(job_requests: Vec, client: &ClientHandler) { - if !job_requests.is_empty() { - for jr in &job_requests { +pub async fn process_request(jobs: Vec, client: &ClientHandler) { + if !jobs.is_empty() { + for jr in &jobs { if !JobCache::contains(jr.job_id) { info!("Fetching job: {}", &jr.job_id); let fetched_job = loop { @@ -35,20 +35,19 @@ pub async fn process_request(job_requests: Vec, client: &ClientHand } info!( "Scheduling jobs: {}", - job_requests - .iter() + jobs.iter() .map(|j| j.job_id.to_string()) .collect::>() .join(", ") ); - let mut builder = JobBuilder::from_request(job_requests); - let errors = builder.pop_errors(); + let mut runner = JobRunner::from_jobs(jobs); + let errors = runner.pop_errors(); if !errors.is_empty() { for e in errors { ErrChan::send(e, "ebld").await; } } - builder.unwrap_one().spawn().await; + runner.unwrap_one().spawn().await; } } @@ -57,7 +56,7 @@ async fn error_reporting(client: Arc) -> ! { match ErrChan::recv().await { Some(err) => { 'retry: for _ in 0..3 { - match client.report(&[Reportable::Error(err.to_string())]).await { + match client.report(Reportable::Error(err.clone())).await { Ok(_) => break 'retry, Err(e) => { debug!("Reporting error: {:?}", e); @@ -71,17 +70,24 @@ async fn error_reporting(client: Arc) -> ! { } } -async fn do_stuff(client: Arc) -> ! { +async fn agent_loop(client: Arc) -> ! { loop { match client.get_personal_jobs(get_self_uid()).await { - Ok(resp) => { - process_request(resp, &client).await; + Ok(jobs) => { + process_request(jobs, &client).await; } Err(err) => ErrChan::send(err, "processing").await, } - let result: Vec = pop_completed().await.into_iter().collect(); + let result: Vec = pop_completed() + .await + .into_iter() + .map(|result| match result { + Ok(r) => Reportable::Assigned(r), + Err(e) => Reportable::Error(e), + }) + .collect(); if !result.is_empty() { - if let Err(err) = client.report(&result).await { + if let Err(err) = client.report(result).await { ErrChan::send(err, "report").await; } } @@ -110,5 +116,5 @@ pub async fn run_forever() -> ! { // } } info!("Startup"); - do_stuff(client).await + agent_loop(client).await } diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 1e69c0c..618d1bc 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -7,23 +7,22 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +actix-cors = "0.6.1" actix-web = "4.1" -structopt = "0.3.21" -uuid = "0.6.5" -serde_json = "1.0.4" -serde = { version = "1.0.114", features = ["derive"] } -tokio = { version = "1.11.0", features = ["rt", "rt-multi-thread"] } -u_lib = { version = "*", path = "../../lib/u_lib", features = ["panel"] } anyhow = "1.0.44" -strum = { version = "0.22.0", features = ["derive"] } +futures-util = "0.3.21" +mime_guess = "2.0.4" once_cell = "1.8.0" -tracing = "0.1.29" -tracing-subscriber = { version = "0.3.3", features = ["env-filter"]} -signal-hook = "0.3.12" -tracing-appender = "0.2.0" rust-embed = { version = "6.3.0", features = ["debug-embed", "compression"] } -mime_guess = "2.0.4" +serde = { workspace = true } +serde_json = { workspace = true } +strum = { version = "0.22.0", features = ["derive"] } +tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +tracing-appender = { workspace = true } shlex = "1.1.0" +structopt = "0.3.21" thiserror = "1.0.31" -futures-util = "0.3.21" -actix-cors = "0.6.1" +uuid = { workspace = true } +u_lib = { version = "*", path = "../../lib/u_lib", features = ["panel"] } diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index b414fe7..ce810ba 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -4,7 +4,7 @@ use u_lib::{ api::ClientHandler, datatypes::PanelResult, messaging::AsMsg, - models::{Agent, AssignedJob, JobMeta, RawJobMeta}, + models::{Agent, AssignedJob, JobMeta}, UError, UResult, }; use uuid::Uuid; @@ -96,9 +96,9 @@ pub async fn process_cmd(client: ClientHandler, args: Args) -> UResult { }, Cmd::Jobs(action) => match action { JobCRUD::Create { job } => { - let raw_job = serde_json::from_str::(&job)?; + let raw_job = serde_json::from_str::(&job)?; let job = raw_job.into_builder().build()?; - to_json(client.upload_jobs(&[job]).await) + to_json(client.upload_jobs(job).await) } JobCRUD::RUD(RUD::Read { uid }) => to_json(client.get_jobs(uid).await), JobCRUD::RUD(RUD::Update { item }) => { @@ -111,7 +111,7 @@ pub async fn process_cmd(client: ClientHandler, args: Args) -> UResult { JobMapCRUD::Create { agent_uid, job_idents, - } => to_json(client.set_jobs(agent_uid, &job_idents).await), + } => to_json(client.set_jobs(agent_uid, job_idents).await), JobMapCRUD::RUD(RUD::Read { uid }) => to_json(client.get_agent_jobs(uid).await), JobMapCRUD::RUD(RUD::Update { item }) => { let assigned = serde_json::from_str::(&item)?; diff --git a/bin/u_run/Cargo.toml b/bin/u_run/Cargo.toml index 9756ea2..1c3b145 100644 --- a/bin/u_run/Cargo.toml +++ b/bin/u_run/Cargo.toml @@ -7,5 +7,5 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -nix = "0.17" libc = "^0.2" +nix = "0.17" diff --git a/bin/u_server/Cargo.toml b/bin/u_server/Cargo.toml index 5b07891..b825bbf 100644 --- a/bin/u_server/Cargo.toml +++ b/bin/u_server/Cargo.toml @@ -5,17 +5,18 @@ name = "u_server" version = "0.1.0" [dependencies] -tracing = "0.1.35" -thiserror = "*" -warp = { version = "0.3.1", features = ["tls"] } -uuid = { version = "0.6.5", features = ["serde", "v4"] } -once_cell = "1.7.2" +diesel = { version = "1.4.5", features = ["postgres", "uuid"] } hyper = "0.14" +once_cell = "1.7.2" openssl = "*" -diesel = { version = "1.4.5", features = ["postgres", "uuid"] } -serde = { version = "1.0", features = ["derive"] } -tokio = { version = "1.9", features = ["macros"] } -u_lib = { path = "../../lib/u_lib", version = "*", features = ["server"] } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = "*" +tracing = { workspace = true } +tokio = { workspace = true, features = ["macros"] } +uuid = { workspace = true, features = ["serde", "v4"] } +u_lib = { path = "../../lib/u_lib", features = ["server"] } +warp = { version = "0.3.1", features = ["tls"] } [dev-dependencies] diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index 8eb3a55..09dcad3 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -11,6 +11,7 @@ use uuid::Uuid; pub struct UDB { pub conn: PgConnection, + __p: (), } static DB: OnceCell> = OnceCell::new(); @@ -33,6 +34,7 @@ impl UDB { ); let instance = UDB { conn: PgConnection::establish(&db_url).unwrap(), + __p: (), }; Mutex::new(instance) }) diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index 4e1eebe..3b0db33 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -6,7 +6,7 @@ use diesel::SaveChangesDsl; use u_lib::{ messaging::{AsMsg, BaseMessage, Reportable}, models::*, - utils::{OneOrVec, Stripped}, + utils::OneOrVec, }; use uuid::Uuid; use warp::Rejection; @@ -88,36 +88,46 @@ impl Endpoints { msg: BaseMessage<'static, Data>, ) -> EndpResult<()> { let id = msg.id; - let mut failed = vec![]; for entry in msg.into_inner().into_vec() { match entry { Reportable::Assigned(mut result) => { - if id != result.agent_id { + let result_agent_id = &result.agent_id; + if id != *result_agent_id { + warn!("Ids are not equal! actual id: {id}, job id: {result_agent_id}"); continue; } result.state = JobState::Finished; result.updated = SystemTime::now(); + match result.exec_type { + JobType::Init => match &result.result { + Some(rbytes) => { + let mut agent: Agent = match serde_json::from_slice(&rbytes) { + Ok(a) => a, + Err(e) => { + warn!("Error deserializing agent from {id}: {e}"); + continue; + } + }; + agent.state = AgentState::Active; + Self::add_agent(agent).await?; + } + None => warn!("Empty agent data"), + }, + JobType::Shell => (), + JobType::Terminate => todo!(), + JobType::Update => todo!(), + } let db = UDB::lock_db(); - if let Err(e) = result + result .save_changes::(&db.conn) - .map_err(Error::from) - { - failed.push(e.to_string()) - } - } - Reportable::Agent(mut a) => { - a.state = AgentState::Active; - Self::add_agent(a).await?; + .map_err(Error::from)?; } Reportable::Error(e) => { - warn!("{} reported an error: {}", id, Stripped(&e.as_str())); + warn!("{} reported an error: {}", id, e); } Reportable::Dummy => (), } } - if !failed.is_empty() { - return Err(Error::ProcessingError(failed.join(", ")).into()); - } Ok(()) } diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index d2e84d5..8c11e0a 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -156,8 +156,8 @@ pub fn prefill_jobs() -> SResult<()> { match if_job_exists { Ok(_) => Ok(()), Err(Error::DBError(diesel::result::Error::NotFound)) => { - let agent_hello = RawJobMeta::builder() - .with_type(misc::JobType::Manage) + let agent_hello = JobMeta::builder() + .with_type(JobType::Init) .with_alias(job_alias) .build() .unwrap(); diff --git a/images/integration-tests/tests_runner.Dockerfile b/images/integration-tests/tests_runner.Dockerfile index 4714c7d..cefab17 100644 --- a/images/integration-tests/tests_runner.Dockerfile +++ b/images/integration-tests/tests_runner.Dockerfile @@ -1,4 +1,5 @@ -FROM rust:1.62 +FROM rust:1.64 RUN rustup target add x86_64-unknown-linux-musl +RUN mkdir -p /tests && chmod 777 /tests CMD ["sleep", "3600"] \ No newline at end of file diff --git a/images/integration-tests/u_db.Dockerfile b/images/integration-tests/u_db.Dockerfile index 8578b44..5f1812f 100644 --- a/images/integration-tests/u_db.Dockerfile +++ b/images/integration-tests/u_db.Dockerfile @@ -1,6 +1,7 @@ FROM postgres:14.5 ENV DEBIAN_FRONTEND=noninteractive + RUN apt update && apt upgrade -y RUN apt install -y curl build-essential libpq-dev iproute2 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal @@ -9,4 +10,8 @@ RUN rustup target add x86_64-unknown-linux-musl RUN cargo install diesel_cli --no-default-features --features postgres RUN mkdir -p /unki +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 +RUN apt install -y locales locales-all COPY u_db_entrypoint.sh /unki/ diff --git a/integration/Cargo.toml b/integration/Cargo.toml index eda6095..706d97d 100644 --- a/integration/Cargo.toml +++ b/integration/Cargo.toml @@ -7,20 +7,16 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process", "time"] } -tracing = "0.1.35" -uuid = { version = "0.6.5", features = ["serde", "v4"] } -reqwest = { version = "0.11", features = ["json"] } -serde_json = "1.0" -serde = { version = "1.0.114", features = ["derive"] } -shlex = "1.0.0" -rstest = "0.12" once_cell = "1.10.0" - -[dependencies.u_lib] -path = "../lib/u_lib" -version = "*" -features = ["panel"] +reqwest = { workspace = true } +rstest = "0.12" +serde = { workspace = true } +serde_json = { workspace = true } +shlex = "1.0.0" +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "time"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["serde", "v4"] } +u_lib = { path = "../lib/u_lib", features = ["panel"] } [[test]] diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml index 04bc87b..d148d39 100644 --- a/integration/docker-compose.yml +++ b/integration/docker-compose.yml @@ -1,4 +1,8 @@ -version: "2.1" +version: "3.4" + +x-global: + user: &user + "${DOCKER_UID:-1000}:${DOCKER_GID:-1000}" networks: u_net: @@ -6,7 +10,7 @@ networks: services: u_server: - user: "${DOCKER_UID:-1000}:${DOCKER_GID:-1000}" + user: *user image: unki/u_server networks: - u_net @@ -53,7 +57,7 @@ services: retries: 3 u_agent: - user: "${DOCKER_UID:-1000}:${DOCKER_GID:-1000}" + user: *user image: unki/u_agent networks: - u_net @@ -71,18 +75,19 @@ services: condition: service_healthy tests_runner: - user: "${DOCKER_UID:-1000}:${DOCKER_GID:-1000}" + user: *user image: unki/tests_runner networks: - u_net volumes: - - ./:/tests/ - - ../certs:/certs + - ../__Cargo_integration.toml:/tests/Cargo.toml + - ./:/tests/integration/ + - ../certs:/tests/certs - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_panel:/u_panel - - ../lib/u_lib:/lib/u_lib - - ../logs:/tests/logs:rw + - ../lib/u_lib:/tests/lib/u_lib + - ../logs:/tests/integration/logs:rw working_dir: - /tests/ + /tests/integration/ depends_on: u_agent: condition: service_started diff --git a/integration/docker.py b/integration/docker.py index 05182bd..355e517 100644 --- a/integration/docker.py +++ b/integration/docker.py @@ -1,4 +1,5 @@ import subprocess +import shlex from utils import * @@ -78,3 +79,81 @@ def rebuild_images_if_needed(force_rebuild=False): if force_rebuild: cmd += ['--no-cache'] docker(cmd) + + +class Compose: + ALL_IMAGES = [ + 'u_agent', + 'u_server', + 'u_db', + 'tests_runner', + ] + + def __init__(self): + self.container_tpl = 'integration_%s_%d' + self.cmd_container = self.container_tpl % ('tests_runner', 1) + self.ALL_CONTAINERS = [self.container_tpl % + (c, 1) for c in self.ALL_IMAGES] + self.scaled_svc = {} + self.scale("u_agent", 2) + + def scale(self, svc, count): + for c in range(1, count): + new_container = self.container_tpl % (svc, c + 1) + self.ALL_CONTAINERS.append(new_container) + self.scaled_svc[svc] = count + + def _call(self, *args): + cmd = [ + 'docker-compose', + '--ansi=never', + ] + list(args) + log(f'Running docker-compose command: {cmd}') + subprocess.check_call(cmd) + + def up(self): + log(f'Instanciating cluster: {self.ALL_CONTAINERS}') + scaled = [f"{k}={v}" for k, v in self.scaled_svc.items()] + if len(scaled) > 0: + scaled.insert(0, '--scale') + self._call('up', '-d', *scaled) + + def down(self): + log('Shutting down cluster') + self._call('down') + + def stop(self): + log('Stopping cluster') + self._call('stop') + + def run(self, cmd): + container = self.cmd_container + if isinstance(cmd, str): + cmd = shlex.split(cmd) + result = docker([ + 'exec', + '-ti', + container + ] + cmd) + return result + + def is_alive(self): + log('Check if all containers are alive') + + errors = check_state(self.ALL_CONTAINERS) + + if errors: + print_errors(errors) + raise TestsError('Error during `is_alive` check') + else: + log('All containers are alive') + + def print_containers_logs(self): + for container in self.ALL_CONTAINERS: + try: + docker([ + 'logs', + container + ]) + except Exception: + pass diff --git a/integration/docker_compose.py b/integration/docker_compose.py deleted file mode 100644 index 0c445dd..0000000 --- a/integration/docker_compose.py +++ /dev/null @@ -1,72 +0,0 @@ -import subprocess -import shlex -from utils import * -from docker import docker, check_state, print_errors - - -class Compose: - ALL_CONTAINERS = [ - 'u_agent', - 'u_server', - 'u_db', - 'tests_runner', - ] - - def __init__(self): - self.container_tpl = 'integration_%s_%d' - self.cmd_container = self.container_tpl % ('tests_runner', 1) - self.ALL_CONTAINERS = [self.container_tpl % - (c, 1) for c in self.ALL_CONTAINERS] - self.scaled_svc = {} - self.scale("u_agent", 2) - - def scale(self, svc, count): - for c in range(1, count): - new_container = self.container_tpl % (svc, c + 1) - self.ALL_CONTAINERS.append(new_container) - self.scaled_svc[svc] = count - - def _call(self, *args): - cmd = [ - 'docker-compose', - '--ansi=never', - ] + list(args) - log(f'Running docker-compose command: {cmd}') - subprocess.check_call(cmd) - - def up(self): - log('Instanciating cluster') - scaled = [f"{k}={v}" for k, v in self.scaled_svc.items()] - if len(scaled) > 0: - scaled.insert(0, '--scale') - self._call('up', '-d', *scaled) - - def down(self): - log('Shutting down cluster') - self._call('down') - - def stop(self): - log('Stopping cluster') - self._call('stop') - - def run(self, cmd): - container = self.cmd_container - if isinstance(cmd, str): - cmd = shlex.split(cmd) - result = docker([ - 'exec', - '-ti', - container - ] + cmd) - return result - - def is_alive(self): - log('Check if all containers are alive') - - errors = check_state(self.ALL_CONTAINERS) - - if errors: - print_errors(errors) - raise TestsError('Error during `is_alive` check') - else: - log('All containers are alive') diff --git a/integration/integration_tests.py b/integration/integration_tests.py index 3402c9a..43c54ff 100644 --- a/integration/integration_tests.py +++ b/integration/integration_tests.py @@ -1,49 +1,68 @@ import signal import sys -from utils import * -from docker import rebuild_images_if_needed -from docker_compose import Compose +import toml +from docker import rebuild_images_if_needed, Compose +from pathlib import Path +from utils import * -cluster = Compose() +CARGO_INTEGRATION_TOML = Path('../__Cargo_integration.toml') +CLUSTER = Compose() -def abort_handler(s, _): - warn(f'Received signal: {s}') - warn(f'Gracefully stopping...') - cluster.down() +def fail(msg): + err(msg) + sys.exit(1) def usage_exit(): usage = f"""Usage: python {__file__.split('/')[-1]} [--rebuild] [--preserve] [--no-run]""" - print(usage) - sys.exit(1) + fail(usage) + + +def create_integration_workspace(): + if CARGO_INTEGRATION_TOML.exists(): + CARGO_INTEGRATION_TOML.unlink() + workspace = toml.load('../Cargo.toml') + workspace['workspace']['members'] = ['integration'] + with open(CARGO_INTEGRATION_TOML, 'w') as fo: + toml.dump(workspace, fo) def run_tests(): - allowed_args = set(["--rebuild", "--preserve", "--no-run"]) + allowed_args = set(["--rebuild", "--preserve", "--no-run", "--release"]) args = sys.argv[1:] if not set(args).issubset(allowed_args): usage_exit() force_rebuild = '--rebuild' in args preserve_containers = '--preserve' in args only_setup_cluster = '--no-run' in args + + def _cleanup(): + if not preserve_containers and not only_setup_cluster: + CLUSTER.down() + CARGO_INTEGRATION_TOML.unlink(missing_ok=True) + + def abort_handler(s, _): + warn(f'Received signal: {s}, gracefully stopping...') + _cleanup() + for s in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): signal.signal(s, abort_handler) rebuild_images_if_needed(force_rebuild) + create_integration_workspace() try: - cluster.up() - cluster.is_alive() + CLUSTER.up() + CLUSTER.is_alive() if not only_setup_cluster: - cluster.run('cargo test --test integration') + CLUSTER.run('cargo test --test integration') except Exception as e: - err(e) - sys.exit(1) + # CLUSTER.print_containers_logs() + fail(e) finally: - if not preserve_containers and not only_setup_cluster: - cluster.down() + _cleanup() if __name__ == '__main__': diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs index b80366d..1096aac 100644 --- a/integration/tests/fixtures/agent.rs +++ b/integration/tests/fixtures/agent.rs @@ -24,12 +24,13 @@ pub async fn register_agent() -> RegisteredAgent { .pop() .unwrap(); let job_id = resp.job_id; - let resp = cli.get_jobs(Some(job_id)).await.unwrap().pop().unwrap(); - assert_eq!(resp.alias, Some("agent_hello".to_string())); - let agent_data = Agent { + let job = cli.get_jobs(Some(job_id)).await.unwrap().pop().unwrap(); + assert_eq!(job.alias, Some("agent_hello".to_string())); + let mut agent_data = AssignedJob::from(&job); + agent_data.set_result(&Agent { id: agent_uid, ..Default::default() - }; - cli.report(&[Reportable::Agent(agent_data)]).await.unwrap(); + }); + cli.report(Reportable::Assigned(agent_data)).await.unwrap(); RegisteredAgent { uid: agent_uid } } diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index 63de9f6..27e2c96 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -7,37 +7,34 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -dotenv = "0.15.0" -serde = { version = "1.0.114", features = ["derive"] } -uuid = { version = "0.6.5", features = ["serde", "v4"] } -nix = "0.17" -libc = "^0.2" -lazy_static = "1.4.0" -futures = "0.3.5" -thiserror = "*" -diesel-derive-enum = { version = "1", features = ["postgres"], optional = true } +anyhow = "1.0.58" chrono = "0.4.19" -strum = { version = "0.20", features = ["derive"] } -once_cell = "1.7.2" -shlex = "1.0.0" -crossbeam = "0.8.1" diesel = { version = "1.4.5", features = ["postgres", "uuid"], optional = true } +diesel-derive-enum = { version = "1", features = ["postgres"], optional = true } +dotenv = "0.15.0" envy = "0.4.2" -serde_json = "1.0.81" -tracing-subscriber = { version = "0.3.14", features = ["env-filter"] } -tracing-appender = "0.2.2" +futures = "0.3.5" +guess_host_triple = "0.1.2" +libc = "^0.2" +lazy_static = "1.4.0" log = "*" -anyhow = "1.0.58" +nix = "0.17" +once_cell = "1.7.2" platforms = "3.0.1" +reqwest = { workspace = true, features = ["native-tls"] } +shlex = "1.0.0" +serde = { workspace = true } +serde_json = { workspace = true } +strum = { version = "0.20", features = ["derive"] } +thiserror = "*" +tokio = { workspace = true, features = ["rt-multi-thread", "sync", "macros", "process", "time"] } +tracing-appender = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } +uuid = { workspace = true, features = ["serde", "v4"] } [features] panel = [] server = ["dep:diesel", "dep:diesel-derive-enum"] -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -reqwest = { version = "0.11", features = ["json", "native-tls"] } -tokio = { version = "1.2.0", features = ["rt-multi-thread", "sync", "macros", "process", "time"] } -guess_host_triple = "0.1.2" - [dev-dependencies] rstest = "0.12" diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index a860365..be723b4 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -5,7 +5,7 @@ use crate::{ config::MASTER_PORT, messaging::{self, AsMsg, BaseMessage, Empty}, models::{self}, - utils::opt_to_string, + utils::{opt_to_string, OneOrVec}, UError, }; use anyhow::{Context, Result}; @@ -78,14 +78,14 @@ impl ClientHandler { } // get jobs for client - pub async fn get_personal_jobs(&self, url_param: Uuid) -> Result> { + pub async fn get_personal_jobs(&self, url_param: Uuid) -> Result> { self._req(format!("get_personal_jobs/{}", url_param), Empty) .await } // send something to server - pub async fn report(&self, payload: &[messaging::Reportable]) -> Result { - self._req("report", payload).await + pub async fn report(&self, payload: impl OneOrVec) -> Result { + self._req("report", payload.into_vec()).await } // download file @@ -115,8 +115,8 @@ impl ClientHandler { } /// create and upload job - pub async fn upload_jobs(&self, payload: &[models::JobMeta]) -> Result { - self._req("upload_jobs", payload).await + pub async fn upload_jobs(&self, payload: impl OneOrVec) -> Result { + self._req("upload_jobs", payload.into_vec()).await } /// delete something @@ -125,8 +125,13 @@ impl ClientHandler { } /// set jobs for any agent - pub async fn set_jobs(&self, agent: Uuid, job_idents: &[String]) -> Result> { - self._req(format!("set_jobs/{agent}"), job_idents).await + pub async fn set_jobs( + &self, + agent: Uuid, + job_idents: impl OneOrVec, + ) -> Result> { + self._req(format!("set_jobs/{agent}"), job_idents.into_vec()) + .await } /// get jobs for any agent diff --git a/lib/u_lib/src/errors/chan.rs b/lib/u_lib/src/errors/chan.rs index 10668a4..ef3b7ea 100644 --- a/lib/u_lib/src/errors/chan.rs +++ b/lib/u_lib/src/errors/chan.rs @@ -1,3 +1,4 @@ +use crate::UError; use anyhow::Error; use once_cell::sync::OnceCell; use tokio::sync::mpsc::{channel, error::TryRecvError, Receiver, Sender}; @@ -28,9 +29,9 @@ impl ErrChan { Self::get().await.tx.try_send(err).unwrap(); } - pub async fn recv() -> Option { + pub async fn recv() -> Option { match Self::get().await.rx.try_recv() { - Ok(r) => Some(r), + Ok(err) => Some(UError::from(err)), Err(TryRecvError::Disconnected) => panic!("err chan disconnected"), Err(TryRecvError::Empty) => None, } diff --git a/lib/u_lib/src/errors/variants.rs b/lib/u_lib/src/errors/variants.rs index 2a9a48f..ff65f45 100644 --- a/lib/u_lib/src/errors/variants.rs +++ b/lib/u_lib/src/errors/variants.rs @@ -6,7 +6,7 @@ use uuid::Uuid; pub type UResult = std::result::Result; -#[derive(Error, Debug, Serialize, Deserialize, Clone)] +#[derive(PartialEq, Eq, Error, Debug, Serialize, Deserialize, Clone)] pub enum UError { #[error("Runtime error: {0}")] Runtime(String), @@ -60,3 +60,12 @@ impl From for UError { UError::DeserializeError(e.to_string()) } } + +impl From for UError { + fn from(e: anyhow::Error) -> Self { + match e.downcast::() { + Ok(err) => err, + Err(err) => UError::Runtime(err.to_string()), + } + } +} diff --git a/lib/u_lib/src/executor.rs b/lib/u_lib/src/executor.rs index c34e08b..169002f 100644 --- a/lib/u_lib/src/executor.rs +++ b/lib/u_lib/src/executor.rs @@ -1,15 +1,16 @@ -use crate::{messaging::Reportable, utils::OneOrVec}; +use crate::{models::AssignedJob, UResult}; use futures::{future::BoxFuture, lock::Mutex}; use lazy_static::lazy_static; use std::collections::HashMap; +use std::future::Future; use tokio::{ - spawn, + runtime::Handle, sync::mpsc::{channel, Receiver, Sender}, - task::JoinHandle, + task::{spawn, spawn_blocking, JoinHandle}, }; use uuid::Uuid; -pub type DynFut = BoxFuture<'static, Reportable>; +pub type ExecResult = UResult; lazy_static! { static ref FUT_RESULTS: Mutex> = Mutex::new(HashMap::new()); @@ -21,33 +22,45 @@ lazy_static! { } struct JoinInfo { - handle: JoinHandle, + handle: JoinHandle>, completed: bool, collectable: bool, // indicates if future can be popped from pool via pop_task_if_completed } +impl JoinInfo { + async fn wait_result(self) -> ExecResult { + self.handle.await.unwrap().await.unwrap() + } +} + fn get_sender() -> Sender { FUT_CHANNEL.0.clone() } pub struct Waiter { - tasks: Vec, + tasks: Vec>, fids: Vec, } impl Waiter { - pub fn new(tasks: impl OneOrVec) -> Self { + pub fn new() -> Self { Self { - tasks: tasks.into_vec(), + tasks: vec![], fids: vec![], } } + pub fn push(&mut self, task: impl Future + Send + 'static) { + self.tasks.push(Box::pin(task)); + } + + /// Spawn prepared tasks pub async fn spawn(mut self) -> Self { let collectable = true; //TODO: self.tasks.len() != 1; for f in self.tasks.drain(..) { - let tx = get_sender(); + let handle = Handle::current(); let fid = Uuid::new_v4(); + let tx = get_sender(); self.fids.push(fid); let task_wrapper = async move { debug!("inside wrapper (started): {}", fid); @@ -56,7 +69,7 @@ impl Waiter { result }; let handler = JoinInfo { - handle: spawn(task_wrapper), + handle: spawn_blocking(move || handle.spawn(task_wrapper)), completed: false, collectable, }; @@ -68,12 +81,11 @@ impl Waiter { /// Wait until a bunch of tasks is finished. /// NOT GUARANTEED that all tasks will be returned due to /// possibility to pop them in other places - pub async fn wait(self) -> Vec { + pub async fn wait(self) -> Vec { let mut result = vec![]; for fid in self.fids { if let Some(task) = pop_task(fid).await { - let r = task.handle.await; - result.push(r.unwrap()); + result.push(task.wait_result().await); } } result @@ -94,26 +106,25 @@ async fn pop_task(fid: Uuid) -> Option { FUT_RESULTS.lock().await.remove(&fid) } -pub async fn pop_task_if_completed(fid: Uuid) -> Option { - let &mut JoinInfo { +pub async fn pop_task_if_completed(fid: Uuid) -> Option { + let &JoinInfo { handle: _, collectable, completed, - } = match FUT_RESULTS.lock().await.get_mut(&fid) { + } = match FUT_RESULTS.lock().await.get(&fid) { Some(t) => t, None => return None, }; if collectable && completed { let task = pop_task(fid).await.unwrap(); - let result = task.handle.await.unwrap(); - Some(result) + Some(task.wait_result().await) } else { None } } -pub async fn pop_completed() -> Vec { - let mut completed: Vec = vec![]; +pub async fn pop_completed() -> Vec { + let mut completed: Vec = vec![]; let fids = FUT_RESULTS .lock() .await diff --git a/lib/u_lib/src/lib.rs b/lib/u_lib/src/lib.rs index b6a94f8..62b1f0b 100644 --- a/lib/u_lib/src/lib.rs +++ b/lib/u_lib/src/lib.rs @@ -3,7 +3,6 @@ #[path = "."] pub mod exports { pub mod api; - pub mod builder; pub mod cache; pub mod config; pub mod datatypes; @@ -12,6 +11,7 @@ pub mod exports { pub mod logging; pub mod messaging; pub mod models; + pub mod runner; pub mod utils; } diff --git a/lib/u_lib/src/messaging/base.rs b/lib/u_lib/src/messaging/base.rs index 06b73a6..862f1ff 100644 --- a/lib/u_lib/src/messaging/base.rs +++ b/lib/u_lib/src/messaging/base.rs @@ -1,8 +1,8 @@ use crate::config::get_self_uid; -use crate::utils::VecDisplay; +//use crate::utils::VecDisplay; use serde::{Deserialize, Serialize}; use std::borrow::Cow; -use std::fmt::Display; +//use std::fmt::Display; use uuid::Uuid; pub struct Moo<'cow, T: AsMsg + Clone>(pub Cow<'cow, T>); @@ -28,7 +28,6 @@ impl<'cow, M: AsMsg> From<&'cow M> for Moo<'cow, M> { } impl AsMsg for Vec {} -impl AsMsg for VecDisplay {} impl<'msg, M: AsMsg> AsMsg for &'msg [M] {} #[derive(Serialize, Deserialize, Debug)] diff --git a/lib/u_lib/src/messaging/mod.rs b/lib/u_lib/src/messaging/mod.rs index e79ef34..b1342bf 100644 --- a/lib/u_lib/src/messaging/mod.rs +++ b/lib/u_lib/src/messaging/mod.rs @@ -2,14 +2,15 @@ mod base; mod files; use crate::models::*; +use crate::UError; pub use base::{AsMsg, BaseMessage}; pub use files::*; use serde::{Deserialize, Serialize}; -use std::fmt; use uuid::Uuid; impl AsMsg for Agent {} impl AsMsg for AssignedJob {} +impl AsMsg for AssignedJobById {} impl AsMsg for DownloadInfo {} impl AsMsg for Reportable {} impl AsMsg for JobMeta {} @@ -22,16 +23,9 @@ impl AsMsg for u8 {} #[derive(Serialize, Deserialize, Clone, Default, Debug)] pub struct Empty; -impl fmt::Display for Empty { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "") - } -} - #[derive(Serialize, Deserialize, Clone, PartialEq, Debug)] pub enum Reportable { Assigned(AssignedJob), - Agent(Agent), Dummy, - Error(String), + Error(UError), } diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index 71483ac..b00a677 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -6,14 +6,13 @@ use serde::{Deserialize, Serialize}; use std::{fmt, time::SystemTime}; use strum::Display; -#[cfg(not(target_arch = "wasm32"))] -use crate::builder::NamedJobBuilder; #[cfg(feature = "server")] use crate::models::schema::*; + use crate::{ config::get_self_uid, - messaging::Reportable, - unwrap_enum, + executor::ExecResult, + runner::NamedJobRunner, utils::{systime_to_string, Platform}, }; @@ -43,6 +42,8 @@ pub struct Agent { pub alias: Option, pub hostname: String, pub id: Uuid, + pub ip_gray: Option, + pub ip_white: Option, pub is_root: bool, pub is_root_allowed: bool, pub last_active: SystemTime, @@ -55,18 +56,17 @@ pub struct Agent { impl fmt::Display for Agent { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut out = format!("Agent: {}", self.id); + write!(f, "Agent: {}", self.id)?; if let Some(ref alias) = self.alias { - out += &format!(" ({})", alias) + write!(f, " ({})", alias)? } - out += &format!("\nUsername: {}", self.username); - out += &format!("\nHostname: {}", self.hostname); - out += &format!("\nIs root: {}", self.is_root); - out += &format!("\nRoot allowed: {}", self.is_root_allowed); - out += &format!("\nLast active: {}", systime_to_string(&self.last_active)); - out += &format!("\nPlatform: {}", self.platform); - out += &format!("\nState: {}", self.state); - write!(f, "{}", out) + writeln!(f, "\nUsername: {}", self.username)?; + writeln!(f, "Hostname: {}", self.hostname)?; + writeln!(f, "Is root: {}", self.is_root)?; + writeln!(f, "Root allowed: {}", self.is_root_allowed)?; + writeln!(f, "Last active: {}", systime_to_string(&self.last_active))?; + writeln!(f, "Platform: {}", self.platform)?; + writeln!(f, "State: {}", self.state) } } @@ -81,7 +81,7 @@ impl Agent { #[cfg(unix)] pub async fn gather() -> Self { - let mut builder = NamedJobBuilder::from_shell(vec![ + let mut builder = NamedJobRunner::from_shell(vec![ ("hostname", "hostname"), ("is_root", "id -u"), ("username", "id -un"), @@ -89,10 +89,8 @@ impl Agent { .unwrap_one() .wait() .await; - let decoder = |job_result: Reportable| { - let assoc_job = unwrap_enum!(job_result, Reportable::Assigned); - assoc_job.to_string_result().trim().to_string() - }; + let decoder = + |job_result: ExecResult| job_result.unwrap().to_string_result().trim().to_string(); Self { hostname: decoder(builder.pop("hostname")), @@ -105,11 +103,11 @@ impl Agent { #[cfg(not(unix))] pub async fn gather() -> Self { - Self::default() + todo!() } - pub async fn run() -> Reportable { - Reportable::Agent(Agent::gather().await) + pub async fn run() -> Agent { + Agent::gather().await } } @@ -127,6 +125,8 @@ impl Default for Agent { state: AgentState::New, token: None, username: String::new(), + ip_gray: None, + ip_white: None, } } } diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index 1122031..2230f70 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -1,18 +1,12 @@ -use super::JobState; +use super::{JobMeta, JobState, JobType}; #[cfg(feature = "server")] use crate::models::schema::*; #[cfg(not(target_arch = "wasm32"))] -use crate::{cache::JobCache, utils::TempFile}; -use crate::{ - config::get_self_uid, - errors::UError, - messaging::Reportable, - utils::{systime_to_string, ProcOutput}, -}; +use crate::{config::get_self_uid, utils::ProcOutput}; #[cfg(feature = "server")] use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; -use std::{fmt, time::SystemTime}; +use std::time::SystemTime; use uuid::Uuid; #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] @@ -21,7 +15,6 @@ use uuid::Uuid; derive(Queryable, Identifiable, Insertable, AsChangeset), table_name = "results" )] - pub struct AssignedJob { pub agent_id: Uuid, pub alias: Option, @@ -30,29 +23,50 @@ pub struct AssignedJob { pub job_id: Uuid, pub result: Option>, pub state: JobState, + pub exec_type: JobType, pub retcode: Option, pub updated: SystemTime, } -impl fmt::Display for AssignedJob { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut out = format!("Result: {}", self.id); - out += &format!("\nAgent: {}", self.agent_id); - out += &format!("\nJob: {}", self.job_id); - if let Some(ref alias) = self.alias { - out += &format!("\nAlias: {}", alias); +#[derive(Serialize, Deserialize, Clone, Copy)] +pub struct AssignedJobById { + pub agent_id: Uuid, + pub id: Uuid, + pub job_id: Uuid, +} + +impl From<(&JobMeta, AssignedJobById)> for AssignedJob { + fn from((meta, pj): (&JobMeta, AssignedJobById)) -> Self { + AssignedJob { + id: pj.id, + agent_id: pj.agent_id, + job_id: pj.job_id, + alias: meta.alias.clone(), + exec_type: meta.exec_type, + ..Default::default() + } + } +} + +impl From<&JobMeta> for AssignedJob { + fn from(meta: &JobMeta) -> Self { + AssignedJob { + job_id: meta.id, + agent_id: get_self_uid(), + alias: meta.alias.clone(), + exec_type: meta.exec_type, + ..Default::default() } - out += &format!("\nUpdated: {}", systime_to_string(&self.updated)); - out += &format!("\nState: {}", self.state); - if self.state == JobState::Finished { - if let Some(ref retcode) = self.retcode { - out += &format!("\nReturn code: {}", retcode); - } - if let Some(ref result) = self.result { - out += &format!("\nResult: {}", String::from_utf8_lossy(result)); - } + } +} + +impl Default for AssignedJobById { + fn default() -> Self { + Self { + agent_id: get_self_uid(), + id: Uuid::new_v4(), + job_id: Uuid::nil(), } - write!(f, "{}", out) } } @@ -68,58 +82,13 @@ impl Default for AssignedJob { state: JobState::Queued, retcode: None, updated: SystemTime::now(), + exec_type: JobType::default(), } } } #[cfg(not(target_arch = "wasm32"))] impl AssignedJob { - pub async fn run(mut self) -> Reportable { - use tokio::process::Command; - let (argv, _payload) = { - let meta = JobCache::get(self.job_id).unwrap(); - if let Some(ref payload) = meta.payload { - let extracted_payload = match TempFile::write_exec(payload) { - Ok(p) => p, - Err(e) => return Reportable::Error(UError::Runtime(e.to_string()).to_string()), - }; - ( - meta.argv.replace("{}", &extracted_payload.get_path()), - Some(extracted_payload), - ) - } else { - (meta.argv.clone(), None) - } - }; - let mut split_cmd = shlex::split(&argv).unwrap().into_iter(); - let cmd = split_cmd.nth(0).unwrap(); - let args = split_cmd.collect::>(); - let cmd_result = Command::new(cmd).args(args).output().await; - let (data, retcode) = match cmd_result { - Ok(output) => ( - ProcOutput::from_output(&output).into_combined(), - output.status.code(), - ), - Err(e) => ( - ProcOutput::new() - .stderr(e.to_string().into_bytes()) - .into_combined(), - None, - ), - }; - self.result = Some(data); - self.retcode = retcode; - Reportable::Assigned(self) - } - - pub fn new(job_id: Uuid, other: Option<&Self>) -> Self { - Self { - agent_id: get_self_uid(), - job_id, - ..other.unwrap_or(&Default::default()).clone() - } - } - pub fn as_job_output(&self) -> Option { self.result .as_ref() @@ -139,4 +108,8 @@ impl AssignedJob { pub fn to_string_result(&self) -> String { String::from_utf8_lossy(&self.to_raw_result()).into_owned() } + + pub fn set_result(&mut self, result: &S) { + self.result = Some(serde_json::to_vec(result).unwrap()); + } } diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 2b76941..e839f8e 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -2,13 +2,11 @@ use super::JobType; #[cfg(feature = "server")] use crate::models::schema::*; use crate::utils::Platform; -use crate::{utils::Stripped, UError, UResult}; +use crate::{UError, UResult}; #[cfg(feature = "server")] use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; -use std::path::PathBuf; -use std::str::from_utf8; -use std::{fmt, fs}; +use std::fs; use uuid::Uuid; #[derive(Serialize, Deserialize, Clone, Debug)] @@ -18,37 +16,37 @@ use uuid::Uuid; table_name = "jobs" )] pub struct JobMeta { + #[serde(default)] pub alias: Option, + /// string like `bash -c {} -a 1 --arg2`, /// where {} is replaced by executable's tmp path + #[serde(default)] pub argv: String, + + #[serde(default = "Uuid::new_v4")] pub id: Uuid, - pub exec_type: JobType, - //pub schedule: JobSchedule, - pub platform: String, - pub payload: Option>, -} -#[derive(Deserialize, Debug)] -pub struct RawJobMeta { - #[serde(default)] - pub alias: Option, - #[serde(default)] - pub argv: String, - #[serde(default)] - pub id: UuidDefaultUnique, #[serde(default)] pub exec_type: JobType, - //pub schedule: JobSchedule, + + ///target triple #[serde(default)] - pub platform: Platform, + pub platform: String, + #[serde(default)] pub payload: Option>, + + /// if payload should be read from external resource #[serde(default)] - pub payload_path: Option, + pub payload_path: Option, + + ///cron-like string + #[serde(default)] + pub schedule: Option, } -impl RawJobMeta { +impl JobMeta { pub fn builder() -> JobMetaBuilder { JobMetaBuilder::default() } @@ -62,29 +60,6 @@ impl RawJobMeta { } } -impl fmt::Display for JobMeta { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut out = format!("Job: {}", self.id); - if let Some(ref alias) = self.alias { - out += &format!(" ({})", alias); - } - out += &format!("\nArgv: {}", self.argv); - out += &format!("\nExecutable type: {}", self.exec_type); - out += &format!("\nPlatform: {}", self.platform); - if let Some(ref payload) = self.payload { - if self.exec_type == JobType::Shell { - let payload = if let Ok(str_payload) = from_utf8(payload) { - Stripped(&str_payload).to_string() - } else { - Stripped(&payload).to_string() - }; - out += &format!("\nPayload: {}", payload); - } - } - write!(f, "{}", out) - } -} - impl Default for JobMeta { fn default() -> Self { Self { @@ -94,19 +69,7 @@ impl Default for JobMeta { exec_type: JobType::Shell, platform: Platform::current().into_string(), payload: None, - } - } -} - -impl Default for RawJobMeta { - fn default() -> Self { - Self { - id: UuidDefaultUnique::default(), - alias: None, - argv: String::new(), - exec_type: JobType::Shell, - platform: Platform::current(), - payload: None, + schedule: None, payload_path: None, } } @@ -114,12 +77,13 @@ impl Default for RawJobMeta { #[derive(Default)] pub struct JobMetaBuilder { - inner: RawJobMeta, + inner: JobMeta, } impl JobMetaBuilder { pub fn with_shell(mut self, shell_cmd: impl Into) -> Self { self.inner.argv = shell_cmd.into(); + self.inner.exec_type = JobType::Shell; self } @@ -128,7 +92,7 @@ impl JobMetaBuilder { self } - pub fn with_payload_src(mut self, path: impl Into) -> Self { + pub fn with_payload_src(mut self, path: impl Into) -> Self { self.inner.payload_path = Some(path.into()); self } @@ -148,6 +112,7 @@ impl JobMetaBuilder { match inner.exec_type { JobType::Shell => { if inner.argv.is_empty() { + // TODO: fix detecting inner.argv = String::from("/bin/bash -c {}") } let argv_parts = @@ -157,9 +122,8 @@ impl JobMetaBuilder { return Err(empty_err.into()); } if let Some(path) = &inner.payload_path { - let data = fs::read(path.clone()).map_err(|e| { - UError::FSError(path.to_string_lossy().to_string(), e.to_string()) - })?; + let data = fs::read(path) + .map_err(|e| UError::FSError(path.to_string(), e.to_string()))?; inner.payload = Some(data) } match inner.payload.as_ref() { @@ -181,38 +145,15 @@ impl JobMetaBuilder { } } }; - if !inner.platform.check() { + if !Platform::new(&inner.platform).check() { return Err(UError::JobArgsError(format!( "Unknown platform {}", - inner.platform.into_string() + inner.platform ))); } Ok(inner.into()) } - JobType::Manage => Ok(inner.into()), - _ => todo!(), - } - } -} - -impl From for JobMeta { - fn from(rjm: RawJobMeta) -> Self { - JobMeta { - alias: rjm.alias, - argv: rjm.argv, - id: rjm.id.0, - exec_type: rjm.exec_type, - platform: rjm.platform.into_string(), - payload: rjm.payload, + _ => Ok(inner.into()), } } } - -#[derive(Deserialize, Debug)] -pub struct UuidDefaultUnique(Uuid); - -impl Default for UuidDefaultUnique { - fn default() -> Self { - Self(Uuid::new_v4()) - } -} diff --git a/lib/u_lib/src/models/jobs/misc.rs b/lib/u_lib/src/models/jobs/misc.rs index df91f0d..ec9b866 100644 --- a/lib/u_lib/src/models/jobs/misc.rs +++ b/lib/u_lib/src/models/jobs/misc.rs @@ -3,21 +3,6 @@ use diesel_derive_enum::DbEnum; use serde::{Deserialize, Serialize}; use strum::Display; -#[derive(Serialize, Deserialize, Clone, Debug)] -pub enum ManageAction { - Ping, - UpdateAvailable, - JobsResultsRequest, - Terminate, -} - -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] -pub enum JobSchedule { - Once, - Permanent, - //Scheduled -} - #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Display)] #[cfg_attr( feature = "server", @@ -26,13 +11,17 @@ pub enum JobSchedule { DieselType = "Jobstate" )] pub enum JobState { - Queued, // server created a job, but client didn't get it yet - //Pending, // client got a job, but not running yet - Running, // client is currently running a job + /// server created a job, but client didn't get it yet + Queued, + + // client got a job, but not running yet + //Pending, + /// client is currently running a job + Running, Finished, } -#[derive(Default, Serialize, Deserialize, Clone, Debug, PartialEq, Display)] +#[derive(Default, Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Display)] #[cfg_attr( feature = "server", derive(DbEnum), @@ -40,8 +29,9 @@ pub enum JobState { DieselType = "Jobtype" )] pub enum JobType { - Manage, + Init, #[default] Shell, - Python, + Terminate, + Update, } diff --git a/lib/u_lib/src/models/schema.rs b/lib/u_lib/src/models/schema.rs index 83c1f59..fd2db7b 100644 --- a/lib/u_lib/src/models/schema.rs +++ b/lib/u_lib/src/models/schema.rs @@ -5,6 +5,8 @@ table! { alias -> Nullable, hostname -> Text, id -> Uuid, + ip_gray -> Nullable, + ip_white -> Nullable, is_root -> Bool, is_root_allowed -> Bool, last_active -> Timestamp, @@ -26,32 +28,6 @@ table! { } } -table! { - use crate::schema_exports::*; - - errors (id) { - agent_id -> Uuid, - created -> Timestamp, - id -> Uuid, - msg -> Nullable, - } -} - -table! { - use crate::schema_exports::*; - - ip_addrs (id) { - agent_id -> Uuid, - check_ts -> Timestamp, - gateway -> Nullable, - id -> Uuid, - iface -> Text, - ip_addr -> Text, - is_gray -> Bool, - netmask -> Text, - } -} - table! { use crate::schema_exports::*; @@ -62,6 +38,8 @@ table! { exec_type -> Jobtype, platform -> Text, payload -> Nullable, + payload_path -> Nullable, + schedule -> Nullable, } } @@ -76,22 +54,14 @@ table! { job_id -> Uuid, result -> Nullable, state -> Jobstate, + exec_type -> Jobtype, retcode -> Nullable, updated -> Timestamp, } } joinable!(certificates -> agents (agent_id)); -joinable!(errors -> agents (agent_id)); -joinable!(ip_addrs -> agents (agent_id)); joinable!(results -> agents (agent_id)); joinable!(results -> jobs (job_id)); -allow_tables_to_appear_in_same_query!( - agents, - certificates, - errors, - ip_addrs, - jobs, - results, -); +allow_tables_to_appear_in_same_query!(agents, certificates, jobs, results,); diff --git a/lib/u_lib/src/builder.rs b/lib/u_lib/src/runner.rs similarity index 54% rename from lib/u_lib/src/builder.rs rename to lib/u_lib/src/runner.rs index 84e46c5..133a6c0 100644 --- a/lib/u_lib/src/builder.rs +++ b/lib/u_lib/src/runner.rs @@ -1,97 +1,136 @@ use crate::{ cache::JobCache, - executor::{DynFut, Waiter}, - messaging::Reportable, - models::{Agent, AssignedJob, JobMeta, JobType, RawJobMeta}, + executor::{ExecResult, Waiter}, + models::{Agent, AssignedJob, AssignedJobById, JobMeta, JobType}, utils::{CombinedResult, OneOrVec, Platform}, + utils::{ProcOutput, TempFile}, UError, UResult, }; use std::collections::HashMap; +use std::process::exit; +use tokio::process::Command; -pub struct JobBuilder { +pub struct JobRunner { waiter: Waiter, } -impl JobBuilder { - pub fn from_request(job_requests: impl OneOrVec) -> CombinedResult { - let job_requests = job_requests.into_vec(); - let mut prepared: Vec = vec![]; - let mut result = CombinedResult::::new(); - for req in job_requests { - let job_meta = JobCache::get(req.job_id); - if job_meta.is_none() { - result.err(UError::NoJob(req.job_id)); - continue; - } - let job_meta = job_meta.unwrap(); +impl JobRunner { + pub fn from_jobs(jobs: impl OneOrVec) -> CombinedResult { + let jobs = jobs.into_vec(); + let mut waiter = Waiter::new(); + let mut result = CombinedResult::::new(); + for job in jobs { //waiting for try-blocks stabilization - let built_req = (|| -> UResult<()> { - Ok(match job_meta.exec_type { - JobType::Shell => { - let meta = JobCache::get(req.job_id).ok_or(UError::NoJob(req.job_id))?; - let curr_platform = Platform::current(); - if !curr_platform.matches(&meta.platform) { - return Err(UError::InsuitablePlatform( - meta.platform.clone(), - curr_platform.into_string(), - ) - .into()); - } - let job = AssignedJob::new(req.job_id, Some(&req)); - prepared.push(Box::pin(job.run())) - } - JobType::Manage => prepared.push(Box::pin(Agent::run())), - _ => todo!(), - }) + let built_job = (|| -> UResult<()> { + let meta = JobCache::get(job.job_id).ok_or(UError::NoJob(job.job_id))?; + let curr_platform = Platform::current(); + if !curr_platform.matches(&meta.platform) { + return Err(UError::InsuitablePlatform( + meta.platform.clone(), + curr_platform.into_string(), + ) + .into()); + } + let job = AssignedJob::from((&*meta, job)); + waiter.push(run_assigned_job(job)); + Ok(()) })(); - if let Err(e) = built_req { + if let Err(e) = built_job { result.err(e) } } - result.ok(Self { - waiter: Waiter::new(prepared), - }); + result.ok(Self { waiter }); result } - pub fn from_meta(job_metas: impl OneOrVec) -> CombinedResult { - let job_requests = job_metas + pub fn from_meta(metas: impl OneOrVec) -> CombinedResult { + let jobs = metas .into_vec() .into_iter() .map(|jm| { - let j_uid = jm.id; + let job_uid = jm.id; JobCache::insert(jm); - AssignedJob::new(j_uid, None) + AssignedJobById { + job_id: job_uid, + ..Default::default() + } }) - .collect::>(); - JobBuilder::from_request(job_requests) + .collect::>(); + JobRunner::from_jobs(jobs) } - /// Spawn jobs and pop results later + /// Spawn jobs pub async fn spawn(mut self) -> Self { self.waiter = self.waiter.spawn().await; self } /// Spawn jobs and wait for result - pub async fn wait(self) -> Vec { + pub async fn wait(self) -> Vec { self.waiter.spawn().await.wait().await } /// Spawn one job and wait for result - pub async fn wait_one(self) -> Reportable { + pub async fn wait_one(self) -> ExecResult { self.waiter.spawn().await.wait().await.pop().unwrap() } } +pub async fn run_assigned_job(mut job: AssignedJob) -> ExecResult { + match job.exec_type { + JobType::Shell => { + let (argv, _payload) = { + let meta = JobCache::get(job.job_id).unwrap(); + if let Some(ref payload) = meta.payload { + let extracted_payload = match TempFile::write_exec(payload) { + Ok(p) => p, + Err(e) => return Err(UError::Runtime(e.to_string())), + }; + ( + meta.argv.replace("{}", &extracted_payload.get_path()), + Some(extracted_payload), + ) + } else { + (meta.argv.clone(), None) + } + }; + let mut split_cmd = shlex::split(&argv).unwrap().into_iter(); + let cmd = split_cmd.nth(0).unwrap(); + let args = split_cmd.collect::>(); + let cmd_result = Command::new(cmd).args(args).output().await; + let (data, retcode) = match cmd_result { + Ok(output) => ( + ProcOutput::from_output(&output).into_combined(), + output.status.code(), + ), + Err(e) => ( + ProcOutput::new() + .stderr(e.to_string().into_bytes()) + .into_combined(), + None, + ), + }; + job.result = Some(data); + job.retcode = retcode; + } + JobType::Init => { + job.set_result(&Agent::run().await); + job.retcode = Some(0); + } + JobType::Update => todo!(), + JobType::Terminate => exit(0), + }; + Ok(job) +} + /// Store jobs and get results by name -pub struct NamedJobBuilder { - builder: Option, +pub struct NamedJobRunner { + builder: Option, job_names: Vec<&'static str>, - results: HashMap<&'static str, Reportable>, + results: HashMap<&'static str, ExecResult>, } -impl NamedJobBuilder { +impl NamedJobRunner { pub fn from_shell( named_jobs: impl OneOrVec<(&'static str, &'static str)>, ) -> CombinedResult { @@ -100,7 +139,7 @@ impl NamedJobBuilder { .into_vec() .into_iter() .filter_map( - |(alias, cmd)| match RawJobMeta::builder().with_shell(cmd).build() { + |(alias, cmd)| match JobMeta::builder().with_shell(cmd).build() { Ok(meta) => Some((alias, meta)), Err(e) => { result.err(e); @@ -124,7 +163,7 @@ impl NamedJobBuilder { }) .collect(); Self { - builder: Some(JobBuilder::from_meta(job_metas).unwrap_one()), + builder: Some(JobRunner::from_meta(job_metas).unwrap_one()), job_names, results: HashMap::new(), } @@ -138,11 +177,11 @@ impl NamedJobBuilder { self } - pub fn pop_opt(&mut self, name: &'static str) -> Option { + pub fn pop_opt(&mut self, name: &'static str) -> Option { self.results.remove(name) } - pub fn pop(&mut self, name: &'static str) -> Reportable { + pub fn pop(&mut self, name: &'static str) -> ExecResult { self.pop_opt(name).unwrap() } } @@ -151,8 +190,8 @@ impl NamedJobBuilder { mod tests { use super::*; use crate::{ - builder::{JobBuilder, NamedJobBuilder}, models::{misc::JobType, JobMeta}, + runner::{JobRunner, NamedJobRunner}, unwrap_enum, }; use std::time::SystemTime; @@ -162,10 +201,10 @@ mod tests { #[tokio::test] async fn test_is_really_async() { const SLEEP_SECS: u64 = 1; - let job = RawJobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); + let job = JobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); let sleep_jobs: Vec = (0..50).map(|_| job.clone()).collect(); let now = SystemTime::now(); - JobBuilder::from_meta(sleep_jobs).unwrap_one().wait().await; + JobRunner::from_meta(sleep_jobs).unwrap_one().wait().await; assert!(now.elapsed().unwrap().as_secs() < SLEEP_SECS + 2) } @@ -199,13 +238,16 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] expected_result: &str, ) -> TestResult { - let mut job = RawJobMeta::builder().with_shell(cmd); + let mut job = JobMeta::builder().with_shell(cmd); if let Some(p) = payload { job = job.with_payload(p); } let job = job.build().unwrap(); - let job_result = JobBuilder::from_meta(job).unwrap_one().wait_one().await; - let result = unwrap_enum!(job_result, Reportable::Assigned); + let result = JobRunner::from_meta(job) + .unwrap_one() + .wait_one() + .await + .unwrap(); let result = result.to_string_result(); assert_eq!(result.trim(), expected_result); Ok(()) @@ -215,29 +257,25 @@ mod tests { async fn test_complex_load() -> TestResult { const SLEEP_SECS: u64 = 1; let now = SystemTime::now(); - let longest_job = RawJobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); - let longest_job = JobBuilder::from_meta(longest_job) - .unwrap_one() - .spawn() - .await; - let ls = JobBuilder::from_meta(RawJobMeta::from_shell("ls")?) + let longest_job = JobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); + let longest_job = JobRunner::from_meta(longest_job).unwrap_one().spawn().await; + let ls = JobRunner::from_meta(JobMeta::from_shell("ls")?) .unwrap_one() .wait_one() - .await; - let ls = unwrap_enum!(ls, Reportable::Assigned); + .await + .unwrap(); assert_eq!(ls.retcode.unwrap(), 0); let folders = ls.to_string_result(); let subfolders_jobs: Vec = folders .lines() - .map(|f| RawJobMeta::from_shell(format!("ls {}", f)).unwrap()) + .map(|f| JobMeta::from_shell(format!("ls {}", f)).unwrap()) .collect(); - let ls_subfolders = JobBuilder::from_meta(subfolders_jobs) + let ls_subfolders = JobRunner::from_meta(subfolders_jobs) .unwrap_one() .wait() .await; for result in ls_subfolders { - let result = unwrap_enum!(result, Reportable::Assigned); - assert_eq!(result.retcode.unwrap(), 0); + assert_eq!(result.unwrap().retcode.unwrap(), 0); } longest_job.wait().await; assert_eq!(now.elapsed().unwrap().as_secs(), SLEEP_SECS); @@ -263,9 +301,12 @@ mod tests { */ #[tokio::test] async fn test_failing_shell_job() -> TestResult { - let job = RawJobMeta::from_shell("lol_kek_puk")?; - let job_result = JobBuilder::from_meta(job).unwrap_one().wait_one().await; - let job_result = unwrap_enum!(job_result, Reportable::Assigned); + let job = JobMeta::from_shell("lol_kek_puk")?; + let job_result = JobRunner::from_meta(job) + .unwrap_one() + .wait_one() + .await + .unwrap(); let output = job_result.to_string_result(); assert!(output.contains("No such file")); assert!(job_result.retcode.is_none()); @@ -281,7 +322,7 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] err_str: &str, ) -> TestResult { - let mut job = RawJobMeta::builder().with_shell(cmd); + let mut job = JobMeta::builder().with_shell(cmd); if let Some(p) = payload { job = job.with_payload(p); } @@ -293,17 +334,17 @@ mod tests { #[tokio::test] async fn test_different_job_types() -> TestResult { - let mut jobs = NamedJobBuilder::from_meta(vec![ - ("sleeper", RawJobMeta::from_shell("sleep 3")?), + let mut jobs = NamedJobRunner::from_meta(vec![ + ("sleeper", JobMeta::from_shell("sleep 3")?), ( "gatherer", - RawJobMeta::builder().with_type(JobType::Manage).build()?, + JobMeta::builder().with_type(JobType::Init).build()?, ), ]) .wait() .await; let gathered = jobs.pop("gatherer"); - assert_eq!(unwrap_enum!(gathered, Reportable::Agent).alias, None); + assert_eq!(gathered.unwrap().alias, None); Ok(()) } } diff --git a/lib/u_lib/src/utils/fmt/stripped.rs b/lib/u_lib/src/utils/fmt/stripped.rs index 21d0032..44af3ca 100644 --- a/lib/u_lib/src/utils/fmt/stripped.rs +++ b/lib/u_lib/src/utils/fmt/stripped.rs @@ -38,9 +38,10 @@ impl<'a> Strippable for &'a Vec { self.iter() } } -pub struct Stripped<'inner, Inner: Strippable + 'inner>(pub &'inner Inner); -impl<'inner, Inner: Strippable + 'inner> Stripped<'inner, Inner> { +pub struct Stripped<'i, Inner: Strippable + 'i>(pub &'i Inner); + +impl<'i, Inner: Strippable + 'i> Stripped<'i, Inner> { fn iter(&self) -> Inner::TypeIter { self.0.iterator() } @@ -54,7 +55,7 @@ impl<'inner, Inner: Strippable + 'inner> Stripped<'inner, Inner> { } } -impl<'inner, Inner: Strippable + 'inner> fmt::Display for Stripped<'inner, Inner> { +impl<'i, Inner: Strippable + 'i> fmt::Display for Stripped<'i, Inner> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let placeholder = self.placeholder(); for c in self.iter().take(MAX_DATA_LEN - placeholder.len()) { diff --git a/lib/u_lib/src/utils/mod.rs b/lib/u_lib/src/utils/mod.rs index 4d6e004..5d9f9b4 100644 --- a/lib/u_lib/src/utils/mod.rs +++ b/lib/u_lib/src/utils/mod.rs @@ -10,7 +10,6 @@ pub mod storage; pub mod tempfile; #[cfg(unix)] pub mod unix; -pub mod vec_display; pub use combined_result::*; pub use conv::*; @@ -22,7 +21,6 @@ pub use proc_output::*; pub use storage::*; #[cfg(not(target_arch = "wasm32"))] pub use tempfile::*; -pub use vec_display::*; #[cfg(unix)] pub use unix::*; diff --git a/lib/u_lib/src/utils/platform.rs b/lib/u_lib/src/utils/platform.rs index 7e13301..6ecb82b 100644 --- a/lib/u_lib/src/utils/platform.rs +++ b/lib/u_lib/src/utils/platform.rs @@ -7,6 +7,10 @@ use std::str::FromStr; pub struct Platform(String); impl Platform { + pub fn new(p: impl Into) -> Self { + Self(p.into()) + } + pub fn current() -> Platform { Self(guess_host_triple().unwrap_or("unknown").to_string()) } diff --git a/migrations/2020-10-24-111622_create_all/up.sql b/migrations/2020-10-24-111622_create_all/up.sql index a26d094..c2d5ac9 100644 --- a/migrations/2020-10-24-111622_create_all/up.sql +++ b/migrations/2020-10-24-111622_create_all/up.sql @@ -1,5 +1,5 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; -CREATE TYPE JobType AS ENUM ('shell', 'manage', 'python'); +CREATE TYPE JobType AS ENUM ('shell', 'init', 'python'); CREATE TYPE JobState AS ENUM ('queued', 'running', 'finished'); CREATE TYPE AgentState AS ENUM ('new', 'active', 'banned'); @@ -7,6 +7,8 @@ CREATE TABLE IF NOT EXISTS agents ( alias TEXT , hostname TEXT NOT NULL , id UUID NOT NULL DEFAULT uuid_generate_v4() + , ip_gray TEXT + , ip_white TEXT , is_root BOOLEAN NOT NULL DEFAULT false , is_root_allowed BOOLEAN NOT NULL DEFAULT false , last_active TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP @@ -17,20 +19,8 @@ CREATE TABLE IF NOT EXISTS agents ( -- is needed to processing requests , token TEXT , username TEXT NOT NULL - , PRIMARY KEY(id) -); -CREATE TABLE IF NOT EXISTS ip_addrs ( - agent_id UUID NOT NULL - , check_ts TIMESTAMP NOT NULL - , gateway TEXT - , id UUID NOT NULL DEFAULT uuid_generate_v4() - , iface TEXT NOT NULL - , ip_addr TEXT NOT NULL - , is_gray BOOLEAN NOT NULL DEFAULT true - , netmask TEXT NOT NULL , PRIMARY KEY(id) - , FOREIGN KEY(agent_id) REFERENCES agents(id) ); CREATE TABLE IF NOT EXISTS jobs ( @@ -40,6 +30,9 @@ CREATE TABLE IF NOT EXISTS jobs ( , exec_type JobType NOT NULL DEFAULT 'shell' , platform TEXT NOT NULL , payload BYTEA + , payload_path TEXT + , schedule TEXT + , PRIMARY KEY(id) ); @@ -51,8 +44,10 @@ CREATE TABLE IF NOT EXISTS results ( , job_id UUID NOT NULL , result BYTEA , state JobState NOT NULL DEFAULT 'queued' + , exec_type JobType NOT NULL DEFAULT 'shell' , retcode INTEGER , updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + , FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE , FOREIGN KEY(job_id) REFERENCES jobs(id) ON DELETE CASCADE , PRIMARY KEY(id) @@ -62,6 +57,7 @@ CREATE TABLE IF NOT EXISTS certificates ( agent_id UUID NOT NULL , id UUID NOT NULL DEFAULT uuid_generate_v4() , is_revoked BOOLEAN NOT NULL DEFAULT FALSE + , PRIMARY KEY(id) , FOREIGN KEY(agent_id) REFERENCES agents(id) ); \ No newline at end of file From 7b59031bfe18157310e56d2c9fae4d2359788e5a Mon Sep 17 00:00:00 2001 From: plazmoid Date: Thu, 29 Sep 2022 01:06:32 +0500 Subject: [PATCH 27/30] fixed integration tests, improve server code - mount cargo registry in tests container - perform cargo update on host machine - add context to db errors - use db.conn field only in UDB - handle rejections - prettier logs - pass agent_uid in user-agent header --- Cargo.toml | 1 + Makefile.toml | 9 +- bin/u_agent/src/lib.rs | 2 +- bin/u_panel/Cargo.toml | 2 +- bin/u_panel/src/argparse.rs | 4 +- bin/u_server/Cargo.toml | 1 + bin/u_server/src/db.rs | 165 +++++++++++------- bin/u_server/src/error.rs | 59 +++++++ bin/u_server/src/errors.rs | 22 --- bin/u_server/src/handlers.rs | 38 ++-- bin/u_server/src/u_server.rs | 72 +++++--- integration/docker-compose.yml | 3 +- integration/integration_tests.py | 2 +- integration/tests/fixtures/agent.rs | 2 + integration/tests/helpers/panel.rs | 4 +- lib/u_lib/Cargo.toml | 2 +- lib/u_lib/src/api.rs | 22 +-- lib/u_lib/src/models/agent.rs | 2 +- lib/u_lib/src/models/jobs/meta.rs | 4 +- lib/u_lib/src/utils/platform.rs | 4 + .../2020-10-24-111622_create_all/up.sql | 90 +++++----- todos.txt => spec.txt | 6 +- 22 files changed, 319 insertions(+), 197 deletions(-) create mode 100644 bin/u_server/src/error.rs delete mode 100644 bin/u_server/src/errors.rs rename todos.txt => spec.txt (60%) diff --git a/Cargo.toml b/Cargo.toml index ece9d2c..8f5feed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ ] [workspace.dependencies] +anyhow = "1.0.58" reqwest = { version = "0.11", features = ["json"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/Makefile.toml b/Makefile.toml index 63e14e2..18100b2 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -48,8 +48,12 @@ dependencies = ["build_static_libs", "build_frontend"] command = "${CARGO}" args = ["build", "--target", "${TARGET}", "${@}"] +[tasks.cargo_update] +command = "${CARGO}" +args = ["update"] + [tasks.release_tasks] -condition = { env = { "PROFILE_OVERRIDE" = "release"} } +condition = { env = { PROFILE_OVERRIDE = "release"} } script = ''' BINS=$(ls ./target/${TARGET}/${PROFILE_OVERRIDE}/u_* -1 | grep -v ".d") echo "Stripping..." @@ -59,7 +63,7 @@ upx -9 $BINS ''' [tasks.build] -dependencies = ["cargo_build", "release_tasks"] +dependencies = ["cargo_update", "cargo_build", "release_tasks"] clear = true [tasks.run] @@ -70,6 +74,7 @@ command = "${CARGO}" args = ["test", "--target", "${TARGET}", "--lib", "--", "${@}"] [tasks.integration] +dependencies = ["cargo_update"] script = ''' [[ ! -d "./target/${TARGET}/${PROFILE_OVERRIDE}" ]] && echo 'No target folder. Build project first' && exit 1 cd ./integration diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 1725e58..e9bc3e9 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -115,6 +115,6 @@ pub async fn run_forever() -> ! { // ErrChan::send(UError::Runtime(e.to_string()), "deeeemon").await // } } - info!("Startup"); + info!("Starting agent {}", get_self_uid()); agent_loop(client).await } diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 618d1bc..e6705f6 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" [dependencies] actix-cors = "0.6.1" actix-web = "4.1" -anyhow = "1.0.44" +anyhow = { workspace = true } futures-util = "0.3.21" mime_guess = "2.0.4" once_cell = "1.8.0" diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index ce810ba..80261f4 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -78,11 +78,11 @@ fn parse_uuid(src: &str) -> Result { pub async fn process_cmd(client: ClientHandler, args: Args) -> UResult { fn to_json(data: AnyResult) -> String { - let data = match data { + let result = match data { Ok(r) => PanelResult::Ok(r), Err(e) => PanelResult::Err(e.downcast().expect("unknown error type")), }; - serde_json::to_string(&data).unwrap() + serde_json::to_string(&result).unwrap() } Ok(match args.cmd { diff --git a/bin/u_server/Cargo.toml b/bin/u_server/Cargo.toml index b825bbf..474ecb1 100644 --- a/bin/u_server/Cargo.toml +++ b/bin/u_server/Cargo.toml @@ -5,6 +5,7 @@ name = "u_server" version = "0.1.0" [dependencies] +anyhow = { workspace = true } diesel = { version = "1.4.5", features = ["postgres", "uuid"] } hyper = "0.14" once_cell = "1.7.2" diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index 09dcad3..e72b2f9 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -1,4 +1,4 @@ -use crate::errors::{Error, SResult}; +use crate::error::Error as ServerError; use diesel::{pg::PgConnection, prelude::*, result::Error as DslError}; use once_cell::sync::OnceCell; use serde::Deserialize; @@ -9,9 +9,10 @@ use u_lib::{ }; use uuid::Uuid; +type Result = std::result::Result; + pub struct UDB { - pub conn: PgConnection, - __p: (), + conn: PgConnection, } static DB: OnceCell> = OnceCell::new(); @@ -34,7 +35,6 @@ impl UDB { ); let instance = UDB { conn: PgConnection::establish(&db_url).unwrap(), - __p: (), }; Mutex::new(instance) }) @@ -42,69 +42,89 @@ impl UDB { .unwrap() } - pub fn insert_jobs(&self, job_metas: &[JobMeta]) -> SResult<()> { + pub fn insert_jobs(&self, job_metas: &[JobMeta]) -> Result<()> { use schema::jobs; + diesel::insert_into(jobs::table) .values(job_metas) - .execute(&self.conn)?; + .execute(&self.conn) + .map_err(with_err_ctx("Can't insert jobs"))?; Ok(()) } - pub fn get_jobs(&self, uid: Option) -> SResult> { + pub fn get_jobs(&self, ouid: Option) -> Result> { use schema::jobs; - let result = if uid.is_some() { - jobs::table - .filter(jobs::id.eq(uid.unwrap())) - .get_results::(&self.conn)? - } else { - jobs::table.load::(&self.conn)? - }; - Ok(result) + + match ouid { + Some(uid) => jobs::table + .filter(jobs::id.eq(uid)) + .get_results::(&self.conn), + None => jobs::table.load::(&self.conn), + } + .map_err(with_err_ctx("Can't get exact jobs")) } - pub fn find_job_by_alias(&self, alias: &str) -> SResult { + pub fn find_job_by_alias(&self, alias: &str) -> Result> { use schema::jobs; + let result = jobs::table .filter(jobs::alias.eq(alias)) - .first::(&self.conn)?; + .first::(&self.conn) + .optional() + .map_err(with_err_ctx(format!("Can't find job by alias {alias}")))?; Ok(result) } - pub fn insert_agent(&self, agent: &Agent) -> SResult<()> { + pub fn insert_agent(&self, agent: &Agent) -> Result<()> { use schema::agents; + diesel::insert_into(agents::table) .values(agent) .on_conflict(agents::id) .do_update() .set(agent) - .execute(&self.conn)?; + .execute(&self.conn) + .map_err(with_err_ctx(format!("Can't insert agent {agent:x?}")))?; + Ok(()) + } + + pub fn insert_result(&self, result: &AssignedJob) -> Result<()> { + use schema::results; + + diesel::insert_into(results::table) + .values(result) + .execute(&self.conn) + .map_err(with_err_ctx(format!("Can't insert result {result:x?}")))?; Ok(()) } - pub fn get_agents(&self, uid: Option) -> SResult> { + pub fn get_agents(&self, ouid: Option) -> Result> { use schema::agents; - let result = if uid.is_some() { - agents::table - .filter(agents::id.eq(uid.unwrap())) - .load::(&self.conn)? - } else { - agents::table.load::(&self.conn)? - }; - Ok(result) + + match ouid { + Some(uid) => agents::table + .filter(agents::id.eq(uid)) + .load::(&self.conn), + None => agents::table.load::(&self.conn), + } + .map_err(with_err_ctx(format!("Can't get agent(s) {ouid:?}"))) } - pub fn update_job_status(&self, uid: Uuid, status: JobState) -> SResult<()> { + pub fn update_job_status(&self, uid: Uuid, status: JobState) -> Result<()> { use schema::results; + diesel::update(results::table) .filter(results::id.eq(uid)) .set(results::state.eq(status)) - .execute(&self.conn)?; + .execute(&self.conn) + .map_err(with_err_ctx(format!("Can't update status of job {uid}")))?; Ok(()) } //TODO: filters possibly could work in a wrong way, check - pub fn get_exact_jobs(&self, uid: Option, personal: bool) -> SResult> { + pub fn get_exact_jobs(&self, uid: Option, personal: bool) -> Result> { use schema::results; + let mut q = results::table.into_boxed(); /*if uid.is_some() { q = q.filter(results::agent_id.eq(uid.unwrap())) @@ -121,32 +141,19 @@ impl UDB { .or_filter(results::job_id.eq(uid.unwrap())) .or_filter(results::id.eq(uid.unwrap())) } - let result = q.load::(&self.conn)?; + let result = q + .load::(&self.conn) + .map_err(with_err_ctx("Can't get exact jobs"))?; Ok(result) } - pub fn set_jobs_for_agent(&self, agent_uid: &Uuid, job_uids: &[Uuid]) -> SResult> { - use schema::{agents::dsl::agents, jobs::dsl::jobs, results}; - if let Err(DslError::NotFound) = agents.find(agent_uid).first::(&self.conn) { - return Err(Error::NotFound(agent_uid.to_string())); - } - let not_found_jobs = job_uids - .iter() - .filter_map(|job_uid| { - if let Err(DslError::NotFound) = jobs.find(job_uid).first::(&self.conn) { - Some(job_uid.to_string()) - } else { - None - } - }) - .collect::>(); - if !not_found_jobs.is_empty() { - return Err(Error::NotFound(not_found_jobs.join(", "))); - } + pub fn set_jobs_for_agent(&self, agent_uid: &Uuid, job_uids: &[Uuid]) -> Result> { + use schema::results; + let job_requests = job_uids .iter() .map(|job_uid| { - info!("set_jobs_for_agent: set {} for {}", job_uid, agent_uid); + debug!("set_jobs_for_agent: set {} for {}", job_uid, agent_uid); AssignedJob { job_id: *job_uid, agent_id: *agent_uid, @@ -154,46 +161,84 @@ impl UDB { } }) .collect::>(); + diesel::insert_into(results::table) .values(&job_requests) - .execute(&self.conn)?; - let assigned_uids = job_requests.iter().map(|aj| aj.id).collect(); - Ok(assigned_uids) + .execute(&self.conn) + .map_err(with_err_ctx(format!( + "Can't setup jobs {job_uids:?} for agent {agent_uid:?}" + )))?; + + Ok(job_requests.iter().map(|aj| aj.id).collect()) } - pub fn del_jobs(&self, uids: &[Uuid]) -> SResult { + pub fn del_jobs(&self, uids: &[Uuid]) -> Result { use schema::jobs; + let mut affected = 0; for &uid in uids { let deleted = diesel::delete(jobs::table) .filter(jobs::id.eq(uid)) - .execute(&self.conn)?; + .execute(&self.conn) + .map_err(with_err_ctx("Can't delete jobs"))?; affected += deleted; } Ok(affected) } - pub fn del_results(&self, uids: &[Uuid]) -> SResult { + pub fn del_results(&self, uids: &[Uuid]) -> Result { use schema::results; + let mut affected = 0; for &uid in uids { let deleted = diesel::delete(results::table) .filter(results::id.eq(uid)) - .execute(&self.conn)?; + .execute(&self.conn) + .map_err(with_err_ctx("Can't delete results"))?; affected += deleted; } Ok(affected) } - pub fn del_agents(&self, uids: &[Uuid]) -> SResult { + pub fn del_agents(&self, uids: &[Uuid]) -> Result { use schema::agents; + let mut affected = 0; for &uid in uids { let deleted = diesel::delete(agents::table) .filter(agents::id.eq(uid)) - .execute(&self.conn)?; + .execute(&self.conn) + .map_err(with_err_ctx("Can't delete agents"))?; affected += deleted; } Ok(affected) } + + pub fn update_agent(&self, agent: &Agent) -> Result<()> { + agent + .save_changes::(&self.conn) + .map_err(with_err_ctx(format!("Can't update agent {agent:x?}")))?; + Ok(()) + } + + pub fn update_job(&self, job: &JobMeta) -> Result<()> { + job.save_changes::(&self.conn) + .map_err(with_err_ctx(format!("Can't update job {job:x?}")))?; + Ok(()) + } + + pub fn update_result(&self, result: &AssignedJob) -> Result<()> { + debug!( + "updating result: id = {}, job_id = {}, agent_id = {}", + result.id, result.job_id, result.agent_id + ); + result + .save_changes::(&self.conn) + .map_err(with_err_ctx(format!("Can't update result {result:x?}")))?; + Ok(()) + } +} + +fn with_err_ctx(msg: impl AsRef) -> impl Fn(DslError) -> ServerError { + move |err| ServerError::DBErrorCtx(format!("{}, reason: {err}", msg.as_ref())) } diff --git a/bin/u_server/src/error.rs b/bin/u_server/src/error.rs new file mode 100644 index 0000000..6f07b72 --- /dev/null +++ b/bin/u_server/src/error.rs @@ -0,0 +1,59 @@ +use diesel::result::Error as DslError; +use thiserror::Error; +use warp::{ + http::StatusCode, + reject::Reject, + reply::{with_status, Response}, + Reply, +}; + +#[derive(Error, Debug)] +pub enum Error { + #[error("Error processing {0}")] + ProcessingError(String), + + #[error(transparent)] + DBError(#[from] DslError), + + #[error("DB error: {0}")] + DBErrorCtx(String), + + #[error("General error: {0}")] + Other(String), +} + +impl Reject for Error {} + +pub struct RejResponse { + message: String, + status: StatusCode, +} + +impl RejResponse { + pub fn not_found(msg: impl Into) -> Self { + Self { + message: msg.into(), + status: StatusCode::NOT_FOUND, + } + } + + pub fn bad_request(msg: impl Into) -> Self { + Self { + message: msg.into(), + status: StatusCode::BAD_REQUEST, + } + } + + pub fn internal() -> Self { + Self { + message: "INTERNAL_SERVER_ERROR".to_string(), + status: StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +impl Reply for RejResponse { + fn into_response(self) -> Response { + with_status(self.message, self.status).into_response() + } +} diff --git a/bin/u_server/src/errors.rs b/bin/u_server/src/errors.rs deleted file mode 100644 index c77472a..0000000 --- a/bin/u_server/src/errors.rs +++ /dev/null @@ -1,22 +0,0 @@ -use diesel::result::Error as DslError; -use thiserror::Error; -use warp::reject::Reject; - -pub type SResult = Result; - -#[derive(Error, Debug)] -pub enum Error { - #[error("{0} is not found")] - NotFound(String), - - #[error("Error processing {0}")] - ProcessingError(String), - - #[error(transparent)] - DBError(#[from] DslError), - - #[error("General error: {0}")] - Other(String), -} - -impl Reject for Error {} diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index 3b0db33..43139ab 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -1,8 +1,7 @@ use std::time::SystemTime; use crate::db::UDB; -use crate::errors::Error; -use diesel::SaveChangesDsl; +use crate::error::Error; use u_lib::{ messaging::{AsMsg, BaseMessage, Reportable}, models::*, @@ -39,7 +38,9 @@ impl Endpoints { if agents.is_empty() { let db = UDB::lock_db(); db.insert_agent(&Agent::with_id(uid.unwrap()))?; - let job = db.find_job_by_alias("agent_hello")?; + let job = db + .find_job_by_alias("agent_hello")? + .expect("agent_hello job not found"); db.set_jobs_for_agent(&uid.unwrap(), &[job.id])?; } let result = UDB::lock_db().get_exact_jobs(uid, true)?; @@ -76,8 +77,16 @@ impl Endpoints { msg.into_inner() .into_iter() .map(|ident| { - Uuid::parse_str(&ident) - .or_else(|_| UDB::lock_db().find_job_by_alias(&ident).map(|j| j.id)) + Uuid::parse_str(&ident).or_else(|_| { + let job_from_db = UDB::lock_db().find_job_by_alias(&ident); + match job_from_db { + Ok(job) => match job { + Some(j) => Ok(j.id), + None => Err(Error::ProcessingError(format!("unknown ident {ident}"))), + }, + Err(e) => Err(e), + } + }) }) .collect::, Error>>() .and_then(|j| UDB::lock_db().set_jobs_for_agent(&agent_uid, &j)) @@ -117,10 +126,7 @@ impl Endpoints { JobType::Terminate => todo!(), JobType::Update => todo!(), } - let db = UDB::lock_db(); - result - .save_changes::(&db.conn) - .map_err(Error::from)?; + UDB::lock_db().update_result(&result)?; } Reportable::Error(e) => { warn!("{} reported an error: {}", id, e); @@ -132,27 +138,19 @@ impl Endpoints { } pub async fn update_agent(agent: BaseMessage<'static, Agent>) -> EndpResult<()> { - agent - .into_inner() - .save_changes::(&UDB::lock_db().conn) - .map_err(Error::from)?; + UDB::lock_db().update_agent(&agent.into_inner())?; Ok(()) } pub async fn update_job(job: BaseMessage<'static, JobMeta>) -> EndpResult<()> { - job.into_inner() - .save_changes::(&UDB::lock_db().conn) - .map_err(Error::from)?; + UDB::lock_db().update_job(&job.into_inner())?; Ok(()) } pub async fn update_assigned_job( assigned: BaseMessage<'static, AssignedJob>, ) -> EndpResult<()> { - assigned - .into_inner() - .save_changes::(&UDB::lock_db().conn) - .map_err(Error::from)?; + UDB::lock_db().update_result(&assigned.into_inner())?; Ok(()) } diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 8c11e0a..4417e8f 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -12,12 +12,12 @@ extern crate diesel; // in this block mod db; -mod errors; +mod error; mod handlers; -use errors::{Error, SResult}; +use error::{Error as ServerError, RejResponse}; use serde::{de::DeserializeOwned, Deserialize}; -use std::path::PathBuf; +use std::{convert::Infallible, path::PathBuf}; use u_lib::{ config::MASTER_PORT, logging::init_logger, @@ -28,7 +28,8 @@ use u_lib::{ use uuid::Uuid; use warp::{ body, - reply::{json, reply, Json}, + log::{custom, Info}, + reply::{json, reply, Json, Response}, Filter, Rejection, Reply, }; @@ -55,7 +56,7 @@ pub fn init_endpoints( auth_token: &str, ) -> impl Filter + Clone { let path = |p: &'static str| warp::post().and(warp::path(p)); - let infallible_none = |_| async { Ok::<(Option,), std::convert::Infallible>((None,)) }; + let infallible_none = |_| async { Ok::<_, Infallible>((None::,)) }; let get_agents = path("get_agents") .and( @@ -131,7 +132,7 @@ pub fn init_endpoints( .map(ok); let auth_token = format!("Bearer {auth_token}",).into_boxed_str(); - let auth_header = warp::header::exact("Authorization", Box::leak(auth_token)); + let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); let auth_zone = (get_agents .or(get_jobs) @@ -150,32 +151,31 @@ pub fn init_endpoints( auth_zone.or(agent_zone) } -pub fn prefill_jobs() -> SResult<()> { +pub fn prefill_jobs() -> Result<(), ServerError> { let job_alias = "agent_hello"; - let if_job_exists = UDB::lock_db().find_job_by_alias(job_alias); - match if_job_exists { - Ok(_) => Ok(()), - Err(Error::DBError(diesel::result::Error::NotFound)) => { - let agent_hello = JobMeta::builder() - .with_type(JobType::Init) - .with_alias(job_alias) - .build() - .unwrap(); - UDB::lock_db().insert_jobs(&[agent_hello]) - } - Err(e) => Err(e), + let if_job_exists = UDB::lock_db().find_job_by_alias(job_alias)?; + if if_job_exists.is_none() { + let agent_hello = JobMeta::builder() + .with_type(JobType::Init) + .with_alias(job_alias) + .build() + .unwrap(); + UDB::lock_db().insert_jobs(&[agent_hello])? } + Ok(()) } -pub async fn serve() -> SResult<()> { +pub async fn serve() -> Result<(), ServerError> { init_logger(Some("u_server")); prefill_jobs()?; - let env = load_env::().map_err(|e| Error::Other(e.to_string()))?; - let routes = init_endpoints(&env.admin_auth_token); let certs_dir = PathBuf::from("certs"); + let env = load_env::().map_err(|e| ServerError::Other(e.to_string()))?; + let routes = init_endpoints(&env.admin_auth_token) + .recover(handle_rejection) + .with(custom(logger)); - warp::serve(routes.with(warp::log("warp"))) + warp::serve(routes) .tls() .cert_path(certs_dir.join("server.crt")) .key_path(certs_dir.join("server.key")) @@ -185,6 +185,32 @@ pub async fn serve() -> SResult<()> { Ok(()) } +async fn handle_rejection(rej: Rejection) -> Result { + let resp = if let Some(err) = rej.find::() { + error!("{:x?}", err); + RejResponse::bad_request(err.to_string()) + } else if rej.is_not_found() { + RejResponse::not_found("not found placeholder") + } else { + RejResponse::internal() + }; + Ok(resp.into_response()) +} + +fn logger(info: Info<'_>) { + info!(target: "warp", + "{raddr} {agent_uid} \"{path}\"", + raddr = info.remote_addr().unwrap_or(([0, 0, 0, 0], 0).into()), + path = info.path(), + agent_uid = info.user_agent() + .map(|uid: &str| uid.splitn(3, '-') + .take(2) + .collect::() + ) + .unwrap_or_else(|| "NO_AGENT".to_string()) + ); +} + fn ok(_: T) -> impl Reply { reply() } diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml index d148d39..f988509 100644 --- a/integration/docker-compose.yml +++ b/integration/docker-compose.yml @@ -29,7 +29,7 @@ services: - ../.env - ../.env.private environment: - RUST_LOG: info + RUST_LOG: warp=info,u_server_lib=debug healthcheck: test: ss -tlpn | grep 63714 interval: 5s @@ -80,6 +80,7 @@ services: networks: - u_net volumes: + - ${HOME}/.cargo/registry/:/usr/local/cargo/registry/ - ../__Cargo_integration.toml:/tests/Cargo.toml - ./:/tests/integration/ - ../certs:/tests/certs diff --git a/integration/integration_tests.py b/integration/integration_tests.py index 43c54ff..e2b2028 100644 --- a/integration/integration_tests.py +++ b/integration/integration_tests.py @@ -59,7 +59,7 @@ def run_tests(): if not only_setup_cluster: CLUSTER.run('cargo test --test integration') except Exception as e: - # CLUSTER.print_containers_logs() + CLUSTER.print_containers_logs() fail(e) finally: _cleanup() diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs index 1096aac..5f825d1 100644 --- a/integration/tests/fixtures/agent.rs +++ b/integration/tests/fixtures/agent.rs @@ -17,6 +17,7 @@ impl RegisteredAgent { pub async fn register_agent() -> RegisteredAgent { let cli = ClientHandler::new(&ENV.u_server, None); let agent_uid = Uuid::new_v4(); + println!("registering agent {agent_uid}"); let resp = cli .get_personal_jobs(agent_uid) .await @@ -27,6 +28,7 @@ pub async fn register_agent() -> RegisteredAgent { let job = cli.get_jobs(Some(job_id)).await.unwrap().pop().unwrap(); assert_eq!(job.alias, Some("agent_hello".to_string())); let mut agent_data = AssignedJob::from(&job); + agent_data.agent_id = agent_uid; agent_data.set_result(&Agent { id: agent_uid, ..Default::default() diff --git a/integration/tests/helpers/panel.rs b/integration/tests/helpers/panel.rs index d381898..4caecb2 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration/tests/helpers/panel.rs @@ -43,8 +43,8 @@ impl Panel { .as_ref(), ); match &result { - PanelResult::Ok(r) => eprintln!("<<<+ {r:?}"), - PanelResult::Err(e) => eprintln!("<< eprintln!("<<<+ {r:02x?}"), + PanelResult::Err(e) => eprintln!("<<) -> Self { let identity = Identity::from_pkcs12_der(AGENT_IDENTITY, "").unwrap(); - let mut client = Client::builder().identity(identity); + let mut default_headers = HashMap::from([( + "user-agent".to_string(), + get_self_uid().hyphenated().to_string(), + )]); + if let Some(pwd) = password { - client = client.default_headers( - HeaderMap::try_from(&HashMap::from([( - "Authorization".to_string(), - format!("Bearer {pwd}"), - )])) - .unwrap(), - ) + default_headers.insert("authorization".to_string(), format!("Bearer {pwd}")); } - let client = client + + let client = Client::builder() + .identity(identity) + .default_headers(HeaderMap::try_from(&default_headers).unwrap()) .add_root_certificate(Certificate::from_pem(ROOT_CA_CERT).unwrap()) .build() .unwrap(); + Self { client, base_url: Url::parse(&format!("https://{}:{}", server, MASTER_PORT)).unwrap(), diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index b00a677..227ce90 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -96,7 +96,7 @@ impl Agent { hostname: decoder(builder.pop("hostname")), is_root: &decoder(builder.pop("is_root")) == "0", username: decoder(builder.pop("username")), - platform: Platform::current().into_string(), + platform: Platform::current_as_string(), ..Default::default() } } diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index e839f8e..8150313 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -31,7 +31,7 @@ pub struct JobMeta { pub exec_type: JobType, ///target triple - #[serde(default)] + #[serde(default = "Platform::current_as_string")] pub platform: String, #[serde(default)] @@ -67,7 +67,7 @@ impl Default for JobMeta { alias: None, argv: String::new(), exec_type: JobType::Shell, - platform: Platform::current().into_string(), + platform: Platform::current_as_string(), payload: None, schedule: None, payload_path: None, diff --git a/lib/u_lib/src/utils/platform.rs b/lib/u_lib/src/utils/platform.rs index 6ecb82b..04aa4ae 100644 --- a/lib/u_lib/src/utils/platform.rs +++ b/lib/u_lib/src/utils/platform.rs @@ -15,6 +15,10 @@ impl Platform { Self(guess_host_triple().unwrap_or("unknown").to_string()) } + pub fn current_as_string() -> String { + Self::current().into_string() + } + pub fn matches(&self, pf: impl AsRef) -> bool { match PlatformReq::from_str(pf.as_ref()) { Ok(p) => p.matches(&_Platform::find(&self.0).unwrap()), diff --git a/migrations/2020-10-24-111622_create_all/up.sql b/migrations/2020-10-24-111622_create_all/up.sql index c2d5ac9..4edc097 100644 --- a/migrations/2020-10-24-111622_create_all/up.sql +++ b/migrations/2020-10-24-111622_create_all/up.sql @@ -4,60 +4,58 @@ CREATE TYPE JobState AS ENUM ('queued', 'running', 'finished'); CREATE TYPE AgentState AS ENUM ('new', 'active', 'banned'); CREATE TABLE IF NOT EXISTS agents ( - alias TEXT - , hostname TEXT NOT NULL - , id UUID NOT NULL DEFAULT uuid_generate_v4() - , ip_gray TEXT - , ip_white TEXT - , is_root BOOLEAN NOT NULL DEFAULT false - , is_root_allowed BOOLEAN NOT NULL DEFAULT false - , last_active TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - -- target triplet - , platform TEXT NOT NULL - , regtime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - , state AgentState NOT NULL DEFAULT 'new' - -- is needed to processing requests - , token TEXT - , username TEXT NOT NULL - - , PRIMARY KEY(id) + alias TEXT, + hostname TEXT NOT NULL, + id UUID NOT NULL DEFAULT uuid_generate_v4(), + ip_gray TEXT, + ip_white TEXT, + is_root BOOLEAN NOT NULL DEFAULT false, + is_root_allowed BOOLEAN NOT NULL DEFAULT false, + last_active TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + platform TEXT NOT NULL, + regtime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + state AgentState NOT NULL DEFAULT 'new', + token TEXT, + username TEXT NOT NULL, + + PRIMARY KEY(id) ); CREATE TABLE IF NOT EXISTS jobs ( - alias TEXT - , argv TEXT NOT NULL - , id UUID NOT NULL DEFAULT uuid_generate_v4() - , exec_type JobType NOT NULL DEFAULT 'shell' - , platform TEXT NOT NULL - , payload BYTEA - , payload_path TEXT - , schedule TEXT - - , PRIMARY KEY(id) + alias TEXT, + argv TEXT NOT NULL, + id UUID NOT NULL DEFAULT uuid_generate_v4(), + exec_type JobType NOT NULL DEFAULT 'shell', + platform TEXT NOT NULL, + payload BYTEA, + payload_path TEXT, + schedule TEXT, + + PRIMARY KEY(id) ); CREATE TABLE IF NOT EXISTS results ( - agent_id UUID NOT NULL - , alias TEXT - , created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - , id UUID NOT NULL DEFAULT uuid_generate_v4() - , job_id UUID NOT NULL - , result BYTEA - , state JobState NOT NULL DEFAULT 'queued' - , exec_type JobType NOT NULL DEFAULT 'shell' - , retcode INTEGER - , updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - - , FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE - , FOREIGN KEY(job_id) REFERENCES jobs(id) ON DELETE CASCADE - , PRIMARY KEY(id) + agent_id UUID NOT NULL, + alias TEXT, + created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + id UUID NOT NULL DEFAULT uuid_generate_v4(), + job_id UUID NOT NULL, + result BYTEA, + state JobState NOT NULL DEFAULT 'queued', + exec_type JobType NOT NULL DEFAULT 'shell', + retcode INTEGER, + updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + + FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE, + FOREIGN KEY(job_id) REFERENCES jobs(id) ON DELETE CASCADE, + PRIMARY KEY(id) ); CREATE TABLE IF NOT EXISTS certificates ( - agent_id UUID NOT NULL - , id UUID NOT NULL DEFAULT uuid_generate_v4() - , is_revoked BOOLEAN NOT NULL DEFAULT FALSE + agent_id UUID NOT NULL, + id UUID NOT NULL DEFAULT uuid_generate_v4(), + is_revoked BOOLEAN NOT NULL DEFAULT FALSE, - , PRIMARY KEY(id) - , FOREIGN KEY(agent_id) REFERENCES agents(id) + PRIMARY KEY(id), + FOREIGN KEY(agent_id) REFERENCES agents(id) ); \ No newline at end of file diff --git a/todos.txt b/spec.txt similarity index 60% rename from todos.txt rename to spec.txt index 5be7b2a..53135f6 100644 --- a/todos.txt +++ b/spec.txt @@ -1,7 +1,9 @@ +todos: Upload/download files More tests Agent update (use more JobType's) Erase log macros in release mode Bump wine version to test agent on windows -Store downloaded payload on disk instead of memory -Improve web interface \ No newline at end of file +Store downloaded payload on disk instead of ram +Improve web interface +Migrator binary From 544c07cf8d558457f448c398446b887a4aa52da4 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Fri, 7 Oct 2022 04:57:49 +0500 Subject: [PATCH 28/30] 4:43 AM commit back: - fix broken deps - add connection check to panel - pretty-print panel errors - touch timestamps of updates - simplify u_server api - simplify u_lib api front: - add routing - use async instead of rxjs - define tables' columns in templates, not in ts - use different templates for different tables --- Cargo.toml | 3 +- bin/u_panel/src/argparse.rs | 4 +- .../server/fe/src/app/app-routing.module.ts | 11 ++- .../src/server/fe/src/app/app.component.html | 17 ++-- .../fe/src/app/core/services/api.service.ts | 36 +++++--- .../src/app/core/tables/agent.component.html | 68 +++++++++++++++ .../fe/src/app/core/tables/agent.component.ts | 83 ++++++++++--------- .../tables/dialogs/agent-info-dialog.html | 3 + .../tables/dialogs/agent_info.component.ts | 2 +- .../fe/src/app/core/tables/job.component.html | 68 +++++++++++++++ .../fe/src/app/core/tables/job.component.ts | 39 +-------- .../src/app/core/tables/result.component.html | 68 +++++++++++++++ .../src/app/core/tables/result.component.ts | 47 +++-------- .../src/app/core/tables/table.component.html | 37 --------- .../src/app/core/tables/table.component.less | 3 +- .../fe/src/app/core/tables/table.component.ts | 44 ++++------ .../src/server/fe/src/app/core/utils.ts | 4 + bin/u_panel/src/server/mod.rs | 10 ++- bin/u_server/Cargo.toml | 2 +- bin/u_server/src/db.rs | 11 +-- bin/u_server/src/handlers.rs | 21 ++--- bin/u_server/src/u_server.rs | 17 ++-- integration/tests/fixtures/agent.rs | 5 +- lib/u_lib/Cargo.toml | 2 +- lib/u_lib/src/api.rs | 50 ++++++----- lib/u_lib/src/config.rs | 1 + lib/u_lib/src/logging.rs | 1 + lib/u_lib/src/messaging/mod.rs | 5 +- lib/u_lib/src/models/agent.rs | 4 + lib/u_lib/src/models/jobs/assigned.rs | 4 + lib/u_lib/src/runner.rs | 33 +++++--- 31 files changed, 427 insertions(+), 276 deletions(-) create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/job.component.html create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/result.component.html delete mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/table.component.html diff --git a/Cargo.toml b/Cargo.toml index 8f5feed..bf4d243 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,10 +9,11 @@ members = [ ] [workspace.dependencies] -anyhow = "1.0.58" +anyhow = "=1.0.63" reqwest = { version = "0.11", features = ["json"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +thiserror = "=1.0.31" tokio = { version = "1.11", features = ["macros"] } tracing = "0.1.35" tracing-appender = "0.2.0" diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 80261f4..267f383 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -20,6 +20,7 @@ enum Cmd { Agents(RUD), Jobs(JobCRUD), Map(JobMapCRUD), + Ping, //TUI(TUIArgs), Serve, } @@ -119,13 +120,14 @@ pub async fn process_cmd(client: ClientHandler, args: Args) -> UResult { } JobMapCRUD::RUD(RUD::Delete { uid }) => to_json(client.del(uid).await), }, + Cmd::Ping => to_json(client.ping().await), /*Cmd::TUI(args) => crate::tui::init_tui(&args) .await .map_err(|e| UError::PanelError(e.to_string()))?,*/ Cmd::Serve => { crate::server::serve(client) .await - .map_err(|e| UError::PanelError(e.to_string()))?; + .map_err(|e| UError::PanelError(format!("{e:?}")))?; String::new() } }) diff --git a/bin/u_panel/src/server/fe/src/app/app-routing.module.ts b/bin/u_panel/src/server/fe/src/app/app-routing.module.ts index 0297262..0ec7bf4 100644 --- a/bin/u_panel/src/server/fe/src/app/app-routing.module.ts +++ b/bin/u_panel/src/server/fe/src/app/app-routing.module.ts @@ -1,7 +1,16 @@ import { NgModule } from '@angular/core'; import { RouterModule, Routes } from '@angular/router'; +import { AgentComponent } from './core/tables/agent.component'; +import { JobComponent } from './core/tables/job.component'; +import { ResultComponent } from './core/tables/result.component'; +import { AgentInfoDialogComponent } from './core/tables/dialogs/agent_info.component'; -const routes: Routes = []; +const routes: Routes = [ + { path: '', redirectTo: 'agents', pathMatch: 'full' }, + { path: 'agents', component: AgentComponent }, + { path: 'jobs', component: JobComponent }, + { path: 'results', component: ResultComponent }, +]; @NgModule({ imports: [RouterModule.forRoot(routes)], diff --git a/bin/u_panel/src/server/fe/src/app/app.component.html b/bin/u_panel/src/server/fe/src/app/app.component.html index 6aedf2e..8bb207a 100644 --- a/bin/u_panel/src/server/fe/src/app/app.component.html +++ b/bin/u_panel/src/server/fe/src/app/app.component.html @@ -1,11 +1,6 @@ - - - - - - - - - - - \ No newline at end of file + + \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts b/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts index cc04493..a7a609b 100644 --- a/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts +++ b/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@angular/core'; import { environment } from 'src/environments/environment'; import { HttpClient } from '@angular/common/http'; -import { Observable } from 'rxjs'; +import { firstValueFrom } from 'rxjs'; interface ServerResponse { status: "ok" | "err", @@ -17,27 +17,37 @@ export class ApiTableService { requestUrl = `${environment.server}/cmd/`; - req(cmd: string): Observable> { - return this.http.post>(this.requestUrl, cmd); + async req(cmd: string): Promise> { + return await firstValueFrom(this.http.post>(this.requestUrl, cmd)) } - getOne(id: string): Observable> { - return this.req(`${this.area} read ${id}`) + async getOne(id: string): Promise> { + const resp = await this.req(`${this.area} read ${id}`) + if (resp.data.length === 0) { + return { + status: 'err', + data: `${id} not found in ${this.area}` + } + } + return { + status: resp.status, + data: resp.data[0] + } } - getMany(): Observable> { - return this.req(`${this.area} read`) + async getMany(): Promise> { + return await this.req(`${this.area} read`) } - update(item: T): Observable> { - return this.req(`${this.area} update '${JSON.stringify(item)}'`) + async update(item: T): Promise> { + return await this.req(`${this.area} update '${JSON.stringify(item)}'`) } - delete(id: string): Observable> { - return this.req(`${this.area} delete ${id}`) + async delete(id: string): Promise> { + return await this.req(`${this.area} delete ${id}`) } - create(item: string): Observable> { - return this.req(`${this.area} create ${item}`) + async create(item: string): Promise> { + return await this.req(`${this.area} create ${item}`) } } \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html new file mode 100644 index 0000000..80318c1 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html @@ -0,0 +1,68 @@ +
+ +
+
+ +
+ + Filter + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ID + {{row.id}} + Alias + {{row.alias}} + User + {{row.username}} + Hostname + {{row.hostname}} + Last active + {{row.last_active.secs_since_epoch * 1000 | date:'long'}} + + +
No data
+
+ + +
\ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts index dea2e61..c7ae033 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts @@ -1,57 +1,62 @@ -import { Component, OnInit } from '@angular/core'; +import { Component, OnDestroy, OnInit } from '@angular/core'; import { TablesComponent } from './table.component'; import { AgentModel } from '../models'; import { AgentInfoDialogComponent } from './dialogs/agent_info.component'; import { HttpClient } from '@angular/common/http'; import { MatDialog } from '@angular/material/dialog'; import { epochToStr } from '../utils'; +import { ActivatedRoute, Router } from '@angular/router'; +import { emitErr } from '../utils'; +import { Subscription } from 'rxjs'; @Component({ selector: 'agent-table', - templateUrl: './table.component.html', + templateUrl: './agent.component.html', styleUrls: ['./table.component.less'] }) -export class AgentComponent extends TablesComponent { +export class AgentComponent extends TablesComponent implements OnDestroy, OnInit { - constructor(public override _httpClient: HttpClient, public override info_dlg: MatDialog) { + dialogSubscr!: Subscription; + area = 'agents' as const; + + displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions'] + + constructor( + public override _httpClient: HttpClient, + public override info_dlg: MatDialog, + public route: ActivatedRoute, + public router: Router + ) { super(_httpClient, info_dlg); } - area = 'agents' as const; + override ngOnInit(): void { + super.ngOnInit() + this.dialogSubscr = this.route.queryParams.subscribe(params => { + const id = params['id'] + if (id) { + this.show_item_dialog(id); + } + }) + } + + show_item_dialog(id: string) { + this.data_source!.getOne(id).then(resp => { + if (resp.status === 'ok') { + const dialog = this.info_dlg.open(AgentInfoDialogComponent, { + data: resp.data as AgentModel + }); + + dialog.afterClosed().subscribe(result => { + this.router.navigate(['.'], { relativeTo: this.route }) + }) + } else { + emitErr(resp.data) + } + }).catch(emitErr) + } - columns = [ - { - def: "id", - name: "ID", - cell: (cell: AgentModel) => cell.id - }, - { - def: "alias", - name: "Alias", - cell: (cell: AgentModel) => cell.alias ?? "" - }, - { - def: "username", - name: "User", - cell: (cell: AgentModel) => cell.username - }, - { - def: "hostname", - name: "Host", - cell: (cell: AgentModel) => cell.hostname - }, - { - def: "last_active", - name: "Last active", - cell: (cell: AgentModel) => epochToStr(cell.last_active.secs_since_epoch) - }, - ] - - displayedColumns = this.columns.map((c) => c.def); - - show_item_dialog(obj: AgentModel) { - const dialog = this.info_dlg.open(AgentInfoDialogComponent, { - data: obj - }); + ngOnDestroy(): void { + this.dialogSubscr.unsubscribe() } } diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html index eaf5e5a..2afdeab 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html @@ -1,9 +1,12 @@
+

ID: {{data.id}}

Alias: {{data.alias}}

Username: {{data.username}}

Hostname: {{data.hostname}}

Platform: {{data.platform}}

+

Registration time: {{data.regtime.secs_since_epoch * 1000 | date:'long'}}

+

Last active time: {{data.last_active.secs_since_epoch * 1000 | date:'long'}}

 diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts index 1c318ca..1d85364 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts @@ -8,4 +8,4 @@ import { AgentModel } from '../../models/agent.model'; }) export class AgentInfoDialogComponent { constructor(@Inject(MAT_DIALOG_DATA) public data: AgentModel) { } -} \ No newline at end of file +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html new file mode 100644 index 0000000..1234aa6 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html @@ -0,0 +1,68 @@ +
+ +
+
+ +
+ + Filter + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ID + {{row.id}} + Alias + {{row.alias}} + Cmd-line args + {{row.argv}} + Platform + {{row.platform}} + Payload + {{row.payload}} + Type + {{row.exec_type}} +
No data
+
+ + +
\ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts index 28f9c92..0403bea 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts @@ -4,45 +4,12 @@ import { JobModel } from '../models'; @Component({ selector: 'job-table', - templateUrl: './table.component.html', + templateUrl: './job.component.html', styleUrls: ['./table.component.less'] }) export class JobComponent extends TablesComponent { area = 'jobs' as const; + displayedColumns = ['id', 'alias', 'argv', 'platform', 'payload', 'exec_type'] - columns = [ - { - def: "id", - name: "ID", - cell: (cell: JobModel) => cell.id - }, - { - def: "alias", - name: "Alias", - cell: (cell: JobModel) => cell.alias - }, - { - def: "argv", - name: "Cmd-line args", - cell: (cell: JobModel) => cell.argv - }, - { - def: "platform", - name: "Platform", - cell: (cell: JobModel) => cell.platform - }, - { - def: "payload", - name: "Payload", - cell: (cell: JobModel) => `${cell.payload}` - }, - { - def: "etype", - name: "Type", - cell: (cell: JobModel) => cell.exec_type - }, - ] - displayedColumns = this.columns.map((c) => c.def); - - show_item_dialog(obj: JobModel) { } + show_item_dialog(id: string) { } } diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html new file mode 100644 index 0000000..e9abbd7 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html @@ -0,0 +1,68 @@ +
+ +
+
+ +
+ + Filter + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ID + {{row.id}} + Alias + {{row.alias}} + Agent + {{row.agent_id}} + Job + {{row.job_id}} + State + {{row.state}} {{(row.state === "Finished") ? '(' + row.retcode + ')' : ''}} + ID + {{row.updated.secs_since_epoch * 1000| date:'long'}} +
No data
+
+ + +
\ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts index 3b7cade..b477094 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts @@ -4,48 +4,23 @@ import { ResultModel } from '../models'; import { epochToStr } from '../utils'; @Component({ - selector: 'result-table', - templateUrl: './table.component.html', + selector: 'results-table', + templateUrl: './result.component.html', styleUrls: ['./table.component.less'] }) export class ResultComponent extends TablesComponent { area = 'map' as const; - columns = [ - { - def: "id", - name: "ID", - cell: (cell: ResultModel) => cell.id - }, - { - def: "alias", - name: "Alias", - cell: (cell: ResultModel) => cell.alias - }, - { - def: "agent_id", - name: "Agent ID", - cell: (cell: ResultModel) => cell.agent_id - }, - { - def: "job_id", - name: "Job ID", - cell: (cell: ResultModel) => cell.job_id - }, - { - def: "state", - name: "State", - cell: (cell: ResultModel) => `${cell.state} `.concat((cell.state === "Finished") ? `(${cell.retcode})` : '') - }, - { - def: "last_updated", - name: "Last updated", - cell: (cell: ResultModel) => epochToStr(cell.updated.secs_since_epoch) - }, - ] - displayedColumns = this.columns.map((c) => c.def); + displayedColumns = [ + 'id', + 'alias', + 'agent_id', + 'job_id', + 'state', + 'last_updated' + ]; - show_item_dialog(obj: ResultModel) { + show_item_dialog(id: string) { } } diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.html deleted file mode 100644 index c54abd9..0000000 --- a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.html +++ /dev/null @@ -1,37 +0,0 @@ -
- -
-
- -
- - Filter - - - - - - - - - - - - - - - - - -
- {{column.name}} - - {{column.cell(row)}} -
No data
-
- - -
\ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less index 931742f..6b3ee6b 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less +++ b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less @@ -12,7 +12,7 @@ left: 0; bottom: 56px; right: 0; - background: rgba(0, 0, 0, 0.15); + //background: rgba(0, 0, 0, 0.15); z-index: 1; display: flex; align-items: center; @@ -25,5 +25,4 @@ .data-table-row:hover { background: whitesmoke; - cursor: pointer; } \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts index 39107d7..f0cb6da 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts @@ -9,7 +9,7 @@ import { MatDialog } from '@angular/material/dialog'; @Directive() export abstract class TablesComponent implements OnInit { abstract area: "agents" | "jobs" | "map"; - data_source!: ApiTableService | null; + data_source!: ApiTableService; table_data!: MatTableDataSource; isLoadingResults = true; @@ -20,34 +20,20 @@ export abstract class TablesComponent implements OnInit { ngOnInit() { this.data_source = new ApiTableService(this._httpClient, this.area); - this.fetch_many(); - // If the user changes the sort order, reset back to the first page. - //this.sort.sortChange.subscribe(() => (this.paginator.pageIndex = 0)); - + this.fetchMany(); } - fetch_many() { - timer(0) - .pipe( - startWith({}), - switchMap(() => { - this.isLoadingResults = true; - return this.data_source!.getMany().pipe(catchError(() => observableOf(null))); - }), - map(data => { - this.isLoadingResults = false; - - if (data === null) { - return "no data returned" - } - - // Only refresh the result length if there is new data. In case of rate - // limit errors, we do not want to reset the paginator to zero, as that - // would prevent users from re-triggering requests. - return data.data; - }), - ) - .subscribe(data => { if (typeof data !== 'string') { this.table_data.data = data } else { alert(`Error: ${data}`) } }); + async fetchMany() { + this.isLoadingResults = true; + //possibly needs try/catch + const data = await this.data_source!.getMany(); + this.isLoadingResults = false; + + if (typeof data.data !== 'string') { + this.table_data.data = data.data + } else { + alert(`Error: ${data}`) + }; } apply_filter(event: Event) { @@ -55,10 +41,8 @@ export abstract class TablesComponent implements OnInit { this.table_data.filter = filterValue.trim().toLowerCase(); } - abstract show_item_dialog(obj: T): void; - - abstract columns: ColumnDef[]; abstract displayedColumns: string[]; + abstract show_item_dialog(id: string): void; } type ColumnDef = { diff --git a/bin/u_panel/src/server/fe/src/app/core/utils.ts b/bin/u_panel/src/server/fe/src/app/core/utils.ts index 27aadd8..128a6a2 100644 --- a/bin/u_panel/src/server/fe/src/app/core/utils.ts +++ b/bin/u_panel/src/server/fe/src/app/core/utils.ts @@ -1,3 +1,7 @@ export function epochToStr(epoch: number): string { return new Date(epoch * 1000).toLocaleString('en-GB') +} + +export function emitErr(e: any) { + alert(e) } \ No newline at end of file diff --git a/bin/u_panel/src/server/mod.rs b/bin/u_panel/src/server/mod.rs index 49b08f7..1640e45 100644 --- a/bin/u_panel/src/server/mod.rs +++ b/bin/u_panel/src/server/mod.rs @@ -77,9 +77,12 @@ async fn send_cmd( ) } -pub async fn serve(client: ClientHandler) -> std::io::Result<()> { +pub async fn serve(client: ClientHandler) -> anyhow::Result<()> { + info!("Connecting to u_server..."); + client.ping().await?; + let addr = "127.0.0.1:8080"; - info!("Serving at http://{}", addr); + info!("Connected, instanciating u_panel at http://{}", addr); HttpServer::new(move || { App::new() @@ -92,5 +95,6 @@ pub async fn serve(client: ClientHandler) -> std::io::Result<()> { }) .bind(addr)? .run() - .await + .await?; + Ok(()) } diff --git a/bin/u_server/Cargo.toml b/bin/u_server/Cargo.toml index 474ecb1..1209d12 100644 --- a/bin/u_server/Cargo.toml +++ b/bin/u_server/Cargo.toml @@ -12,7 +12,7 @@ once_cell = "1.7.2" openssl = "*" serde = { workspace = true } serde_json = { workspace = true } -thiserror = "*" +thiserror = { workspace = true } tracing = { workspace = true } tokio = { workspace = true, features = ["macros"] } uuid = { workspace = true, features = ["serde", "v4"] } diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index e72b2f9..e18824f 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -152,13 +152,10 @@ impl UDB { let job_requests = job_uids .iter() - .map(|job_uid| { - debug!("set_jobs_for_agent: set {} for {}", job_uid, agent_uid); - AssignedJob { - job_id: *job_uid, - agent_id: *agent_uid, - ..Default::default() - } + .map(|job_uid| AssignedJob { + job_id: *job_uid, + agent_id: *agent_uid, + ..Default::default() }) .collect::>(); diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index 43139ab..39f3398 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -1,5 +1,3 @@ -use std::time::SystemTime; - use crate::db::UDB; use crate::error::Error; use u_lib::{ @@ -33,19 +31,22 @@ impl Endpoints { .map_err(From::from) } - pub async fn get_personal_jobs(uid: Option) -> EndpResult> { - let agents = UDB::lock_db().get_agents(uid)?; + pub async fn get_personal_jobs(uid: Uuid) -> EndpResult> { + let db = UDB::lock_db(); + let mut agents = db.get_agents(Some(uid))?; if agents.is_empty() { - let db = UDB::lock_db(); - db.insert_agent(&Agent::with_id(uid.unwrap()))?; + db.insert_agent(&Agent::with_id(uid))?; let job = db .find_job_by_alias("agent_hello")? .expect("agent_hello job not found"); - db.set_jobs_for_agent(&uid.unwrap(), &[job.id])?; + db.set_jobs_for_agent(&uid, &[job.id])?; + } else { + let mut agent = agents.pop().unwrap(); + agent.touch(); + db.update_agent(&agent)?; } - let result = UDB::lock_db().get_exact_jobs(uid, true)?; + let result = db.get_exact_jobs(Some(uid), true)?; - let db = UDB::lock_db(); for j in result.iter() { db.update_job_status(j.id, JobState::Running)?; } @@ -106,7 +107,7 @@ impl Endpoints { continue; } result.state = JobState::Finished; - result.updated = SystemTime::now(); + result.touch(); match result.exec_type { JobType::Init => match &result.result { Some(rbytes) => { diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 4417e8f..70a5b9f 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -91,7 +91,7 @@ pub fn init_endpoints( .map(into_message); let get_personal_jobs = path("get_personal_jobs") - .and(warp::path::param::().map(Some)) + .and(warp::path::param::()) .and_then(Endpoints::get_personal_jobs) .map(into_message); @@ -131,6 +131,8 @@ pub fn init_endpoints( .and_then(Endpoints::download) .map(ok); + let ping = path("ping").map(reply); + let auth_token = format!("Bearer {auth_token}",).into_boxed_str(); let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); @@ -143,7 +145,8 @@ pub fn init_endpoints( .or(update_agent) .or(update_job) .or(update_assigned_job) - .or(download)) + .or(download) + .or(ping)) .and(auth_header); let agent_zone = get_jobs.or(get_personal_jobs).or(report).or(download); @@ -151,7 +154,7 @@ pub fn init_endpoints( auth_zone.or(agent_zone) } -pub fn prefill_jobs() -> Result<(), ServerError> { +pub fn preload_jobs() -> Result<(), ServerError> { let job_alias = "agent_hello"; let if_job_exists = UDB::lock_db().find_job_by_alias(job_alias)?; if if_job_exists.is_none() { @@ -167,7 +170,7 @@ pub fn prefill_jobs() -> Result<(), ServerError> { pub async fn serve() -> Result<(), ServerError> { init_logger(Some("u_server")); - prefill_jobs()?; + preload_jobs()?; let certs_dir = PathBuf::from("certs"); let env = load_env::().map_err(|e| ServerError::Other(e.to_string()))?; @@ -192,6 +195,7 @@ async fn handle_rejection(rej: Rejection) -> Result { } else if rej.is_not_found() { RejResponse::not_found("not found placeholder") } else { + error!("{:?}", rej); RejResponse::internal() }; Ok(resp.into_response()) @@ -199,7 +203,7 @@ async fn handle_rejection(rej: Rejection) -> Result { fn logger(info: Info<'_>) { info!(target: "warp", - "{raddr} {agent_uid} \"{path}\"", + "{raddr} {agent_uid} \"{path}\" {status}", raddr = info.remote_addr().unwrap_or(([0, 0, 0, 0], 0).into()), path = info.path(), agent_uid = info.user_agent() @@ -207,7 +211,8 @@ fn logger(info: Info<'_>) { .take(2) .collect::() ) - .unwrap_or_else(|| "NO_AGENT".to_string()) + .unwrap_or_else(|| "NO_AGENT".to_string()), + status = info.status() ); } diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs index 5f825d1..2e6cdce 100644 --- a/integration/tests/fixtures/agent.rs +++ b/integration/tests/fixtures/agent.rs @@ -29,10 +29,7 @@ pub async fn register_agent() -> RegisteredAgent { assert_eq!(job.alias, Some("agent_hello".to_string())); let mut agent_data = AssignedJob::from(&job); agent_data.agent_id = agent_uid; - agent_data.set_result(&Agent { - id: agent_uid, - ..Default::default() - }); + agent_data.set_result(&Agent::with_id(agent_uid)); cli.report(Reportable::Assigned(agent_data)).await.unwrap(); RegisteredAgent { uid: agent_uid } } diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index 24b943f..1696f34 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -26,7 +26,7 @@ shlex = "1.0.0" serde = { workspace = true } serde_json = { workspace = true } strum = { version = "0.20", features = ["derive"] } -thiserror = "*" +thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "sync", "macros", "process", "time"] } tracing-appender = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index b18cc5c..b64913c 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -3,7 +3,7 @@ use std::fmt::Debug; use crate::{ config::{get_self_uid, MASTER_PORT}, - messaging::{self, AsMsg, BaseMessage, Empty}, + messaging::{self, AsMsg, BaseMessage}, models::{self}, utils::{opt_to_string, OneOrVec}, UError, @@ -48,17 +48,24 @@ impl ClientHandler { } } - async fn _req( + async fn req(&self, url: impl AsRef) -> Result { + self.req_with_payload(url, ()).await + } + + async fn req_with_payload( &self, url: impl AsRef, payload: P, - ) -> Result { + ) -> Result { let request = self .client .post(self.base_url.join(url.as_ref()).unwrap()) .json(&payload.as_message()); - let response = request.send().await.context("send")?; + let response = request + .send() + .await + .context("error while sending request")?; let content_len = response.content_length(); let is_success = match response.error_for_status_ref() { Ok(_) => Ok(()), @@ -67,7 +74,7 @@ impl ClientHandler { let resp = response.text().await.context("resp")?; debug!("url = {}, resp = {}", url.as_ref(), resp); match is_success { - Ok(_) => from_str::>(&resp) + Ok(_) => from_str::>(&resp) .map(|msg| msg.into_inner()) .or_else(|e| match content_len { Some(0) => Ok(Default::default()), @@ -81,24 +88,22 @@ impl ClientHandler { // get jobs for client pub async fn get_personal_jobs(&self, url_param: Uuid) -> Result> { - self._req(format!("get_personal_jobs/{}", url_param), Empty) - .await + self.req(format!("get_personal_jobs/{}", url_param)).await } // send something to server - pub async fn report(&self, payload: impl OneOrVec) -> Result { - self._req("report", payload.into_vec()).await + pub async fn report(&self, payload: impl OneOrVec) -> Result<()> { + self.req_with_payload("report", payload.into_vec()).await } // download file pub async fn dl(&self, file: String) -> Result> { - self._req(format!("dl/{file}"), Empty).await + self.req(format!("dl/{file}")).await } /// get all available jobs pub async fn get_jobs(&self, job: Option) -> Result> { - self._req(format!("get_jobs/{}", opt_to_string(job)), Empty) - .await + self.req(format!("get_jobs/{}", opt_to_string(job))).await } } @@ -107,23 +112,24 @@ impl ClientHandler { impl ClientHandler { /// agent listing pub async fn get_agents(&self, agent: Option) -> Result> { - self._req(format!("get_agents/{}", opt_to_string(agent)), Empty) + self.req(format!("get_agents/{}", opt_to_string(agent))) .await } /// update something - pub async fn update_item(&self, item: impl AsMsg + Debug) -> Result { - self._req("update_item", item).await + pub async fn update_item(&self, item: impl AsMsg + Debug) -> Result<()> { + self.req_with_payload("update_item", item).await } /// create and upload job - pub async fn upload_jobs(&self, payload: impl OneOrVec) -> Result { - self._req("upload_jobs", payload.into_vec()).await + pub async fn upload_jobs(&self, payload: impl OneOrVec) -> Result<()> { + self.req_with_payload("upload_jobs", payload.into_vec()) + .await } /// delete something pub async fn del(&self, item: Uuid) -> Result { - self._req(format!("del/{item}"), Empty).await + self.req(format!("del/{item}")).await } /// set jobs for any agent @@ -132,13 +138,17 @@ impl ClientHandler { agent: Uuid, job_idents: impl OneOrVec, ) -> Result> { - self._req(format!("set_jobs/{agent}"), job_idents.into_vec()) + self.req_with_payload(format!("set_jobs/{agent}"), job_idents.into_vec()) .await } /// get jobs for any agent pub async fn get_agent_jobs(&self, agent: Option) -> Result> { - self._req(format!("get_agent_jobs/{}", opt_to_string(agent)), Empty) + self.req(format!("get_agent_jobs/{}", opt_to_string(agent))) .await } + + pub async fn ping(&self) -> Result<()> { + self.req("ping").await + } } diff --git a/lib/u_lib/src/config.rs b/lib/u_lib/src/config.rs index 50d3408..3ab8fce 100644 --- a/lib/u_lib/src/config.rs +++ b/lib/u_lib/src/config.rs @@ -7,6 +7,7 @@ lazy_static! { static ref UID: Uuid = Uuid::new_v4(); } +#[inline] pub fn get_self_uid() -> Uuid { *UID } diff --git a/lib/u_lib/src/logging.rs b/lib/u_lib/src/logging.rs index 7c4b991..9c4f1db 100644 --- a/lib/u_lib/src/logging.rs +++ b/lib/u_lib/src/logging.rs @@ -12,6 +12,7 @@ pub fn init_logger(logfile: Option + Send + Sync + 'static>) { let reg = registry() .with(EnvFilter::from_default_env()) .with(fmt::layer()); + match logfile { Some(file) => reg .with( diff --git a/lib/u_lib/src/messaging/mod.rs b/lib/u_lib/src/messaging/mod.rs index b1342bf..f1e82e5 100644 --- a/lib/u_lib/src/messaging/mod.rs +++ b/lib/u_lib/src/messaging/mod.rs @@ -16,12 +16,9 @@ impl AsMsg for Reportable {} impl AsMsg for JobMeta {} impl AsMsg for String {} impl AsMsg for Uuid {} -impl AsMsg for Empty {} impl AsMsg for i32 {} impl AsMsg for u8 {} - -#[derive(Serialize, Deserialize, Clone, Default, Debug)] -pub struct Empty; +impl AsMsg for () {} #[derive(Serialize, Deserialize, Clone, PartialEq, Debug)] pub enum Reportable { diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index 227ce90..5ad5549 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -79,6 +79,10 @@ impl Agent { } } + pub fn touch(&mut self) { + self.last_active = SystemTime::now(); + } + #[cfg(unix)] pub async fn gather() -> Self { let mut builder = NamedJobRunner::from_shell(vec![ diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index 2230f70..78850a4 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -112,4 +112,8 @@ impl AssignedJob { pub fn set_result(&mut self, result: &S) { self.result = Some(serde_json::to_vec(result).unwrap()); } + + pub fn touch(&mut self) { + self.updated = SystemTime::now() + } } diff --git a/lib/u_lib/src/runner.rs b/lib/u_lib/src/runner.rs index 133a6c0..6383cea 100644 --- a/lib/u_lib/src/runner.rs +++ b/lib/u_lib/src/runner.rs @@ -12,24 +12,24 @@ use tokio::process::Command; pub struct JobRunner { waiter: Waiter, + is_running: bool, } impl JobRunner { pub fn from_jobs(jobs: impl OneOrVec) -> CombinedResult { let jobs = jobs.into_vec(); let mut waiter = Waiter::new(); - let mut result = CombinedResult::::new(); + let mut result = CombinedResult::new(); for job in jobs { //waiting for try-blocks stabilization - let built_job = (|| -> UResult<()> { + let built_job: UResult<()> = (|| { let meta = JobCache::get(job.job_id).ok_or(UError::NoJob(job.job_id))?; let curr_platform = Platform::current(); if !curr_platform.matches(&meta.platform) { return Err(UError::InsuitablePlatform( meta.platform.clone(), curr_platform.into_string(), - ) - .into()); + )); } let job = AssignedJob::from((&*meta, job)); waiter.push(run_assigned_job(job)); @@ -39,7 +39,10 @@ impl JobRunner { result.err(e) } } - result.ok(Self { waiter }); + result.ok(Self { + waiter, + is_running: false, + }); result } @@ -48,10 +51,10 @@ impl JobRunner { .into_vec() .into_iter() .map(|jm| { - let job_uid = jm.id; + let job_id = jm.id; JobCache::insert(jm); AssignedJobById { - job_id: job_uid, + job_id, ..Default::default() } }) @@ -62,17 +65,23 @@ impl JobRunner { /// Spawn jobs pub async fn spawn(mut self) -> Self { self.waiter = self.waiter.spawn().await; + self.is_running = true; self } /// Spawn jobs and wait for result pub async fn wait(self) -> Vec { - self.waiter.spawn().await.wait().await + let waiter = if !self.is_running { + self.spawn().await.waiter + } else { + self.waiter + }; + waiter.wait().await } /// Spawn one job and wait for result pub async fn wait_one(self) -> ExecResult { - self.waiter.spawn().await.wait().await.pop().unwrap() + self.wait().await.pop().unwrap() } } @@ -125,7 +134,7 @@ pub async fn run_assigned_job(mut job: AssignedJob) -> ExecResult { /// Store jobs and get results by name pub struct NamedJobRunner { - builder: Option, + runner: Option, job_names: Vec<&'static str>, results: HashMap<&'static str, ExecResult>, } @@ -163,14 +172,14 @@ impl NamedJobRunner { }) .collect(); Self { - builder: Some(JobRunner::from_meta(job_metas).unwrap_one()), + runner: Some(JobRunner::from_meta(job_metas).unwrap_one()), job_names, results: HashMap::new(), } } pub async fn wait(mut self) -> Self { - let results = self.builder.take().unwrap().wait().await; + let results = self.runner.take().unwrap().wait().await; for (name, result) in self.job_names.iter().zip(results.into_iter()) { self.results.insert(name, result); } From 862fb6b338a13e6b6060e2f3d540729e8f8c5293 Mon Sep 17 00:00:00 2001 From: plazmoid Date: Wed, 19 Oct 2022 03:27:16 +0500 Subject: [PATCH 29/30] more like crud - now can edit and delete agents - add host info --- .../src/server/fe/src/app/app.module.ts | 4 ++ .../fe/src/app/core/models/agent.model.ts | 1 + .../fe/src/app/core/services/api.service.ts | 6 +- .../src/app/core/tables/agent.component.html | 9 ++- .../fe/src/app/core/tables/agent.component.ts | 14 +++- .../tables/dialogs/agent-info-dialog.html | 69 ++++++++++++++++--- .../tables/dialogs/agent_info.component.ts | 14 +++- .../tables/dialogs/info-dialog.component.less | 3 + .../fe/src/app/core/tables/job.component.html | 2 +- .../src/app/core/tables/result.component.html | 2 +- .../src/app/core/tables/table.component.less | 4 ++ .../fe/src/app/core/tables/table.component.ts | 4 +- bin/u_server/src/handlers.rs | 2 +- integration/docker.py | 2 +- lib/u_lib/src/models/agent.rs | 11 ++- lib/u_lib/src/models/schema.rs | 1 + lib/u_lib/src/runner.rs | 7 +- .../2020-10-24-111622_create_all/up.sql | 1 + 18 files changed, 125 insertions(+), 31 deletions(-) create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/dialogs/info-dialog.component.less diff --git a/bin/u_panel/src/server/fe/src/app/app.module.ts b/bin/u_panel/src/server/fe/src/app/app.module.ts index a78705c..037df53 100644 --- a/bin/u_panel/src/server/fe/src/app/app.module.ts +++ b/bin/u_panel/src/server/fe/src/app/app.module.ts @@ -11,6 +11,8 @@ import { MatInputModule } from '@angular/material/input'; import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; import { HttpClientModule } from '@angular/common/http'; import { MatDialogModule } from '@angular/material/dialog'; +import { MatIconModule } from '@angular/material/icon'; +import { FormsModule } from '@angular/forms'; import { AgentComponent, JobComponent, ResultComponent } from './core/tables'; import { AgentInfoDialogComponent } from './core/tables/dialogs'; @@ -33,6 +35,8 @@ import { AgentInfoDialogComponent } from './core/tables/dialogs'; MatInputModule, MatDialogModule, MatProgressSpinnerModule, + MatIconModule, + FormsModule, BrowserAnimationsModule ], providers: [], diff --git a/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts b/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts index 8015ee2..cd8ef76 100644 --- a/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts +++ b/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts @@ -3,6 +3,7 @@ import { UTCDate } from "."; export interface AgentModel { alias: string | null, hostname: string, + host_info: string, id: string, is_root: boolean, is_root_allowed: boolean, diff --git a/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts b/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts index a7a609b..26d5810 100644 --- a/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts +++ b/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts @@ -21,12 +21,12 @@ export class ApiTableService { return await firstValueFrom(this.http.post>(this.requestUrl, cmd)) } - async getOne(id: string): Promise> { - const resp = await this.req(`${this.area} read ${id}`) + async getOne(id: string, area: string = this.area): Promise> { + const resp = await this.req(`${area} read ${id}`) if (resp.data.length === 0) { return { status: 'err', - data: `${id} not found in ${this.area}` + data: `${id} not found in ${area}` } } return { diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html index 80318c1..1bfd13a 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html +++ b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html @@ -8,7 +8,7 @@ Filter - + @@ -51,7 +51,12 @@ diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts index c7ae033..dae0ad0 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts @@ -44,10 +44,16 @@ export class AgentComponent extends TablesComponent implements OnDes this.data_source!.getOne(id).then(resp => { if (resp.status === 'ok') { const dialog = this.info_dlg.open(AgentInfoDialogComponent, { - data: resp.data as AgentModel + data: resp.data as AgentModel, + width: '500px', }); + const saveSub = dialog.componentInstance.onSave.subscribe(result => { + this.data_source!.update(result).then(_ => this.loadTableData()).catch(emitErr) + }) + dialog.afterClosed().subscribe(result => { + saveSub.unsubscribe() this.router.navigate(['.'], { relativeTo: this.route }) }) } else { @@ -56,6 +62,12 @@ export class AgentComponent extends TablesComponent implements OnDes }).catch(emitErr) } + deleteItem(id: string) { + if (confirm(`Delete ${id}?`)) { + this.data_source!.delete(id).catch(emitErr) + } + } + ngOnDestroy(): void { this.dialogSubscr.unsubscribe() } diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html index 2afdeab..260b345 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html @@ -1,15 +1,64 @@ +

Agent info

+

Editing agent info

-
-

ID: {{data.id}}

-

Alias: {{data.alias}}

-

Username: {{data.username}}

-

Hostname: {{data.hostname}}

-

Platform: {{data.platform}}

-

Registration time: {{data.regtime.secs_since_epoch * 1000 | date:'long'}}

-

Last active time: {{data.last_active.secs_since_epoch * 1000 | date:'long'}}

-
+

+ + ID + + +

+

+ + Alias + + +

+

+ + Username + + +

+

+ + Hostname + + +

+

+ + Host info + + +

+

+ + Platform + + +

+

+ + Is root + + +

+

+ + Registration time + + +

+

+ + Last active time + + +

- + + \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts index 1d85364..9195e61 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts @@ -1,11 +1,23 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA } from '@angular/material/dialog'; import { AgentModel } from '../../models/agent.model'; +import { EventEmitter } from '@angular/core'; +import { Input } from '@angular/core'; @Component({ selector: 'agent-info-dialog', templateUrl: 'agent-info-dialog.html', + styleUrls: ['info-dialog.component.less'] }) export class AgentInfoDialogComponent { + is_preview = true; + onSave = new EventEmitter(); + + constructor(@Inject(MAT_DIALOG_DATA) public data: AgentModel) { } -} \ No newline at end of file + + updateAgent() { + console.log(this.data); + this.onSave.emit(this.data); + } +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/info-dialog.component.less b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/info-dialog.component.less new file mode 100644 index 0000000..bc8e5fb --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/info-dialog.component.less @@ -0,0 +1,3 @@ +.info-dlg-field { + width: 100%; +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html index 1234aa6..f99fbfd 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html +++ b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html @@ -8,7 +8,7 @@ Filter - +
- + +
diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html index e9abbd7..d8bf8d9 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html +++ b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html @@ -8,7 +8,7 @@ Filter - +
diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less index 6b3ee6b..167dd69 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less +++ b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less @@ -23,6 +23,10 @@ margin-left: 10px; } +.data-table-row { + height: 30px; +} + .data-table-row:hover { background: whitesmoke; } \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts index f0cb6da..062fb90 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts @@ -20,10 +20,10 @@ export abstract class TablesComponent implements OnInit { ngOnInit() { this.data_source = new ApiTableService(this._httpClient, this.area); - this.fetchMany(); + this.loadTableData(); } - async fetchMany() { + async loadTableData() { this.isLoadingResults = true; //possibly needs try/catch const data = await this.data_source!.getMany(); diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index 39f3398..eb890e4 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -103,7 +103,7 @@ impl Endpoints { Reportable::Assigned(mut result) => { let result_agent_id = &result.agent_id; if id != *result_agent_id { - warn!("Ids are not equal! actual id: {id}, job id: {result_agent_id}"); + warn!("Ids are not equal! actual id: {id}, id from job: {result_agent_id}"); continue; } result.state = JobState::Finished; diff --git a/integration/docker.py b/integration/docker.py index 355e517..36035a3 100644 --- a/integration/docker.py +++ b/integration/docker.py @@ -90,7 +90,7 @@ class Compose: ] def __init__(self): - self.container_tpl = 'integration_%s_%d' + self.container_tpl = 'integration-%s-%d' self.cmd_container = self.container_tpl % ('tests_runner', 1) self.ALL_CONTAINERS = [self.container_tpl % (c, 1) for c in self.ALL_IMAGES] diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index 5ad5549..61caf8d 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -41,6 +41,7 @@ pub enum AgentState { pub struct Agent { pub alias: Option, pub hostname: String, + pub host_info: String, pub id: Uuid, pub ip_gray: Option, pub ip_white: Option, @@ -86,7 +87,8 @@ impl Agent { #[cfg(unix)] pub async fn gather() -> Self { let mut builder = NamedJobRunner::from_shell(vec![ - ("hostname", "hostname"), + ("hostname", "hostnamectl hostname"), + ("host_info", "hostnamectl --json=pretty"), ("is_root", "id -u"), ("username", "id -un"), ]) @@ -98,6 +100,7 @@ impl Agent { Self { hostname: decoder(builder.pop("hostname")), + host_info: decoder(builder.pop("host_info")), is_root: &decoder(builder.pop("is_root")) == "0", username: decoder(builder.pop("username")), platform: Platform::current_as_string(), @@ -105,11 +108,6 @@ impl Agent { } } - #[cfg(not(unix))] - pub async fn gather() -> Self { - todo!() - } - pub async fn run() -> Agent { Agent::gather().await } @@ -121,6 +119,7 @@ impl Default for Agent { alias: None, id: get_self_uid(), hostname: String::new(), + host_info: String::new(), is_root: false, is_root_allowed: false, last_active: SystemTime::now(), diff --git a/lib/u_lib/src/models/schema.rs b/lib/u_lib/src/models/schema.rs index fd2db7b..1b47bc7 100644 --- a/lib/u_lib/src/models/schema.rs +++ b/lib/u_lib/src/models/schema.rs @@ -4,6 +4,7 @@ table! { agents (id) { alias -> Nullable, hostname -> Text, + host_info -> Text, id -> Uuid, ip_gray -> Nullable, ip_white -> Nullable, diff --git a/lib/u_lib/src/runner.rs b/lib/u_lib/src/runner.rs index 6383cea..9d5576b 100644 --- a/lib/u_lib/src/runner.rs +++ b/lib/u_lib/src/runner.rs @@ -52,7 +52,9 @@ impl JobRunner { .into_iter() .map(|jm| { let job_id = jm.id; - JobCache::insert(jm); + if !JobCache::contains(job_id) { + JobCache::insert(jm); + } AssignedJobById { job_id, ..Default::default() @@ -166,8 +168,9 @@ impl NamedJobRunner { let job_metas: Vec = named_jobs .into_vec() .into_iter() - .map(|(alias, meta)| { + .map(|(alias, mut meta)| { job_names.push(alias); + meta.alias = Some(alias.to_string()); meta }) .collect(); diff --git a/migrations/2020-10-24-111622_create_all/up.sql b/migrations/2020-10-24-111622_create_all/up.sql index 4edc097..c99082f 100644 --- a/migrations/2020-10-24-111622_create_all/up.sql +++ b/migrations/2020-10-24-111622_create_all/up.sql @@ -6,6 +6,7 @@ CREATE TYPE AgentState AS ENUM ('new', 'active', 'banned'); CREATE TABLE IF NOT EXISTS agents ( alias TEXT, hostname TEXT NOT NULL, + host_info TEXT NOT NULL, id UUID NOT NULL DEFAULT uuid_generate_v4(), ip_gray TEXT, ip_white TEXT, From aad7772a713303c38ad840f63240a1e92657309d Mon Sep 17 00:00:00 2001 From: plazmoid Date: Fri, 21 Oct 2022 01:17:22 +0500 Subject: [PATCH 30/30] done. back: - remove tui deadcode - redirect all unknown links to index.html - fix integration tests - fix linking errors by removing wrong libc.a - split update_item into concrete endpoints - simplify ProcOutput (retain only stderr header) - fix PanelResult front: - load all assets from /core/ - improve current tab highlighting - show, edit and delete job and result info - allow to add and assign jobs --- bin/u_panel/src/argparse.rs | 115 +++---- bin/u_panel/src/main.rs | 4 +- .../src/server/{errors.rs => error.rs} | 0 bin/u_panel/src/server/fe/angular.json | 3 +- .../src/server/fe/src/app/app.component.html | 5 +- .../src/server/fe/src/app/app.component.ts | 5 + .../src/server/fe/src/app/app.module.ts | 21 +- .../fe/src/app/core/models/agent.model.ts | 4 +- .../server/fe/src/app/core/models/index.ts | 8 +- .../fe/src/app/core/models/job.model.ts | 8 +- .../fe/src/app/core/models/result.model.ts | 6 +- .../fe/src/app/core/services/api.service.ts | 18 +- .../src/app/core/tables/agent.component.html | 11 +- .../fe/src/app/core/tables/agent.component.ts | 58 +--- .../tables/dialogs/agent-info-dialog.html | 2 +- .../tables/dialogs/agent_info.component.ts | 4 +- .../tables/dialogs/assign-job-dialog.html | 12 + .../tables/dialogs/assign_job.component.ts | 33 ++ .../fe/src/app/core/tables/dialogs/index.ts | 5 +- .../tables/dialogs/info-dialog.component.less | 11 + .../core/tables/dialogs/job-info-dialog.html | 50 +++ .../core/tables/dialogs/job_info.component.ts | 30 ++ .../tables/dialogs/result-info-dialog.html | 53 +++ .../tables/dialogs/result_info.component.ts | 20 ++ .../fe/src/app/core/tables/job.component.html | 27 +- .../fe/src/app/core/tables/job.component.ts | 48 ++- .../src/app/core/tables/result.component.html | 19 +- .../src/app/core/tables/result.component.ts | 19 +- .../fe/src/app/core/tables/table.component.ts | 60 +++- .../src/server/fe/src/app/core/utils.ts | 4 - .../fe/src/environments/environment.prod.ts | 2 +- .../server/fe/src/environments/environment.ts | 2 +- bin/u_panel/src/server/fe/src/index.html | 6 +- bin/u_panel/src/server/mod.rs | 52 ++- bin/u_panel/src/tui/impls.rs | 61 ---- bin/u_panel/src/tui/mod.rs | 187 ---------- bin/u_panel/src/tui/utils.rs | 58 ---- bin/u_panel/src/tui/windows/confirm.rs | 127 ------- bin/u_panel/src/tui/windows/main_wnd.rs | 325 ------------------ bin/u_panel/src/tui/windows/mod.rs | 223 ------------ bin/u_server/src/db.rs | 8 +- bin/u_server/src/handlers.rs | 5 +- bin/u_server/src/u_server.rs | 14 +- images/musl-libs.Dockerfile | 2 +- integration/integration_tests.py | 9 +- integration/tests/helpers/panel.rs | 2 +- integration/tests/integration/behaviour.rs | 2 +- lib/u_lib/src/api.rs | 18 +- lib/u_lib/src/datatypes.rs | 16 + lib/u_lib/src/models/agent.rs | 40 +-- lib/u_lib/src/models/jobs/assigned.rs | 26 +- lib/u_lib/src/models/jobs/meta.rs | 9 +- lib/u_lib/src/models/schema.rs | 18 +- lib/u_lib/src/runner.rs | 10 +- lib/u_lib/src/utils/proc_output.rs | 113 ++---- scripts/build_musl_libs.sh | 4 +- 56 files changed, 645 insertions(+), 1357 deletions(-) rename bin/u_panel/src/server/{errors.rs => error.rs} (100%) create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign-job-dialog.html create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign_job.component.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job-info-dialog.html create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job_info.component.ts create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result-info-dialog.html create mode 100644 bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result_info.component.ts delete mode 100644 bin/u_panel/src/tui/impls.rs delete mode 100644 bin/u_panel/src/tui/mod.rs delete mode 100644 bin/u_panel/src/tui/utils.rs delete mode 100644 bin/u_panel/src/tui/windows/confirm.rs delete mode 100644 bin/u_panel/src/tui/windows/main_wnd.rs delete mode 100644 bin/u_panel/src/tui/windows/mod.rs diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 267f383..4e43067 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -1,4 +1,4 @@ -use anyhow::Result as AnyResult; +use serde_json::{from_str, to_value, Value}; use structopt::StructOpt; use u_lib::{ api::ClientHandler, @@ -21,16 +21,9 @@ enum Cmd { Jobs(JobCRUD), Map(JobMapCRUD), Ping, - //TUI(TUIArgs), Serve, } -#[derive(StructOpt, Debug)] -pub struct TUIArgs { - #[structopt(long)] - pub nogui: bool, -} - #[derive(StructOpt, Debug)] enum JobCRUD { Create { @@ -77,58 +70,60 @@ fn parse_uuid(src: &str) -> Result { Uuid::parse_str(src).map_err(|e| e.to_string()) } -pub async fn process_cmd(client: ClientHandler, args: Args) -> UResult { - fn to_json(data: AnyResult) -> String { - let result = match data { - Ok(r) => PanelResult::Ok(r), - Err(e) => PanelResult::Err(e.downcast().expect("unknown error type")), - }; - serde_json::to_string(&result).unwrap() - } +pub fn into_value(data: M) -> Value { + to_value(data).unwrap() +} - Ok(match args.cmd { - Cmd::Agents(action) => match action { - RUD::Read { uid } => to_json(client.get_agents(uid).await), - RUD::Update { item } => { - let agent = serde_json::from_str::(&item)?; - to_json(client.update_item(agent).await) +pub async fn process_cmd(client: ClientHandler, args: Args) -> PanelResult { + let catcher: UResult = (|| async { + Ok(match args.cmd { + Cmd::Agents(action) => match action { + RUD::Read { uid } => into_value(client.get_agents(uid).await?), + RUD::Update { item } => { + let agent = from_str::(&item)?; + into_value(client.update_agent(agent).await?) + } + RUD::Delete { uid } => into_value(client.del(uid).await?), + }, + Cmd::Jobs(action) => match action { + JobCRUD::Create { job } => { + let raw_job = from_str::(&job)?; + let job = raw_job.validated()?; + into_value(client.upload_jobs(job).await?) + } + JobCRUD::RUD(RUD::Read { uid }) => into_value(client.get_jobs(uid).await?), + JobCRUD::RUD(RUD::Update { item }) => { + let raw_job = from_str::(&item)?; + let job = raw_job.validated()?; + into_value(client.update_job(job).await?) + } + JobCRUD::RUD(RUD::Delete { uid }) => into_value(client.del(uid).await?), + }, + Cmd::Map(action) => match action { + JobMapCRUD::Create { + agent_uid, + job_idents, + } => into_value(client.set_jobs(agent_uid, job_idents).await?), + JobMapCRUD::RUD(RUD::Read { uid }) => into_value(client.get_agent_jobs(uid).await?), + JobMapCRUD::RUD(RUD::Update { item }) => { + let assigned = from_str::(&item)?; + into_value(client.update_result(assigned).await?) + } + JobMapCRUD::RUD(RUD::Delete { uid }) => into_value(client.del(uid).await?), + }, + Cmd::Ping => into_value(client.ping().await?), + Cmd::Serve => { + crate::server::serve(client) + .await + .map_err(|e| UError::PanelError(format!("{e:?}")))?; + Value::Null } - RUD::Delete { uid } => to_json(client.del(uid).await), - }, - Cmd::Jobs(action) => match action { - JobCRUD::Create { job } => { - let raw_job = serde_json::from_str::(&job)?; - let job = raw_job.into_builder().build()?; - to_json(client.upload_jobs(job).await) - } - JobCRUD::RUD(RUD::Read { uid }) => to_json(client.get_jobs(uid).await), - JobCRUD::RUD(RUD::Update { item }) => { - let job = serde_json::from_str::(&item)?; - to_json(client.update_item(job).await) - } - JobCRUD::RUD(RUD::Delete { uid }) => to_json(client.del(uid).await), - }, - Cmd::Map(action) => match action { - JobMapCRUD::Create { - agent_uid, - job_idents, - } => to_json(client.set_jobs(agent_uid, job_idents).await), - JobMapCRUD::RUD(RUD::Read { uid }) => to_json(client.get_agent_jobs(uid).await), - JobMapCRUD::RUD(RUD::Update { item }) => { - let assigned = serde_json::from_str::(&item)?; - to_json(client.update_item(assigned).await) - } - JobMapCRUD::RUD(RUD::Delete { uid }) => to_json(client.del(uid).await), - }, - Cmd::Ping => to_json(client.ping().await), - /*Cmd::TUI(args) => crate::tui::init_tui(&args) - .await - .map_err(|e| UError::PanelError(e.to_string()))?,*/ - Cmd::Serve => { - crate::server::serve(client) - .await - .map_err(|e| UError::PanelError(format!("{e:?}")))?; - String::new() - } - }) + }) + })() + .await; + + match catcher { + Ok(r) => PanelResult::Ok(r), + Err(e) => PanelResult::Err(e), + } } diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index ea263b8..595cbfa 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -1,6 +1,5 @@ mod argparse; mod server; -//mod tui; #[macro_use] extern crate tracing; @@ -27,7 +26,8 @@ async fn main() -> AnyResult<()> { let args = Args::from_args(); init_logger(None::<&str>); - let result = process_cmd(client, args).await?; + + let result = process_cmd(client, args).await.to_string(); println!("{result}"); Ok(()) } diff --git a/bin/u_panel/src/server/errors.rs b/bin/u_panel/src/server/error.rs similarity index 100% rename from bin/u_panel/src/server/errors.rs rename to bin/u_panel/src/server/error.rs diff --git a/bin/u_panel/src/server/fe/angular.json b/bin/u_panel/src/server/fe/angular.json index 3972326..d35487b 100644 --- a/bin/u_panel/src/server/fe/angular.json +++ b/bin/u_panel/src/server/fe/angular.json @@ -38,6 +38,7 @@ }, "configurations": { "production": { + "baseHref": "/core/", "budgets": [ { "type": "initial", @@ -110,4 +111,4 @@ } }, "defaultProject": "fe" -} +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/app.component.html b/bin/u_panel/src/server/fe/src/app/app.component.html index 8bb207a..b35dfd7 100644 --- a/bin/u_panel/src/server/fe/src/app/app.component.html +++ b/bin/u_panel/src/server/fe/src/app/app.component.html @@ -1,6 +1,5 @@ \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/app.component.ts b/bin/u_panel/src/server/fe/src/app/app.component.ts index a76961e..566f46b 100644 --- a/bin/u_panel/src/server/fe/src/app/app.component.ts +++ b/bin/u_panel/src/server/fe/src/app/app.component.ts @@ -6,4 +6,9 @@ import { Component, ViewChild, AfterViewInit } from '@angular/core'; styleUrls: ['./app.component.less'] }) export class AppComponent { + tabs = [ + { name: 'Agents', link: '/agents' }, + { name: 'Jobs', link: '/jobs' }, + { name: 'Results', link: '/results' } + ]; } diff --git a/bin/u_panel/src/server/fe/src/app/app.module.ts b/bin/u_panel/src/server/fe/src/app/app.module.ts index 037df53..c1572f4 100644 --- a/bin/u_panel/src/server/fe/src/app/app.module.ts +++ b/bin/u_panel/src/server/fe/src/app/app.module.ts @@ -14,7 +14,16 @@ import { MatDialogModule } from '@angular/material/dialog'; import { MatIconModule } from '@angular/material/icon'; import { FormsModule } from '@angular/forms'; import { AgentComponent, JobComponent, ResultComponent } from './core/tables'; -import { AgentInfoDialogComponent } from './core/tables/dialogs'; +import { + AgentInfoDialogComponent, + AssignJobDialogComponent, + JobInfoDialogComponent, + ResultInfoDialogComponent +} from './core/tables/dialogs'; +import { APP_BASE_HREF } from '@angular/common'; +import { MatTooltipModule } from '@angular/material/tooltip'; +import { MatSnackBarModule } from '@angular/material/snack-bar'; +import { MatListModule } from '@angular/material/list'; @NgModule({ declarations: [ @@ -22,7 +31,10 @@ import { AgentInfoDialogComponent } from './core/tables/dialogs'; AgentComponent, JobComponent, ResultComponent, - AgentInfoDialogComponent + AgentInfoDialogComponent, + JobInfoDialogComponent, + ResultInfoDialogComponent, + AssignJobDialogComponent ], imports: [ BrowserModule, @@ -36,10 +48,13 @@ import { AgentInfoDialogComponent } from './core/tables/dialogs'; MatDialogModule, MatProgressSpinnerModule, MatIconModule, + MatTooltipModule, + MatSnackBarModule, + MatListModule, FormsModule, BrowserAnimationsModule ], - providers: [], + providers: [{ provide: APP_BASE_HREF, useValue: '/' }], bootstrap: [AppComponent] }) export class AppModule { } diff --git a/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts b/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts index cd8ef76..d798f0e 100644 --- a/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts +++ b/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts @@ -1,6 +1,6 @@ -import { UTCDate } from "."; +import { UTCDate, ApiModel } from "."; -export interface AgentModel { +export interface AgentModel extends ApiModel { alias: string | null, hostname: string, host_info: string, diff --git a/bin/u_panel/src/server/fe/src/app/core/models/index.ts b/bin/u_panel/src/server/fe/src/app/core/models/index.ts index 38623ef..dad077a 100644 --- a/bin/u_panel/src/server/fe/src/app/core/models/index.ts +++ b/bin/u_panel/src/server/fe/src/app/core/models/index.ts @@ -5,4 +5,10 @@ export * from './job.model'; export interface UTCDate { secs_since_epoch: number, nanos_since_epoch: number -} \ No newline at end of file +} + +export abstract class ApiModel { } + +export interface Empty extends ApiModel { } + +export type Area = "agents" | "jobs" | "map"; \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/models/job.model.ts b/bin/u_panel/src/server/fe/src/app/core/models/job.model.ts index 66f1012..9db303f 100644 --- a/bin/u_panel/src/server/fe/src/app/core/models/job.model.ts +++ b/bin/u_panel/src/server/fe/src/app/core/models/job.model.ts @@ -1,8 +1,12 @@ -export interface JobModel { +import { ApiModel } from "."; + +export interface JobModel extends ApiModel { alias: string, argv: string, id: string, exec_type: string, platform: string, - payload: Uint8Array | null, + payload: number[] | null, + payload_path: string | null, + schedule: string | null, } \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/models/result.model.ts b/bin/u_panel/src/server/fe/src/app/core/models/result.model.ts index 825323b..c699787 100644 --- a/bin/u_panel/src/server/fe/src/app/core/models/result.model.ts +++ b/bin/u_panel/src/server/fe/src/app/core/models/result.model.ts @@ -1,12 +1,12 @@ -import { UTCDate } from "."; +import { UTCDate, ApiModel } from "."; -export interface ResultModel { +export interface ResultModel extends ApiModel { agent_id: string, alias: string, created: UTCDate, id: string, job_id: string, - result: Uint8Array, + result: number[], state: "Queued" | "Running" | "Finished", retcode: number | null, updated: UTCDate, diff --git a/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts b/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts index 26d5810..84348e2 100644 --- a/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts +++ b/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts @@ -1,23 +1,23 @@ -import { Injectable } from '@angular/core'; import { environment } from 'src/environments/environment'; import { HttpClient } from '@angular/common/http'; import { firstValueFrom } from 'rxjs'; +import { ApiModel, Empty, Area } from '../models'; -interface ServerResponse { +interface ServerResponse { status: "ok" | "err", data: T | string } -export class ApiTableService { - area: string; +export class ApiTableService { + area: Area; - constructor(private http: HttpClient, area: string) { + constructor(private http: HttpClient, area: Area) { this.area = area; } requestUrl = `${environment.server}/cmd/`; - async req(cmd: string): Promise> { + async req(cmd: string): Promise> { return await firstValueFrom(this.http.post>(this.requestUrl, cmd)) } @@ -39,15 +39,15 @@ export class ApiTableService { return await this.req(`${this.area} read`) } - async update(item: T): Promise> { + async update(item: T): Promise> { return await this.req(`${this.area} update '${JSON.stringify(item)}'`) } - async delete(id: string): Promise> { + async delete(id: string): Promise> { return await this.req(`${this.area} delete ${id}`) } - async create(item: string): Promise> { + async create(item: string): Promise> { return await this.req(`${this.area} create ${item}`) } } \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html index 1bfd13a..fbaaba9 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html +++ b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html @@ -10,8 +10,8 @@ -
+
@@ -51,9 +51,14 @@
ID - + | + + | diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts index dae0ad0..c72b8dc 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts @@ -2,54 +2,35 @@ import { Component, OnDestroy, OnInit } from '@angular/core'; import { TablesComponent } from './table.component'; import { AgentModel } from '../models'; import { AgentInfoDialogComponent } from './dialogs/agent_info.component'; -import { HttpClient } from '@angular/common/http'; -import { MatDialog } from '@angular/material/dialog'; -import { epochToStr } from '../utils'; -import { ActivatedRoute, Router } from '@angular/router'; -import { emitErr } from '../utils'; -import { Subscription } from 'rxjs'; +import { HttpErrorResponse } from '@angular/common/http'; +import { AssignJobDialogComponent } from './dialogs'; @Component({ selector: 'agent-table', templateUrl: './agent.component.html', styleUrls: ['./table.component.less'] }) -export class AgentComponent extends TablesComponent implements OnDestroy, OnInit { +export class AgentComponent extends TablesComponent implements OnInit { - dialogSubscr!: Subscription; + //dialogSubscr!: Subscription; area = 'agents' as const; displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions'] - constructor( - public override _httpClient: HttpClient, - public override info_dlg: MatDialog, - public route: ActivatedRoute, - public router: Router - ) { - super(_httpClient, info_dlg); - } - - override ngOnInit(): void { - super.ngOnInit() - this.dialogSubscr = this.route.queryParams.subscribe(params => { - const id = params['id'] - if (id) { - this.show_item_dialog(id); - } - }) - } - show_item_dialog(id: string) { this.data_source!.getOne(id).then(resp => { if (resp.status === 'ok') { - const dialog = this.info_dlg.open(AgentInfoDialogComponent, { + const dialog = this.infoDialog.open(AgentInfoDialogComponent, { data: resp.data as AgentModel, - width: '500px', + width: '1000px', }); const saveSub = dialog.componentInstance.onSave.subscribe(result => { - this.data_source!.update(result).then(_ => this.loadTableData()).catch(emitErr) + this.data_source!.update(result).then(_ => { + this.openSnackBar('Saved', false) + this.loadTableData() + }) + .catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) }) dialog.afterClosed().subscribe(result => { @@ -57,18 +38,15 @@ export class AgentComponent extends TablesComponent implements OnDes this.router.navigate(['.'], { relativeTo: this.route }) }) } else { - emitErr(resp.data) + this.openSnackBar(resp.data) } - }).catch(emitErr) - } - - deleteItem(id: string) { - if (confirm(`Delete ${id}?`)) { - this.data_source!.delete(id).catch(emitErr) - } + }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) } - ngOnDestroy(): void { - this.dialogSubscr.unsubscribe() + assignJobs(id: string) { + const dialog = this.infoDialog.open(AssignJobDialogComponent, { + data: id, + width: '1000px', + }); } } diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html index 260b345..ff66ad5 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html @@ -28,7 +28,7 @@

Host info -

diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts index 9195e61..d24d3a9 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts @@ -2,7 +2,6 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA } from '@angular/material/dialog'; import { AgentModel } from '../../models/agent.model'; import { EventEmitter } from '@angular/core'; -import { Input } from '@angular/core'; @Component({ selector: 'agent-info-dialog', @@ -13,11 +12,10 @@ export class AgentInfoDialogComponent { is_preview = true; onSave = new EventEmitter(); - constructor(@Inject(MAT_DIALOG_DATA) public data: AgentModel) { } updateAgent() { console.log(this.data); this.onSave.emit(this.data); } -} \ No newline at end of file +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign-job-dialog.html b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign-job-dialog.html new file mode 100644 index 0000000..2ebe970 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign-job-dialog.html @@ -0,0 +1,12 @@ +

Assign job

+ + + + {{row}} + + + + + + + \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign_job.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign_job.component.ts new file mode 100644 index 0000000..cd278d6 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign_job.component.ts @@ -0,0 +1,33 @@ +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { HttpClient } from '@angular/common/http'; +import { ApiTableService } from '../../services'; +import { JobModel } from '../../models'; +import { MatListOption } from '@angular/material/list'; + +@Component({ + selector: 'assign-job-dialog', + templateUrl: 'assign-job-dialog.html', + styleUrls: [] +}) +export class AssignJobDialogComponent { + rows: string[] = []; + selected_rows: string[] = []; + + constructor(@Inject(MAT_DIALOG_DATA) public agent_id: string, private http: HttpClient) { + new ApiTableService(http, "jobs").getMany().then(result => { + if (result.status == "ok") { + const jobs = result.data as JobModel[] + this.rows = jobs.map(j => `${j.id} ${j.alias}`) + } else { + alert(result.data as string) + } + }).catch(err => alert(err)) + } + + assignSelectedJobs() { + const job_ids = this.selected_rows.map(row => row.split(' ', 1)[0]).join(' '); + const request = `${this.agent_id} ${job_ids}` + new ApiTableService(this.http, "map").create(request).catch(err => alert(err)) + } +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/index.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/index.ts index 1aa08ec..4bdb1aa 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/index.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/index.ts @@ -1 +1,4 @@ -export * from './agent_info.component'; \ No newline at end of file +export * from './agent_info.component'; +export * from './result_info.component'; +export * from './job_info.component'; +export * from './assign_job.component'; \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/info-dialog.component.less b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/info-dialog.component.less index bc8e5fb..23b8bc5 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/info-dialog.component.less +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/info-dialog.component.less @@ -1,3 +1,14 @@ .info-dlg-field { width: 100%; +} + +div.info-dialog-forms-box { + width: 100%; + margin-right: 10px; +} + +div.info-dialog-forms-box-smol { + width: 30%; + float: left; + margin-right: 10px; } \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job-info-dialog.html b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job-info-dialog.html new file mode 100644 index 0000000..79108e6 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job-info-dialog.html @@ -0,0 +1,50 @@ +

Job info

+

Editing job info

+ +
+ + ID + + + + Alias + + + + Args + + +
+
+ + Type + + + + Platform + + + + Schedule + + +
+
+ + Payload path + + +
+
+ + Payload + + +
+ + + + + + \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job_info.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job_info.component.ts new file mode 100644 index 0000000..08c29da --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job_info.component.ts @@ -0,0 +1,30 @@ +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { JobModel } from '../../models/job.model'; +import { EventEmitter } from '@angular/core'; + +@Component({ + selector: 'job-info-dialog', + templateUrl: 'job-info-dialog.html', + styleUrls: ['info-dialog.component.less'] +}) +export class JobInfoDialogComponent { + is_preview = true; + decodedPayload: string; + onSave = new EventEmitter(); + + constructor(@Inject(MAT_DIALOG_DATA) public data: JobModel) { + if (data.payload !== null) { + this.decodedPayload = new TextDecoder().decode(new Uint8Array(data.payload)) + } else { + this.decodedPayload = "" + } + } + + updateJob() { + if (this.decodedPayload.length > 0) { + this.data.payload = Array.from(new TextEncoder().encode(this.decodedPayload)) + } + this.onSave.emit(this.data); + } +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result-info-dialog.html b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result-info-dialog.html new file mode 100644 index 0000000..6ce43c2 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result-info-dialog.html @@ -0,0 +1,53 @@ +

Result

+ +
+ + ID + + + + Job ID + + + + Agent ID + + +
+
+ + Alias + + + + State + + + + Return code + + +
+
+ + Created + + + + Updated + + +
+
+

+ + Result + + +

+
+ + + + \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result_info.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result_info.component.ts new file mode 100644 index 0000000..b02fae5 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result_info.component.ts @@ -0,0 +1,20 @@ +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { ResultModel } from '../../models/result.model'; + +@Component({ + selector: 'result-info-dialog', + templateUrl: 'result-info-dialog.html', + styleUrls: ['info-dialog.component.less'] +}) +export class ResultInfoDialogComponent { + decodedResult: string; + + constructor(@Inject(MAT_DIALOG_DATA) public data: ResultModel) { + if (data.result !== null) { + this.decodedResult = new TextDecoder().decode(new Uint8Array(data.result)) + } else { + this.decodedResult = "" + } + } +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html index f99fbfd..bfd99b2 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html +++ b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html @@ -8,10 +8,12 @@ Filter - + + - +
@@ -41,10 +43,10 @@ - - + + @@ -55,6 +57,19 @@ + + + + + diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts index 0403bea..4a5e1cf 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts @@ -1,6 +1,8 @@ import { Component, OnInit } from '@angular/core'; import { TablesComponent } from './table.component'; import { JobModel } from '../models'; +import { JobInfoDialogComponent } from './dialogs'; +import { HttpErrorResponse } from '@angular/common/http'; @Component({ selector: 'job-table', @@ -9,7 +11,49 @@ import { JobModel } from '../models'; }) export class JobComponent extends TablesComponent { area = 'jobs' as const; - displayedColumns = ['id', 'alias', 'argv', 'platform', 'payload', 'exec_type'] + displayedColumns = ['id', 'alias', 'platform', 'schedule', 'exec_type', 'actions'] - show_item_dialog(id: string) { } + show_item_dialog(id: string | null) { + const show_dlg = (id: string, edit: boolean) => { + this.data_source!.getOne(id).then(resp => { + if (resp.status === 'ok') { + var dialog = this.infoDialog.open(JobInfoDialogComponent, { + data: resp.data as JobModel, + width: '1000px', + }); + if (edit) { + dialog.componentInstance.is_preview = false + } + + const saveSub = dialog.componentInstance.onSave.subscribe(result => { + this.data_source!.update(result) + .then(_ => { + this.openSnackBar("Saved", false) + this.loadTableData() + }) + .catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) + }) + + dialog.afterClosed().subscribe(result => { + saveSub.unsubscribe() + this.router.navigate(['.'], { relativeTo: this.route }) + }) + } else { + this.openSnackBar(resp.data) + } + }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) + } + + if (id) { + show_dlg(id, false) + } else { + this.data_source!.create('"{}"').then(resp => { + if (resp.status === 'ok') { + show_dlg(resp.data[0], true) + } else { + this.openSnackBar(resp.data) + } + }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) + } + } } diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html index d8bf8d9..afda1cb 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html +++ b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html @@ -10,8 +10,8 @@ -
ID PayloadSchedule - {{row.payload}} + {{row.schedule}} + + | + +
+
@@ -37,7 +37,7 @@ @@ -55,6 +55,19 @@ + + + + + diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts index b477094..ec98ab7 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts @@ -1,7 +1,8 @@ import { Component, OnInit } from '@angular/core'; import { TablesComponent } from './table.component'; import { ResultModel } from '../models'; -import { epochToStr } from '../utils'; +import { ResultInfoDialogComponent } from './dialogs'; +import { HttpErrorResponse } from '@angular/common/http'; @Component({ selector: 'results-table', @@ -17,10 +18,24 @@ export class ResultComponent extends TablesComponent { 'agent_id', 'job_id', 'state', - 'last_updated' + 'last_updated', + 'actions' ]; show_item_dialog(id: string) { + this.data_source!.getOne(id).then(resp => { + if (resp.status === 'ok') { + const dialog = this.infoDialog.open(ResultInfoDialogComponent, { + data: resp.data as ResultModel, + width: '1000px', + }); + dialog.afterClosed().subscribe(result => { + this.router.navigate(['.'], { relativeTo: this.route }) + }) + } else { + this.openSnackBar(resp.data) + } + }).catch((err: HttpErrorResponse) => this.openSnackBar(err.message)) } } diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts index 062fb90..5a7bd23 100644 --- a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts +++ b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts @@ -1,26 +1,45 @@ -import { Component, OnInit, Directive } from '@angular/core'; -import { timer, of as observableOf } from 'rxjs'; -import { catchError, map, startWith, switchMap } from 'rxjs/operators'; +import { OnInit, Directive } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { ApiTableService } from '../'; import { MatTableDataSource } from '@angular/material/table'; import { MatDialog } from '@angular/material/dialog'; +import { ApiModel, Area } from '../models'; +import { ActivatedRoute, Router } from '@angular/router'; +import { interval } from 'rxjs'; +import { MatSnackBar, MatSnackBarConfig } from '@angular/material/snack-bar'; @Directive() -export abstract class TablesComponent implements OnInit { - abstract area: "agents" | "jobs" | "map"; +export abstract class TablesComponent implements OnInit { + abstract area: Area; data_source!: ApiTableService; table_data!: MatTableDataSource; isLoadingResults = true; - constructor(public _httpClient: HttpClient, public info_dlg: MatDialog) { + constructor( + public httpClient: HttpClient, + public infoDialog: MatDialog, + public route: ActivatedRoute, + public router: Router, + public snackBar: MatSnackBar + ) { this.table_data = new MatTableDataSource; } ngOnInit() { - this.data_source = new ApiTableService(this._httpClient, this.area); + this.data_source = new ApiTableService(this.httpClient, this.area); this.loadTableData(); + this.route.queryParams.subscribe(params => { + const id = params['id'] + const new_agent = params['new'] + if (id) { + this.show_item_dialog(id); + } + if (new_agent) { + this.show_item_dialog(null); + } + }) + //interval(10000).subscribe(_ => this.loadTableData()); } async loadTableData() { @@ -41,12 +60,25 @@ export abstract class TablesComponent implements OnInit { this.table_data.filter = filterValue.trim().toLowerCase(); } - abstract displayedColumns: string[]; - abstract show_item_dialog(id: string): void; -} + deleteItem(id: string) { + if (confirm(`Delete ${id}?`)) { + this.data_source!.delete(id).catch(this.openSnackBar) + } + } -type ColumnDef = { - def: string, - name: string, - cell: (cell: C) => string + openSnackBar(message: any, error: boolean = true) { + const msg = JSON.stringify(message) + const _config = (duration: number): MatSnackBarConfig => { + return { + horizontalPosition: 'right', + verticalPosition: 'bottom', + duration + } + } + const cfg = error ? _config(0) : _config(2000) + this.snackBar.open(msg, 'Ok', cfg); + } + + abstract displayedColumns: string[]; + abstract show_item_dialog(id: string | null): void; } \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/utils.ts b/bin/u_panel/src/server/fe/src/app/core/utils.ts index 128a6a2..bc6c422 100644 --- a/bin/u_panel/src/server/fe/src/app/core/utils.ts +++ b/bin/u_panel/src/server/fe/src/app/core/utils.ts @@ -1,7 +1,3 @@ export function epochToStr(epoch: number): string { return new Date(epoch * 1000).toLocaleString('en-GB') } - -export function emitErr(e: any) { - alert(e) -} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/environments/environment.prod.ts b/bin/u_panel/src/server/fe/src/environments/environment.prod.ts index 95d0187..cc43895 100644 --- a/bin/u_panel/src/server/fe/src/environments/environment.prod.ts +++ b/bin/u_panel/src/server/fe/src/environments/environment.prod.ts @@ -1,4 +1,4 @@ export const environment = { production: true, - server: "" + server: "", }; diff --git a/bin/u_panel/src/server/fe/src/environments/environment.ts b/bin/u_panel/src/server/fe/src/environments/environment.ts index c016fed..1b3b824 100644 --- a/bin/u_panel/src/server/fe/src/environments/environment.ts +++ b/bin/u_panel/src/server/fe/src/environments/environment.ts @@ -4,7 +4,7 @@ export const environment = { production: false, - server: "http://127.0.0.1:8080" + server: "http://127.0.0.1:8080", }; /* diff --git a/bin/u_panel/src/server/fe/src/index.html b/bin/u_panel/src/server/fe/src/index.html index 94a6658..e299aec 100644 --- a/bin/u_panel/src/server/fe/src/index.html +++ b/bin/u_panel/src/server/fe/src/index.html @@ -1,16 +1,18 @@ + Fe - + - + + \ No newline at end of file diff --git a/bin/u_panel/src/server/mod.rs b/bin/u_panel/src/server/mod.rs index 1640e45..897be93 100644 --- a/bin/u_panel/src/server/mod.rs +++ b/bin/u_panel/src/server/mod.rs @@ -1,22 +1,9 @@ -/* -Tabs: Agents, Tasks, Summary -every tab has list page and item page with more info/actions - -Agents: - -| id | alias | ..see struct | tasks done -| stripped | alias | ... | clickable number of assigned jobs - -almost all fields are editable, rows are deletable - -*/ - -mod errors; +mod error; use crate::{process_cmd, Args}; use actix_cors::Cors; use actix_web::{get, middleware::Logger, post, web, App, HttpResponse, HttpServer, Responder}; -use errors::Error; +use error::Error; use futures_util::StreamExt; use rust_embed::RustEmbed; use std::borrow::Cow; @@ -34,16 +21,16 @@ impl Files { } } -#[get("/")] -async fn main_page() -> impl Responder { +async fn spa_main() -> impl Responder { let index = Files::get_static("index.html").unwrap(); HttpResponse::Ok().body(index) } -#[get("/{path}")] -async fn static_files_adapter(path: web::Path<(String,)>) -> impl Responder { +#[get("/core/{path}")] +async fn resources_adapter(path: web::Path<(String,)>) -> impl Responder { let path = path.into_inner().0; let mimetype = mime_guess::from_path(&path).first_or_octet_stream(); + match Files::get_static(path) { Some(data) => HttpResponse::Ok() .content_type(mimetype.to_string()) @@ -58,23 +45,33 @@ async fn send_cmd( client: web::Data, ) -> Result { let mut bytes = web::BytesMut::new(); + while let Some(item) = body.next().await { bytes.extend_from_slice( &item.map_err(|e| Error::JustError(format!("payload loading failure: {e}")))?, ); } + let cmd = String::from_utf8(bytes.to_vec()) .map_err(|_| Error::JustError("cmd contains non-utf8 data".to_string()))?; let mut cmd = shlex::split(&cmd).ok_or(Error::JustError("argparse failed".to_string()))?; + info!("cmd: {:?}", cmd); cmd.insert(0, String::from("u_panel")); + let parsed_cmd = Args::from_iter_safe(cmd)?; - Ok( - match process_cmd(client.as_ref().clone(), parsed_cmd).await { - Ok(r) => HttpResponse::Ok().body(r), - Err(e) => HttpResponse::BadRequest().body(e.to_string()), - }, - ) + let result = process_cmd(client.as_ref().clone(), parsed_cmd).await; + let result_string = result.to_string(); + + let response = if result.is_ok() { + HttpResponse::Ok().body(result_string) + } else if result.is_err() { + HttpResponse::BadRequest().body(result_string) + } else { + unreachable!() + }; + + Ok(response) } pub async fn serve(client: ClientHandler) -> anyhow::Result<()> { @@ -89,9 +86,10 @@ pub async fn serve(client: ClientHandler) -> anyhow::Result<()> { .wrap(Logger::default()) .wrap(Cors::permissive()) .app_data(web::Data::new(client.clone())) - .service(main_page) .service(send_cmd) - .service(static_files_adapter) + .service(resources_adapter) + .service(web::resource("/").to(spa_main)) + .service(web::resource("/{_}").to(spa_main)) }) .bind(addr)? .run() diff --git a/bin/u_panel/src/tui/impls.rs b/bin/u_panel/src/tui/impls.rs deleted file mode 100644 index a20adb4..0000000 --- a/bin/u_panel/src/tui/impls.rs +++ /dev/null @@ -1,61 +0,0 @@ -use crate::tui::windows::{ConfirmWnd, MainWnd, WndId}; -use crate::CLIENT; -use u_lib::models::{Agent, AssignedJob, JobMeta}; -use u_lib::UResult; -use uuid::Uuid; - -pub trait Id { - fn id(&self) -> T; -} - -#[macro_export] -macro_rules! impl_id { - ($id_type:tt => $($type:tt),+) => { - $( - impl Id<$id_type> for $type { - fn id(&self) -> $id_type { - self.id - } - })+ - }; -} - -impl_id!(Uuid => Agent, JobMeta, AssignedJob); -impl_id!(WndId => MainWnd, ConfirmWnd); - -#[async_trait] -pub trait CRUD: Id -where - Self: Sized, -{ - async fn read() -> UResult>; - - async fn delete(uid: Uuid) -> UResult { - CLIENT.del(Some(uid)).await - } - //TODO: other crud -} - -#[async_trait] -impl CRUD for Agent { - async fn read() -> UResult> { - CLIENT.get_agents(None).await.map(|r| r.into_builtin_vec()) - } -} - -#[async_trait] -impl CRUD for AssignedJob { - async fn read() -> UResult> { - CLIENT - .get_agent_jobs(None) - .await - .map(|r| r.into_builtin_vec()) - } -} - -#[async_trait] -impl CRUD for JobMeta { - async fn read() -> UResult> { - CLIENT.get_jobs(None).await.map(|r| r.into_builtin_vec()) - } -} diff --git a/bin/u_panel/src/tui/mod.rs b/bin/u_panel/src/tui/mod.rs deleted file mode 100644 index 3a86c84..0000000 --- a/bin/u_panel/src/tui/mod.rs +++ /dev/null @@ -1,187 +0,0 @@ -mod impls; -mod utils; -mod windows; - -use crate::argparse::TUIArgs; -use anyhow::Result as AResult; -use backtrace::Backtrace; -use crossterm::event::{self, DisableMouseCapture, EnableMouseCapture, Event, KeyCode}; -use crossterm::execute; -use crossterm::terminal::{ - disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen, -}; -use once_cell::sync::Lazy; -use std::{ - env, - io::{stdout, Stdout}, - panic::set_hook, - process::exit, - sync::atomic::{AtomicBool, Ordering}, - thread, - time::Duration, -}; -use tui::{backend::CrosstermBackend, Terminal}; -use utils::{AsyncChannel, Channel}; -use windows::{MainWnd, SharedWnd, WndId, WM}; - -pub type Backend = CrosstermBackend; -pub type Frame<'f> = tui::Frame<'f, Backend>; - -const EVENT_GEN_PERIOD: Duration = Duration::from_millis(70); - -static GENERAL_EVENT_CHANNEL: Lazy> = Lazy::new(|| Channel::new()); -static ACTIVE_LOOP: AtomicBool = AtomicBool::new(true); - -enum GEvent { - CreateWnd(SharedWnd), - CloseWnd { wid: WndId, force: bool }, - Exit, - Key(KeyCode), - Tick, -} - -fn get_terminal() -> AResult> { - let backend = CrosstermBackend::new(stdout()); - Ok(Terminal::new(backend)?) -} - -pub async fn init_tui(args: &TUIArgs) -> AResult<()> { - let gui = !args.nogui; - init_logger(); - info!("Initializing u_panel"); - - init_signal_handlers(gui); - init_panic_handler(gui); - term_setup(gui)?; - info!("Starting loop"); - - let result = init_loop(args).await; - info!("Exiting"); - term_teardown(gui)?; - result -} - -async fn init_loop(args: &TUIArgs) -> AResult<()> { - let gui = !args.nogui; - if gui { - WM::lock().await.clear()?; - } - WM::lock().await.push_new::().await; - - thread::spawn(move || { - while is_running() { - if event::poll(EVENT_GEN_PERIOD).unwrap() { - match event::read().unwrap() { - Event::Key(key) => { - GENERAL_EVENT_CHANNEL.send(GEvent::Key(key.code)); - GENERAL_EVENT_CHANNEL.send(GEvent::Tick); - } - _ => (), - } - } else { - GENERAL_EVENT_CHANNEL.send(GEvent::Tick) - } - } - }); - - while is_running() { - match GENERAL_EVENT_CHANNEL.recv() { - GEvent::Tick => { - let mut wh = WM::lock().await; - wh.update().await?; - if gui { - wh.draw().await?; - } - } - GEvent::CloseWnd { wid, force } => { - WM::lock().await.close(wid, force).await; - } - GEvent::Key(key) => { - info!(?key, "pressed"); - if let KeyCode::Char('q') = key { - break_global(); - } else { - WM::lock().await.send_handle_kbd(key).await; - } - } - GEvent::CreateWnd(wnd) => { - WM::lock().await.push_dyn(wnd).await; - } - GEvent::Exit => { - break_global(); - } - } - } - Ok(()) -} - -#[inline] -fn is_running() -> bool { - ACTIVE_LOOP.load(Ordering::Relaxed) -} - -fn break_global() { - ACTIVE_LOOP.store(false, Ordering::Relaxed); -} - -fn init_signal_handlers(gui: bool) { - use signal_hook::{ - consts::{SIGINT, SIGTERM}, - iterator::Signals, - }; - - thread::spawn(move || { - let mut signals = Signals::new(&[SIGINT, SIGTERM]).unwrap(); - for sig in signals.forever() { - match sig { - SIGINT => break_global(), - SIGTERM => { - break_global(); - term_teardown(gui).ok(); - exit(3); - } - _ => (), - } - } - }); -} - -fn init_logger() { - use tracing_appender::rolling::{RollingFileAppender, Rotation}; - use tracing_subscriber::EnvFlter; - - if env::var("RUST_LOG").is_err() { - env::set_var("RUST_LOG", "info") - } - - tracing_subscriber::fmt::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .with_writer(|| RollingFileAppender::new(Rotation::NEVER, ".", "u_panel.log")) - .init(); -} - -fn init_panic_handler(gui: bool) { - set_hook(Box::new(move |panic_info| { - term_teardown(gui).ok(); - eprintln!("CRITICAL PANIK ENCOUNTRD OMFG!11! go see logz lul"); - error!("{}\n{:?}", panic_info, Backtrace::new()); - exit(254); - })); -} - -fn term_setup(gui: bool) -> AResult<()> { - enable_raw_mode()?; - if gui { - execute!(stdout(), EnterAlternateScreen, EnableMouseCapture)?; - } - Ok(()) -} - -fn term_teardown(gui: bool) -> AResult<()> { - disable_raw_mode()?; - if gui { - get_terminal()?.show_cursor()?; - execute!(stdout(), LeaveAlternateScreen, DisableMouseCapture)?; - } - Ok(()) -} diff --git a/bin/u_panel/src/tui/utils.rs b/bin/u_panel/src/tui/utils.rs deleted file mode 100644 index 6b508be..0000000 --- a/bin/u_panel/src/tui/utils.rs +++ /dev/null @@ -1,58 +0,0 @@ -use async_channel::{ - unbounded as unbounded_async, Receiver as AsyncReceiver, RecvError, SendError, - Sender as AsyncSender, -}; -use crossbeam::channel::{unbounded, Receiver, Sender}; - -pub struct Channel { - pub tx: Sender, - pub rx: Receiver, -} - -impl Channel { - pub fn new() -> Self { - let (tx, rx) = unbounded::(); - Self { tx, rx } - } - - pub fn send(&self, msg: T) { - self.tx.send(msg).unwrap() - } - - pub fn recv(&self) -> T { - self.rx.recv().unwrap() - } -} - -impl Default for Channel { - fn default() -> Self { - Channel::new() - } -} - -#[derive(Clone)] -pub struct AsyncChannel { - pub tx: AsyncSender, - pub rx: AsyncReceiver, -} - -impl AsyncChannel { - pub fn new() -> Self { - let (tx, rx) = unbounded_async::(); - Self { tx, rx } - } - - pub async fn send(&self, msg: T) -> Result<(), SendError> { - self.tx.send(msg).await - } - - pub async fn recv(&self) -> Result { - self.rx.recv().await - } -} - -impl Default for AsyncChannel { - fn default() -> Self { - AsyncChannel::new() - } -} diff --git a/bin/u_panel/src/tui/windows/confirm.rs b/bin/u_panel/src/tui/windows/confirm.rs deleted file mode 100644 index 319c10c..0000000 --- a/bin/u_panel/src/tui/windows/confirm.rs +++ /dev/null @@ -1,127 +0,0 @@ -use super::{ReturnVal, Window, WndId}; -use crate::tui::Frame; -use anyhow::Result as AResult; -use crossterm::event::KeyCode; -use tui::layout::{Alignment, Constraint, Direction, Layout, Rect}; -use tui::style::{Color, Modifier, Style}; -use tui::widgets::{Block, Borders, Clear, List, ListItem, ListState, Paragraph}; - -#[derive(Default)] -pub struct ConfirmWnd { - pub id: WndId, - msg: String, - variants: Vec, - state: ListState, -} - -impl ConfirmWnd { - pub fn new(msg: impl Into, variants: &[&str]) -> Self { - let variants = if !variants.is_empty() { - variants - } else { - &["Yes", "No"] - }; - let mut this = Self { - msg: msg.into(), - variants: variants.into_iter().map(|s| s.to_string()).collect(), - ..Default::default() - }; - this.state.select(Some(0)); - this - } - - pub fn on_right(&mut self) { - let selected = self.state.selected().unwrap_or(0); - self.state - .select(Some((selected + 1).rem_euclid(self.variants.len()))); - } - - pub fn on_left(&mut self) { - let selected = self.state.selected().unwrap_or(0); - let vars_len = self.variants.len(); - self.state - .select(Some((selected + vars_len - 1).rem_euclid(vars_len))); - } -} - -#[async_trait] -impl Window for ConfirmWnd { - async fn handle_kbd(&mut self, k: KeyCode) -> AResult<()> { - match k { - KeyCode::Right => self.on_right(), - KeyCode::Left => self.on_left(), - KeyCode::Enter => self.close(false, true).await, - KeyCode::Esc => self.close(false, false).await, - _ => (), - } - Ok(()) - } - - async fn handle_update(&mut self) -> AResult<()> { - Ok(()) - } - - async fn handle_close(&mut self) -> bool { - true - } - - fn draw(&mut self, f: &mut Frame) { - let size = f.size(); - let rect = centered_rect(60, 40, size); - let popup = Block::default().title("Popup").borders(Borders::ALL); - f.render_widget(Clear, rect); - f.render_widget(popup, rect); - - let chunks = Layout::default() - .direction(Direction::Vertical) - .constraints(vec![Constraint::Percentage(40), Constraint::Percentage(30)]) - .split(rect); - let msg = Paragraph::new(self.msg.as_ref()); - f.render_widget(msg, chunks[0]); - - let options = self - .variants - .iter() - .map(AsRef::as_ref) - .map(ListItem::new) - .collect::>(); - let list = - List::new(options).highlight_style(Style::default().bg(Color::Gray).fg(Color::Black)); - f.render_stateful_widget(list, chunks[1], &mut self.state); - } - - fn retval(&self) -> ReturnVal { - let value = self - .variants - .get(self.state.selected().unwrap()) - .unwrap() - .to_owned(); - ReturnVal::String(value) - } -} - -fn centered_rect(percent_x: u16, percent_y: u16, r: Rect) -> Rect { - let popup_layout = Layout::default() - .direction(Direction::Vertical) - .constraints( - [ - Constraint::Percentage((100 - percent_y) / 2), - Constraint::Percentage(percent_y), - Constraint::Percentage((100 - percent_y) / 2), - ] - .as_ref(), - ) - .split(r); - - Layout::default() - .direction(Direction::Horizontal) - .constraints( - [ - Constraint::Percentage((100 - percent_x) / 2), - Constraint::Percentage(percent_x), - Constraint::Percentage((100 - percent_x) / 2), - ] - .as_ref(), - ) - .split(popup_layout[1])[1] -} diff --git a/bin/u_panel/src/tui/windows/main_wnd.rs b/bin/u_panel/src/tui/windows/main_wnd.rs deleted file mode 100644 index d6494ee..0000000 --- a/bin/u_panel/src/tui/windows/main_wnd.rs +++ /dev/null @@ -1,325 +0,0 @@ -use super::{ConfirmWnd, ReturnVal, Window}; -use crate::tui::{impls::CRUD, windows::WndId, Frame}; -use anyhow::Result as AResult; -use crossterm::event::KeyCode; -use std::{fmt::Display, str::FromStr}; -use strum::VariantNames; -use tokio::join; -use tui::widgets::ListState; -use u_lib::models::{Agent, AssignedJob, JobMeta}; -use uuid::Uuid; - -use tui::layout::{Constraint, Direction, Layout}; -use tui::style::{Color, Modifier, Style}; -use tui::text::Spans; -use tui::widgets::{Block, Borders, List, ListItem, Tabs}; - -#[derive(strum::Display, strum::EnumVariantNames, strum::EnumString)] -pub enum UiTabs { - Agents, - Jobs, - Map, -} - -impl UiTabs { - pub fn variants() -> &'static [&'static str] { - Self::VARIANTS - } - - pub fn index(&self) -> usize { - let ss = self.to_string(); - Self::VARIANTS.iter().position(|el| **el == ss).unwrap() - } - - pub fn next(&self) -> Self { - let next_idx = (self.index() + 1).rem_euclid(Self::VARIANTS.len()); - Self::from_str(Self::VARIANTS[next_idx]).unwrap() - } - - pub fn prev(&self) -> Self { - let vlen = Self::VARIANTS.len(); - let next_idx = (self.index() + vlen - 1).rem_euclid(vlen); - Self::from_str(Self::VARIANTS[next_idx]).unwrap() - } -} - -pub struct StatefulList { - pub updated: bool, - pub inner: Vec, - pub state: ListState, -} - -impl StatefulList { - pub async fn update(&mut self) -> AResult<()> { - if !self.updated { - let new_values = T::read().await?; - self.inner = new_values; - self.updated = true; - } - Ok(()) - } - - pub async fn delete(&mut self) -> AResult<()> { - if let Some(s) = self.state.selected() { - let uid = self.inner[s].id(); - T::delete(uid).await?; - } - Ok(()) - } - - pub fn get(&self, id: Uuid) -> Option<&T> { - for item in self.inner.iter() { - if item.id() == id { - return Some(item); - } - } - None - } -} - -impl Default for StatefulList { - fn default() -> Self { - let mut state = ListState::default(); - state.select(Some(0)); - StatefulList { - updated: false, - inner: vec![], - state, - } - } -} - -pub struct MainWnd { - pub id: WndId, - pub active_tab: UiTabs, - pub last_error: Option, - pub agents: StatefulList, - pub jobs: StatefulList, - pub map: StatefulList, -} - -impl Default for MainWnd { - fn default() -> Self { - MainWnd { - active_tab: UiTabs::Agents, - last_error: None, - agents: Default::default(), - jobs: Default::default(), - map: Default::default(), - id: Default::default(), - } - } -} - -impl MainWnd { - pub fn next_tab(&mut self) { - self.active_tab = self.active_tab.next() - } - - pub fn prev_tab(&mut self) { - self.active_tab = self.active_tab.prev() - } - - fn check_err(&mut self, res: AResult<()>) -> bool { - if let Err(e) = res { - self.last_error = Some(e.to_string()); - true - } else { - false - } - } - - pub async fn check_updates(&mut self) { - if !self.agents.updated || !self.jobs.updated || !self.map.updated { - let state = self.tab_list_state(); - if let None = state.selected() { - state.select(Some(0)) - } - - let (a, j, m) = join! { - self.agents.update(), - self.jobs.update(), - self.map.update() - }; - for res in [a, j, m] { - self.check_err(res); - } - } - } - - pub fn tab_data(&self) -> Vec { - match self.active_tab { - UiTabs::Agents => self - .agents - .inner - .iter() - .map(|i| format!("{}: {}-{}", crop(i.id, 6), i.username, i.hostname)) - .collect(), - UiTabs::Jobs => self - .jobs - .inner - .iter() - .map(|i| format!("{}: {}", crop(i.id, 6), i.alias.clone().unwrap_or_default())) - .collect(), - UiTabs::Map => self - .map - .inner - .iter() - .map(|i| { - let job = self.jobs.get(i.job_id).unwrap(); - let job_id = crop(i.job_id, 6); - let job_ident = if let Some(alias) = job.alias.as_ref() { - format!("{} ({})", alias, job_id) - } else { - format!("{}", job_id) - }; - let agent = self.agents.get(i.agent_id).unwrap(); - let agent_id = crop(i.agent_id, 6); - let agent_ident = if let Some(alias) = agent.alias.as_ref() { - format!("{} ({})", alias, agent_id) - } else { - format!("{}-{} ({})", agent.username, agent.hostname, agent_id) - }; - format!("{}: {} for {}", crop(i.id, 6), job_ident, agent_ident) - }) - .collect(), - } - } - - pub fn tab_list_state(&mut self) -> &mut ListState { - match self.active_tab { - UiTabs::Agents => &mut self.agents.state, - UiTabs::Jobs => &mut self.jobs.state, - UiTabs::Map => &mut self.map.state, - } - } - - pub fn update_tab(&mut self) { - match self.active_tab { - UiTabs::Agents => { - self.agents.updated = false; - self.jobs.updated = false; - self.map.updated = false; - } - UiTabs::Jobs => self.jobs.updated = false, - UiTabs::Map => self.map.updated = false, - } - } - - pub fn on_down(&mut self) { - let (list_len, list_state) = match self.active_tab { - UiTabs::Agents => (self.agents.inner.len(), &mut self.agents.state), - UiTabs::Jobs => (self.jobs.inner.len(), &mut self.jobs.state), - UiTabs::Map => (self.map.inner.len(), &mut self.map.state), - }; - if list_len == 0 { - list_state.select(None); - } else { - let selected = list_state.selected().unwrap_or(list_len - 1); - list_state.select(Some((selected + 1).rem_euclid(list_len))); - } - } - - pub fn on_up(&mut self) { - let (list_len, list_state) = match self.active_tab { - UiTabs::Agents => (self.agents.inner.len(), &mut self.agents.state), - UiTabs::Jobs => (self.jobs.inner.len(), &mut self.jobs.state), - UiTabs::Map => (self.map.inner.len(), &mut self.map.state), - }; - if list_len == 0 { - list_state.select(None); - } else { - let selected = list_state.selected().unwrap_or(1); - list_state.select(Some((selected + list_len - 1).rem_euclid(list_len))); - } - } - - pub async fn delete(&mut self) { - let res = match self.active_tab { - UiTabs::Agents => self.agents.delete().await, - UiTabs::Jobs => self.jobs.delete().await, - UiTabs::Map => self.map.delete().await, - }; - if !self.check_err(res) { - self.on_up(); - } - } -} - -#[async_trait] -impl Window for MainWnd { - async fn handle_kbd(&mut self, k: KeyCode) -> AResult<()> { - match k { - KeyCode::Esc => self.close(false, false).await, - KeyCode::Left => self.prev_tab(), - KeyCode::Right => self.next_tab(), - KeyCode::Up => self.on_up(), - KeyCode::Down => self.on_down(), - KeyCode::Delete => { - if let ReturnVal::String(ref s) = - ConfirmWnd::new("Delete?", &["Yes", "No"]).wait_retval() - { - if s == "Yes" { - self.delete().await; - self.update_tab(); - } - } - } - KeyCode::F(5) => self.update_tab(), - _ => (), - }; - Ok(()) - } - - async fn handle_update(&mut self) -> AResult<()> { - self.check_updates().await; - Ok(()) - } - - async fn handle_close(&mut self) -> bool { - true - } - - fn draw(&mut self, f: &mut Frame) { - let size = f.size(); - let chunks = Layout::default() - .direction(Direction::Vertical) - .margin(1) - .constraints([Constraint::Length(3), Constraint::Min(0)].as_ref()) - .split(size); - let titles = UiTabs::variants() - .iter() - .cloned() - .map(Spans::from) - .collect(); - let tabs = Tabs::new(titles) - .block( - Block::default() - .title("The whole that you need to know") - .borders(Borders::ALL), - ) - .style(Style::default().fg(Color::White)) - .highlight_style(Style::default().fg(Color::Yellow)) - .divider("-") - .select(self.active_tab.index()); - f.render_widget(tabs, chunks[0]); - - let tab_data = self - .tab_data() - .into_iter() - .map(ListItem::new) - .collect::>(); - let list = List::new(tab_data) - .block(Block::default().borders(Borders::ALL)) - .highlight_style(Style::default().add_modifier(Modifier::BOLD)); - f.render_stateful_widget(list, chunks[1], self.tab_list_state()); - } - - fn retval(&self) -> ReturnVal { - ReturnVal::None - } -} - -fn crop(data: T, retain: usize) -> String { - data.to_string()[..retain].to_string() -} diff --git a/bin/u_panel/src/tui/windows/mod.rs b/bin/u_panel/src/tui/windows/mod.rs deleted file mode 100644 index ec86b3f..0000000 --- a/bin/u_panel/src/tui/windows/mod.rs +++ /dev/null @@ -1,223 +0,0 @@ -mod confirm; -mod main_wnd; -pub use confirm::ConfirmWnd; -pub use main_wnd::MainWnd; - -use crate::tui::{ - get_terminal, impls::Id, AsyncChannel, Backend, Channel, Frame, GEvent, GENERAL_EVENT_CHANNEL, -}; -use anyhow::Result as AResult; -use crossterm::event::KeyCode; -use once_cell::sync::{Lazy, OnceCell}; -use std::collections::BTreeMap; -use std::sync::{Arc, Mutex as StdMutex, MutexGuard as StdMutexGuard}; -use tokio::sync::{Mutex, MutexGuard}; -use tokio::task::{self, JoinHandle}; -use tui::Terminal; - -static WINDOWS: Lazy>> = - Lazy::new(|| Arc::new(Mutex::new(WM::new(get_terminal().unwrap())))); - -static LAST_WND_ID: OnceCell> = OnceCell::new(); - -static RETVAL: Lazy> = Lazy::new(|| Channel::new()); - -pub type SharedWnd = Arc>; - -pub enum ReturnVal { - String(String), - None, -} - -#[derive(Clone, Debug)] -pub enum WndEvent { - Key(KeyCode), -} - -#[derive(PartialEq, PartialOrd, Eq, Ord, Hash, Copy, Clone, Debug)] -pub struct WndId(u64); - -impl WndId { - fn last_wid() -> StdMutexGuard<'static, WndId> { - LAST_WND_ID - .get_or_init(|| StdMutex::new(WndId(0))) - .lock() - .unwrap() - } -} - -impl Default for WndId { - fn default() -> Self { - let mut wid = Self::last_wid(); - wid.0 += 1; - *wid - } -} - -#[async_trait] -pub trait Window: Id + Send { - async fn handle_event(&mut self, ev: WndEvent) { - match ev { - WndEvent::Key(k) => { - self.handle_kbd(k).await.unwrap(); - } - } - } - - async fn close(&mut self, force: bool, save_retval: bool) { - let rv = if save_retval { - self.retval() - } else { - ReturnVal::None - }; - RETVAL.send(rv); - GENERAL_EVENT_CHANNEL.send(GEvent::CloseWnd { - wid: self.id(), - force, - }); - } - - fn retval(&self) -> ReturnVal; - - fn wait_retval(self) -> ReturnVal - where - Self: Sized + 'static, - { - GENERAL_EVENT_CHANNEL.send(GEvent::CreateWnd(Arc::new(Mutex::new(self)))); - RETVAL.recv() - } - - async fn handle_update(&mut self) -> AResult<()>; - - async fn handle_close(&mut self) -> bool; - - async fn handle_kbd(&mut self, k: KeyCode) -> AResult<()>; - - fn draw(&mut self, f: &mut Frame); -} - -pub struct WndLoop { - chan: AsyncChannel, - wnd_loop: JoinHandle<()>, - window: SharedWnd, -} - -impl WndLoop { - pub async fn with_wnd(window: SharedWnd) -> Self { - let wnd = window.clone(); - let chan = AsyncChannel::::new(); - let ch = chan.clone(); - let wnd_loop = async move { - loop { - match ch.recv().await { - Ok(ev) => wnd.lock().await.handle_event(ev).await, - Err(_) => break, - } - } - }; - WndLoop { - chan, - window, - wnd_loop: task::spawn(wnd_loop), - } - } - - pub async fn send(&self, ev: WndEvent) { - let wnd_id = self.window.lock().await.id(); - let event = ev.clone(); - debug!(?event, ?wnd_id, "sending"); - self.chan.send(ev).await.expect("send failed"); - } -} - -pub struct WM { - queue: BTreeMap, - term: Terminal, -} - -impl WM { - fn get_last_wnd(&self) -> &WndLoop { - let last_id = self.queue.keys().rev().next().expect("No windows found"); - self.queue.get(last_id).unwrap() - } - - pub async fn lock<'l>() -> MutexGuard<'l, Self> { - WINDOWS.lock().await - } - - fn new(term: Terminal) -> Self { - Self { - term, - queue: BTreeMap::new(), - } - } - - pub async fn push(&mut self, window: W) -> SharedWnd { - self.push_dyn(Arc::new(Mutex::new(window))).await - } - - pub async fn push_new(&mut self) -> SharedWnd { - self.push(W::default()).await - } - - pub async fn push_dyn(&mut self, window: SharedWnd) -> SharedWnd { - let wid = window.lock().await.id(); - self.queue - .insert(wid, WndLoop::with_wnd(window.clone()).await); - window - } - - pub fn clear(&mut self) -> AResult<()> { - self.term.clear()?; - Ok(()) - } - - pub async fn draw(&mut self) -> AResult<()> { - for wid in self.queue.keys().collect::>() { - let mut wnd_locked = match self.queue.get(&wid) { - Some(w) => match w.window.try_lock() { - Ok(w) => w, - Err(_) => { - warn!("Can't lock window {:?}", wid); - continue; - } - }, - None => { - warn!("Can't redraw window {:?}, not found", wid); - continue; - } - }; - self.term.draw(move |f| wnd_locked.draw(f))?; - } - Ok(()) - } - - pub async fn close(&mut self, wid: WndId, force: bool) { - let wnd = match self.queue.get(&wid) { - Some(w) => w.clone(), - None => { - warn!("Can't close window {:?}, not found", wid); - return; - } - }; - if wnd.window.lock().await.handle_close().await || force { - let WndLoop { wnd_loop, .. } = self.queue.remove(&wid).unwrap(); - wnd_loop.abort(); - } - - if self.queue.is_empty() { - GENERAL_EVENT_CHANNEL.send(GEvent::Exit); - } - } - - pub async fn send_handle_kbd(&self, k: KeyCode) { - let current_wnd = self.get_last_wnd(); - current_wnd.send(WndEvent::Key(k)).await; - } - - pub async fn update(&mut self) -> AResult<()> { - let current_wnd = self.get_last_wnd(); - current_wnd.window.lock().await.handle_update().await?; - Ok(()) - } -} diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index e18824f..5bee420 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -42,14 +42,14 @@ impl UDB { .unwrap() } - pub fn insert_jobs(&self, job_metas: &[JobMeta]) -> Result<()> { + pub fn insert_jobs(&self, job_metas: &[JobMeta]) -> Result> { use schema::jobs; diesel::insert_into(jobs::table) .values(job_metas) - .execute(&self.conn) - .map_err(with_err_ctx("Can't insert jobs"))?; - Ok(()) + .get_results(&self.conn) + .map(|rows| rows.iter().map(|job: &JobMeta| job.id).collect()) + .map_err(with_err_ctx("Can't insert jobs")) } pub fn get_jobs(&self, ouid: Option) -> Result> { diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index eb890e4..c0907c9 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -35,7 +35,8 @@ impl Endpoints { let db = UDB::lock_db(); let mut agents = db.get_agents(Some(uid))?; if agents.is_empty() { - db.insert_agent(&Agent::with_id(uid))?; + let new_agent = Agent::with_id(uid); + db.insert_agent(&new_agent)?; let job = db .find_job_by_alias("agent_hello")? .expect("agent_hello job not found"); @@ -53,7 +54,7 @@ impl Endpoints { Ok(result) } - pub async fn upload_jobs(msg: BaseMessage<'static, Vec>) -> EndpResult<()> { + pub async fn upload_jobs(msg: BaseMessage<'static, Vec>) -> EndpResult> { UDB::lock_db() .insert_jobs(&msg.into_inner()) .map_err(From::from) diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 70a5b9f..4a6d2db 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -70,7 +70,7 @@ pub fn init_endpoints( let upload_jobs = path("upload_jobs") .and(get_content::>()) .and_then(Endpoints::upload_jobs) - .map(ok); + .map(into_message); let get_jobs = path("get_jobs") .and( @@ -111,17 +111,17 @@ pub fn init_endpoints( .and_then(Endpoints::report) .map(ok); - let update_agent = path("update_item") + let update_agent = path("update_agent") .and(get_content::()) .and_then(Endpoints::update_agent) .map(ok); - let update_job = path("update_item") + let update_job = path("update_job") .and(get_content::()) .and_then(Endpoints::update_job) .map(ok); - let update_assigned_job = path("update_item") + let update_assigned_job = path("update_result") .and(get_content::()) .and_then(Endpoints::update_assigned_job) .map(ok); @@ -142,9 +142,7 @@ pub fn init_endpoints( .or(del) .or(set_jobs) .or(get_agent_jobs) - .or(update_agent) - .or(update_job) - .or(update_assigned_job) + .or(update_agent.or(update_job).or(update_assigned_job)) .or(download) .or(ping)) .and(auth_header); @@ -163,7 +161,7 @@ pub fn preload_jobs() -> Result<(), ServerError> { .with_alias(job_alias) .build() .unwrap(); - UDB::lock_db().insert_jobs(&[agent_hello])? + UDB::lock_db().insert_jobs(&[agent_hello])?; } Ok(()) } diff --git a/images/musl-libs.Dockerfile b/images/musl-libs.Dockerfile index 2441b80..c2e0b01 100644 --- a/images/musl-libs.Dockerfile +++ b/images/musl-libs.Dockerfile @@ -38,7 +38,7 @@ RUN apt-get update && apt-get install -y \ # This helps continuing manually if anything breaks. ENV SSL_VER="1.0.2u" \ CURL_VER="7.77.0" \ - ZLIB_VER="1.2.11" \ + ZLIB_VER="1.2.13" \ PQ_VER="11.12" \ SQLITE_VER="3350500" \ CC=musl-gcc \ diff --git a/integration/integration_tests.py b/integration/integration_tests.py index e2b2028..512ea98 100644 --- a/integration/integration_tests.py +++ b/integration/integration_tests.py @@ -17,7 +17,7 @@ def fail(msg): def usage_exit(): usage = f"""Usage: - python {__file__.split('/')[-1]} [--rebuild] [--preserve] [--no-run]""" + python {__file__.split('/')[-1]} [--rebuild] [--preserve] [--no-run] [--down]""" fail(usage) @@ -32,13 +32,14 @@ def create_integration_workspace(): def run_tests(): - allowed_args = set(["--rebuild", "--preserve", "--no-run", "--release"]) + allowed_args = set(["--rebuild", "--preserve", "--no-run", "--release", "--down"]) args = sys.argv[1:] if not set(args).issubset(allowed_args): usage_exit() force_rebuild = '--rebuild' in args preserve_containers = '--preserve' in args only_setup_cluster = '--no-run' in args + down_cluster = "--down" in args def _cleanup(): if not preserve_containers and not only_setup_cluster: @@ -49,6 +50,10 @@ def run_tests(): warn(f'Received signal: {s}, gracefully stopping...') _cleanup() + if down_cluster: + _cleanup() + return + for s in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): signal.signal(s, abort_handler) rebuild_images_if_needed(force_rebuild) diff --git a/integration/tests/helpers/panel.rs b/integration/tests/helpers/panel.rs index 4caecb2..f8fa576 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration/tests/helpers/panel.rs @@ -18,7 +18,7 @@ impl Panel { pub fn output_argv(argv: &[&str]) -> PanelResult { let result = Self::run(argv); - let output = ProcOutput::from_output(&result).to_appropriate(); + let output = ProcOutput::from_output(&result).into_vec(); from_slice(&output) .map_err(|e| { eprintln!( diff --git a/integration/tests/integration/behaviour.rs b/integration/tests/integration/behaviour.rs index 6472505..932cb03 100644 --- a/integration/tests/integration/behaviour.rs +++ b/integration/tests/integration/behaviour.rs @@ -31,7 +31,7 @@ async fn test_setup_tasks() -> TestResult { }; let job_alias = "passwd_contents"; let job = json!( - {"alias": job_alias, "payload": b"cat /etc/passwd" } + {"alias": job_alias, "payload": b"cat /etc/passwd", "argv": "/bin/bash {}" } ); let cmd = format!("jobs create '{}'", to_string(&job).unwrap()); Panel::check_status(cmd); diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index b64913c..28463d4 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -116,13 +116,23 @@ impl ClientHandler { .await } - /// update something - pub async fn update_item(&self, item: impl AsMsg + Debug) -> Result<()> { - self.req_with_payload("update_item", item).await + /// update agent + pub async fn update_agent(&self, agent: models::Agent) -> Result<()> { + self.req_with_payload("update_agent", agent).await + } + + /// update job + pub async fn update_job(&self, job: models::JobMeta) -> Result<()> { + self.req_with_payload("update_job", job).await + } + + /// update result + pub async fn update_result(&self, result: models::AssignedJob) -> Result<()> { + self.req_with_payload("update_result", result).await } /// create and upload job - pub async fn upload_jobs(&self, payload: impl OneOrVec) -> Result<()> { + pub async fn upload_jobs(&self, payload: impl OneOrVec) -> Result> { self.req_with_payload("upload_jobs", payload.into_vec()) .await } diff --git a/lib/u_lib/src/datatypes.rs b/lib/u_lib/src/datatypes.rs index f7d09a6..25b954a 100644 --- a/lib/u_lib/src/datatypes.rs +++ b/lib/u_lib/src/datatypes.rs @@ -8,3 +8,19 @@ pub enum PanelResult { Ok(M), Err(UError), } + +impl PanelResult { + pub fn is_ok(&self) -> bool { + matches!(self, PanelResult::Ok(_)) + } + + pub fn is_err(&self) -> bool { + matches!(self, PanelResult::Err(_)) + } +} + +impl ToString for PanelResult { + fn to_string(&self) -> String { + serde_json::to_string(self).unwrap() + } +} diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index 61caf8d..ba28da5 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -3,18 +3,13 @@ use diesel::{AsChangeset, Identifiable, Insertable, Queryable}; #[cfg(feature = "server")] use diesel_derive_enum::DbEnum; use serde::{Deserialize, Serialize}; -use std::{fmt, time::SystemTime}; +use std::time::SystemTime; use strum::Display; #[cfg(feature = "server")] use crate::models::schema::*; -use crate::{ - config::get_self_uid, - executor::ExecResult, - runner::NamedJobRunner, - utils::{systime_to_string, Platform}, -}; +use crate::{config::get_self_uid, executor::ExecResult, runner::NamedJobRunner, utils::Platform}; use uuid::Uuid; @@ -55,22 +50,6 @@ pub struct Agent { pub username: String, } -impl fmt::Display for Agent { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "Agent: {}", self.id)?; - if let Some(ref alias) = self.alias { - write!(f, " ({})", alias)? - } - writeln!(f, "\nUsername: {}", self.username)?; - writeln!(f, "Hostname: {}", self.hostname)?; - writeln!(f, "Is root: {}", self.is_root)?; - writeln!(f, "Root allowed: {}", self.is_root_allowed)?; - writeln!(f, "Last active: {}", systime_to_string(&self.last_active))?; - writeln!(f, "Platform: {}", self.platform)?; - writeln!(f, "State: {}", self.state) - } -} - #[cfg(not(target_arch = "wasm32"))] impl Agent { pub fn with_id(uid: Uuid) -> Self { @@ -87,7 +66,7 @@ impl Agent { #[cfg(unix)] pub async fn gather() -> Self { let mut builder = NamedJobRunner::from_shell(vec![ - ("hostname", "hostnamectl hostname"), + ("hostname", "uname -a"), ("host_info", "hostnamectl --json=pretty"), ("is_root", "id -u"), ("username", "id -un"), @@ -96,7 +75,7 @@ impl Agent { .wait() .await; let decoder = - |job_result: ExecResult| job_result.unwrap().to_string_result().trim().to_string(); + |job_result: ExecResult| job_result.unwrap().to_str_result().trim().to_string(); Self { hostname: decoder(builder.pop("hostname")), @@ -133,14 +112,3 @@ impl Default for Agent { } } } - -// #[cfg(test)] -// mod tests { -// use super::*; - -// #[tokio::test] -// async fn test_gather() { -// let cli_info = Agent::gather().await; -// assert_eq!(cli_info.alias, None) -// } -// } diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index 78850a4..112671d 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -1,12 +1,12 @@ use super::{JobMeta, JobState, JobType}; +#[cfg(not(target_arch = "wasm32"))] +use crate::config::get_self_uid; #[cfg(feature = "server")] use crate::models::schema::*; -#[cfg(not(target_arch = "wasm32"))] -use crate::{config::get_self_uid, utils::ProcOutput}; #[cfg(feature = "server")] use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; -use std::time::SystemTime; +use std::{borrow::Cow, time::SystemTime}; use uuid::Uuid; #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] @@ -87,26 +87,16 @@ impl Default for AssignedJob { } } -#[cfg(not(target_arch = "wasm32"))] impl AssignedJob { - pub fn as_job_output(&self) -> Option { - self.result - .as_ref() - .and_then(|r| ProcOutput::from_combined(r)) - } - - pub fn to_raw_result(&self) -> Vec { + pub fn to_raw_result(&self) -> &[u8] { match self.result.as_ref() { - Some(r) => match ProcOutput::from_combined(r) { - Some(o) => o.to_appropriate(), - None => r.clone(), - }, - None => b"No data".to_vec(), + Some(r) => r, + None => b"No data yet", } } - pub fn to_string_result(&self) -> String { - String::from_utf8_lossy(&self.to_raw_result()).into_owned() + pub fn to_str_result(&self) -> Cow<'_, str> { + String::from_utf8_lossy(self.to_raw_result()) } pub fn set_result(&mut self, result: &S) { diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 8150313..c8980b7 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -51,8 +51,8 @@ impl JobMeta { JobMetaBuilder::default() } - pub fn into_builder(self) -> JobMetaBuilder { - JobMetaBuilder { inner: self } + pub fn validated(self) -> UResult { + JobMetaBuilder { inner: self }.build() } pub fn from_shell(cmd: impl Into) -> UResult { @@ -113,7 +113,7 @@ impl JobMetaBuilder { JobType::Shell => { if inner.argv.is_empty() { // TODO: fix detecting - inner.argv = String::from("/bin/bash -c {}") + inner.argv = String::from("echo 'hello, world!'") } let argv_parts = shlex::split(&inner.argv).ok_or(UError::JobArgsError("Shlex failed".into()))?; @@ -127,7 +127,7 @@ impl JobMetaBuilder { inner.payload = Some(data) } match inner.payload.as_ref() { - Some(_) => { + Some(p) if p.len() > 0 => { if !inner.argv.contains("{}") { return Err(UError::JobArgsError( "Argv contains no executable placeholder".into(), @@ -144,6 +144,7 @@ impl JobMetaBuilder { .into()); } } + _ => (), }; if !Platform::new(&inner.platform).check() { return Err(UError::JobArgsError(format!( diff --git a/lib/u_lib/src/models/schema.rs b/lib/u_lib/src/models/schema.rs index 1b47bc7..11da1d0 100644 --- a/lib/u_lib/src/models/schema.rs +++ b/lib/u_lib/src/models/schema.rs @@ -1,4 +1,6 @@ -table! { +// @generated automatically by Diesel CLI. + +diesel::table! { use crate::schema_exports::*; agents (id) { @@ -19,7 +21,7 @@ table! { } } -table! { +diesel::table! { use crate::schema_exports::*; certificates (id) { @@ -29,7 +31,7 @@ table! { } } -table! { +diesel::table! { use crate::schema_exports::*; jobs (id) { @@ -44,7 +46,7 @@ table! { } } -table! { +diesel::table! { use crate::schema_exports::*; results (id) { @@ -61,8 +63,8 @@ table! { } } -joinable!(certificates -> agents (agent_id)); -joinable!(results -> agents (agent_id)); -joinable!(results -> jobs (job_id)); +diesel::joinable!(certificates -> agents (agent_id)); +diesel::joinable!(results -> agents (agent_id)); +diesel::joinable!(results -> jobs (job_id)); -allow_tables_to_appear_in_same_query!(agents, certificates, jobs, results,); +diesel::allow_tables_to_appear_in_same_query!(agents, certificates, jobs, results,); diff --git a/lib/u_lib/src/runner.rs b/lib/u_lib/src/runner.rs index 9d5576b..2025abd 100644 --- a/lib/u_lib/src/runner.rs +++ b/lib/u_lib/src/runner.rs @@ -111,13 +111,13 @@ pub async fn run_assigned_job(mut job: AssignedJob) -> ExecResult { let cmd_result = Command::new(cmd).args(args).output().await; let (data, retcode) = match cmd_result { Ok(output) => ( - ProcOutput::from_output(&output).into_combined(), + ProcOutput::from_output(&output).into_vec(), output.status.code(), ), Err(e) => ( ProcOutput::new() .stderr(e.to_string().into_bytes()) - .into_combined(), + .into_vec(), None, ), }; @@ -260,7 +260,7 @@ mod tests { .wait_one() .await .unwrap(); - let result = result.to_string_result(); + let result = result.to_str_result(); assert_eq!(result.trim(), expected_result); Ok(()) } @@ -277,7 +277,7 @@ mod tests { .await .unwrap(); assert_eq!(ls.retcode.unwrap(), 0); - let folders = ls.to_string_result(); + let folders = ls.to_str_result(); let subfolders_jobs: Vec = folders .lines() .map(|f| JobMeta::from_shell(format!("ls {}", f)).unwrap()) @@ -319,7 +319,7 @@ mod tests { .wait_one() .await .unwrap(); - let output = job_result.to_string_result(); + let output = job_result.to_str_result(); assert!(output.contains("No such file")); assert!(job_result.retcode.is_none()); Ok(()) diff --git a/lib/u_lib/src/utils/proc_output.rs b/lib/u_lib/src/utils/proc_output.rs index d060e89..ded026e 100644 --- a/lib/u_lib/src/utils/proc_output.rs +++ b/lib/u_lib/src/utils/proc_output.rs @@ -7,26 +7,12 @@ pub struct ProcOutput { } impl ProcOutput { - const STREAM_BORDER: &'static str = "***"; - const STDOUT: &'static str = "STDOUT"; - const STDERR: &'static str = "STDERR"; - - #[inline] - fn create_delim(header: &'static str) -> Vec { - format!( - "<{border}{head}{border}>", - border = Self::STREAM_BORDER, - head = header - ) - .into_bytes() - } + const STDERR_DELIMETER: &[u8] = b"\n[STDERR]\n"; pub fn from_output(output: &Output) -> Self { - let mut this = Self::new().stdout(output.stdout.to_vec()); - if !output.status.success() && output.stderr.len() > 0 { - this.stderr = output.stderr.to_vec(); - } - this + Self::new() + .stdout(output.stdout.to_vec()) + .stderr(output.stderr.to_vec()) } pub fn new() -> Self { @@ -46,114 +32,73 @@ impl ProcOutput { self } - /// Make bytestring like '<***STDOUT***>...<***STDERR***>...' - pub fn into_combined(self) -> Vec { + pub fn into_vec(self) -> Vec { let mut result: Vec = vec![]; if !self.stdout.is_empty() { - result.extend(Self::create_delim(Self::STDOUT)); result.extend(self.stdout); } if !self.stderr.is_empty() { - result.extend(Self::create_delim(Self::STDERR)); + result.extend(Self::STDERR_DELIMETER); result.extend(self.stderr); } result } - pub fn from_combined(raw: &[u8]) -> Option { - enum ParseFirst { - Stdout, - Stderr, - } - fn split_by_subslice<'s>(slice: &'s [u8], subslice: &[u8]) -> Option<(&'s [u8], &'s [u8])> { - slice - .windows(subslice.len()) - .position(|w| w == subslice) - .map(|split_pos| { - let splitted = slice.split_at(split_pos); - (&splitted.0[..split_pos], &splitted.1[subslice.len()..]) - }) - } - let splitter = |p: ParseFirst| { - let (first_hdr, second_hdr) = match p { - ParseFirst::Stdout => (Self::STDOUT, Self::STDERR), - ParseFirst::Stderr => (Self::STDERR, Self::STDOUT), - }; - let first_hdr = Self::create_delim(first_hdr); - let second_hdr = Self::create_delim(second_hdr); - split_by_subslice(raw, &first_hdr).map(|(_, p2)| { - match split_by_subslice(p2, &second_hdr) { - Some((p2_1, p2_2)) => Self::new().stdout(p2_1.to_vec()).stderr(p2_2.to_vec()), - None => Self::new().stdout(p2.to_vec()), + pub fn from_raw_proc_output(raw: &[u8]) -> Option { + let stderr_delim_len = Self::STDERR_DELIMETER.len(); + raw.windows(stderr_delim_len) + .position(|w| w == Self::STDERR_DELIMETER) + .map(|split_pos| { + let (stdout, stderr) = raw.split_at(split_pos); + let result = Self::new().stdout(stdout.to_vec()); + if stderr.len() <= stderr_delim_len { + result.stderr(stderr[stderr_delim_len..].to_vec()) + } else { + result } }) - }; - splitter(ParseFirst::Stdout).or_else(|| splitter(ParseFirst::Stderr)) - } - - /// Chooses between stdout and stderr or both wisely - pub fn to_appropriate(&self) -> Vec { - let mut result: Vec = vec![]; - let mut altered = false; - if !self.stdout.is_empty() { - result.extend(&self.stdout); - altered = true; - } - if !self.stderr.is_empty() { - if altered { - result.push(b'\n'); - } - result.extend(&self.stderr); - altered = true; - } - if !altered { - result.extend(b""); - } - result } } #[cfg(test)] mod tests { use crate::utils::{bytes_to_string, ProcOutput}; + use std::str; - const STDOUT: &str = "<***STDOUT***>"; - const STDERR: &str = "<***STDERR***>"; + const STDERR_DELIMETER: &'static str = + unsafe { str::from_utf8_unchecked(ProcOutput::STDERR_DELIMETER) }; #[rstest] #[case::stdout_stderr( "lol", "kek", - &format!("{}lol{}kek", STDOUT, STDERR) + &format!("lol{}kek", STDERR_DELIMETER) )] #[case::stderr( "", "kek", - &format!("{}kek", STDERR) + &format!("{}kek", STDERR_DELIMETER) )] fn test_to_combined(#[case] stdout: &str, #[case] stderr: &str, #[case] result: &str) { let output = ProcOutput::new() .stdout(stdout.as_bytes().to_vec()) .stderr(stderr.as_bytes().to_vec()); - assert_eq!(&bytes_to_string(&output.into_combined()), result) + assert_eq!(&bytes_to_string(&output.into_vec()), result) } #[rstest] #[case::stdout_stderr( - &format!("{}lal{}kik", STDOUT, STDERR), - "lal\nkik" + &format!("lal{}kik", STDERR_DELIMETER), )] #[case::stdout( - &format!("{}qeq", STDOUT), - "qeq" + &format!("qeq"), )] #[case::stderr( - &format!("{}vev", STDERR), - "vev" + &format!("{}vev", STDERR_DELIMETER), )] - fn test_from_combined(#[case] src: &str, #[case] result: &str) { - let output = ProcOutput::from_combined(src.as_bytes()).unwrap(); - assert_eq!(bytes_to_string(&output.to_appropriate()).trim(), result); + fn test_from_combined(#[case] src_result: &str) { + let output = ProcOutput::from_raw_proc_output(src_result.as_bytes()).unwrap(); + assert_eq!(bytes_to_string(&output.into_vec()).trim(), src_result); } } diff --git a/scripts/build_musl_libs.sh b/scripts/build_musl_libs.sh index 8c688a5..2ffe66b 100755 --- a/scripts/build_musl_libs.sh +++ b/scripts/build_musl_libs.sh @@ -5,8 +5,8 @@ ARGS=$@ STATIC_LIBS=./static DOCKER_EXCHG=/musl-share IMAGE=unki/musllibs -if [[ ! -d ./static ]]; then - mkdir $STATIC_LIBS +if [[ ! $(find ./static ! -empty -type d) ]]; then + mkdir -p $STATIC_LIBS cd $ROOTDIR/images && docker build -t $IMAGE . -f musl-libs.Dockerfile docker run \ -v $ROOTDIR/$STATIC_LIBS:$DOCKER_EXCHG \
IDJob - {{row.job_id}} + {{row.job_id}} + + | + +