diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..433fea9 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,6 @@ +[build] +rustflags = [ + "-L", "/home/ortem/src/rust/unki/static/lib", + "--remap-path-prefix=/home/ortem/src/rust/unki=src", + "--remap-path-prefix=/home/ortem/.cargo=cargo" +] diff --git a/.env b/.env index 60ff916..70e3dfb 100644 --- a/.env +++ b/.env @@ -1,4 +1,5 @@ DB_HOST=u_db DB_NAME=u_db DB_USER=postgres -RUST_BACKTRACE=1 \ No newline at end of file +RUST_BACKTRACE=1 +U_SERVER=u_server \ No newline at end of file diff --git a/.gitignore b/.gitignore index 0f6b5f5..fd55330 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,15 @@ target/ -**/*.rs.bk .idea/ data/ +certs/ +static/ +.vscode/ +release/ +**/node_modules/ + +**/*.rs.bk **/*.pyc -certs/* *.log echoer -.env.private \ No newline at end of file +.env.private +*.lock \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index ede90c2..bf4d243 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,13 +5,25 @@ members = [ "bin/u_run", "bin/u_server", "lib/u_lib", - "lib/u_api_proc_macro", "integration" ] +[workspace.dependencies] +anyhow = "=1.0.63" +reqwest = { version = "0.11", features = ["json"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +thiserror = "=1.0.31" +tokio = { version = "1.11", features = ["macros"] } +tracing = "0.1.35" +tracing-appender = "0.2.0" +tracing-subscriber = { version = "0.3.0", features = ["env-filter"]} +uuid = "0.6.5" + [profile.release] panic = "abort" +strip = "symbols" [profile.dev] debug = true # Добавляет флаг `-g` для компилятора; -opt-level = 0 \ No newline at end of file +opt-level = 0 diff --git a/Makefile b/Makefile deleted file mode 100644 index 969e896..0000000 --- a/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -.PHONY: _pre_build debug release run clean unit-tests integration-tests test - -CARGO=./scripts/cargo_musl.sh - -clean: - ${CARGO} clean - -_pre_build: - docker build -t unki/musllibs ./muslrust - -debug: _pre_build - ${CARGO} build - -release: _pre_build - ${CARGO} build --release - -run: build - ${CARGO} run - -unit-tests: - ${CARGO} test --lib - -integration-tests: - cd ./integration && ./integration_tests.sh - -test: unit-tests integration-tests \ No newline at end of file diff --git a/Makefile.toml b/Makefile.toml new file mode 100644 index 0000000..18100b2 --- /dev/null +++ b/Makefile.toml @@ -0,0 +1,99 @@ +# i need to preserve --release in args, not to pass cargo make -p release +# due to cargo failing to parse "" argument +env_scripts = [''' +#!@duckscript +args = array ${1} ${2} ${3} ${4} ${5} ${6} ${7} +set_env PROFILE_OVERRIDE debug + +for arg in ${args} + e = eq ${arg} "--release" + if ${e} + set_env PROFILE_OVERRIDE release + end +end + +profile = get_env PROFILE_OVERRIDE +echo PROFILE_OVERRIDE=${profile} +'''] + +[config] +default_to_workspace = false + +[env] +TARGET = "x86_64-unknown-linux-musl" +CARGO = "cargo" +ROOTDIR = "${CARGO_MAKE_WORKING_DIRECTORY}" +STATIC_PREFIX = "${ROOTDIR}/static" +PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true" +PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = "${STATIC_PREFIX}/bin/pg_config" +OPENSSL_STATIC = "true" +OPENSSL_DIR = "${STATIC_PREFIX}" + + +[tasks.build_static_libs] +script = "./scripts/build_musl_libs.sh" + +[tasks.build_frontend] +script = ''' +cd ./bin/u_panel/src/server/fe +ng build +''' + +[tasks.clean] +command = "${CARGO}" +args = ["clean"] + +[tasks.cargo_build] +dependencies = ["build_static_libs", "build_frontend"] +command = "${CARGO}" +args = ["build", "--target", "${TARGET}", "${@}"] + +[tasks.cargo_update] +command = "${CARGO}" +args = ["update"] + +[tasks.release_tasks] +condition = { env = { PROFILE_OVERRIDE = "release"} } +script = ''' +BINS=$(ls ./target/${TARGET}/${PROFILE_OVERRIDE}/u_* -1 | grep -v ".d") +echo "Stripping..." +strip $BINS +echo "Packing..." +upx -9 $BINS +''' + +[tasks.build] +dependencies = ["cargo_update", "cargo_build", "release_tasks"] +clear = true + +[tasks.run] +disabled = true + +[tasks.unit] +command = "${CARGO}" +args = ["test", "--target", "${TARGET}", "--lib", "--", "${@}"] + +[tasks.integration] +dependencies = ["cargo_update"] +script = ''' +[[ ! -d "./target/${TARGET}/${PROFILE_OVERRIDE}" ]] && echo 'No target folder. Build project first' && exit 1 +cd ./integration +bash integration_tests.sh ${@} +''' + +[tasks.gen_schema] +script = ''' +docker run --rm \ + --env-file=$PWD/.env \ + --env-file=$PWD/.env.private \ + -v $PWD:/unki \ + -w /unki \ + unki/u_db \ + /unki/images/integration-tests/u_db_entrypoint.sh || true +''' + +[tasks.test] +dependencies = ["unit", "integration"] + +[tasks.deploy] +script = './scripts/deploy.sh' \ No newline at end of file diff --git a/bin/u_agent/Cargo.toml b/bin/u_agent/Cargo.toml index 3f7a884..77259f2 100644 --- a/bin/u_agent/Cargo.toml +++ b/bin/u_agent/Cargo.toml @@ -2,19 +2,15 @@ name = "u_agent" version = "0.1.0" authors = ["plazmoid "] -edition = "2018" +edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process", "time"] } -sysinfo = "0.10.5" log = "^0.4" -env_logger = "0.8.3" -uuid = "0.6.5" -reqwest = { version = "0.11", features = ["json"] } -openssl = "*" -u_lib = { version = "*", path = "../../lib/u_lib" } +reqwest = { workspace = true } +sysinfo = "0.10.5" +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "time"] } +uuid = { workspace = true } +u_lib = { path = "../../lib/u_lib" } -[build-dependencies] -openssl = "*" \ No newline at end of file diff --git a/bin/u_agent/build.rs b/bin/u_agent/build.rs index a9a9abd..0588b1c 100644 --- a/bin/u_agent/build.rs +++ b/bin/u_agent/build.rs @@ -3,6 +3,6 @@ use std::path::PathBuf; fn main() { let server_cert = PathBuf::from("../../certs/ca.crt"); if !server_cert.exists() { - panic!("CA certificate doesn't exist. Create it first with certs/gen_certs.sh"); + panic!("CA certificate doesn't exist. Create it first with scripts/gen_certs.sh"); } } diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 5db6b54..e9bc3e9 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -1,88 +1,120 @@ // TODO: // поддержка питона // резолв адреса управляющего сервера через DoT -// кроссплатформенность (реализовать интерфейс для винды и никсов) -// проверка обнов -// самоуничтожение #[macro_use] extern crate log; -extern crate env_logger; -use std::env; +//use daemonize::Daemonize; +use std::sync::Arc; use tokio::time::{sleep, Duration}; use u_lib::{ - api::ClientHandler, - builder::JobBuilder, - cache::JobCache, - executor::pop_completed, - models::{AssignedJob, ExecResult}, - UID, - //daemonize + api::ClientHandler, cache::JobCache, config::get_self_uid, errors::ErrChan, + executor::pop_completed, logging::init_logger, messaging::Reportable, models::AssignedJobById, + runner::JobRunner, utils::load_env_default, }; -#[macro_export] -macro_rules! retry_until_ok { - ( $body:expr ) => { - loop { - match $body { - Ok(r) => break r, - Err(e) => error!("{:?}", e), - }; - sleep(Duration::from_secs(5)).await; - } - }; -} +const ITERATION_LATENCY: u64 = 5; -pub async fn process_request(job_requests: Vec, client: &ClientHandler) { - if job_requests.len() > 0 { - for jr in &job_requests { - if !JobCache::contains(&jr.job_id) { +pub async fn process_request(jobs: Vec, client: &ClientHandler) { + if !jobs.is_empty() { + for jr in &jobs { + if !JobCache::contains(jr.job_id) { info!("Fetching job: {}", &jr.job_id); - let fetched_job = retry_until_ok!(client.get_jobs(Some(jr.job_id)).await) - .pop() - .unwrap(); + let fetched_job = loop { + match client.get_jobs(Some(jr.job_id)).await { + Ok(mut result) => break result.pop().unwrap(), + Err(err) => { + debug!("{:?} \nretrying...", err); + sleep(Duration::from_secs(ITERATION_LATENCY)).await; + } + } + }; JobCache::insert(fetched_job); } } info!( "Scheduling jobs: {}", - job_requests - .iter() + jobs.iter() .map(|j| j.job_id.to_string()) .collect::>() .join(", ") ); - let mut builder = JobBuilder::from_request(job_requests); - let errors = builder.pop_errors(); - if errors.len() > 0 { - error!( - "Some errors encountered: \n{}", - errors - .iter() - .map(|j| j.to_string()) - .collect::>() - .join("\n") - ); + let mut runner = JobRunner::from_jobs(jobs); + let errors = runner.pop_errors(); + if !errors.is_empty() { + for e in errors { + ErrChan::send(e, "ebld").await; + } + } + runner.unwrap_one().spawn().await; + } +} + +async fn error_reporting(client: Arc) -> ! { + loop { + match ErrChan::recv().await { + Some(err) => { + 'retry: for _ in 0..3 { + match client.report(Reportable::Error(err.clone())).await { + Ok(_) => break 'retry, + Err(e) => { + debug!("Reporting error: {:?}", e); + sleep(Duration::from_secs(10)).await; + } + } + } + } + None => sleep(Duration::from_secs(3)).await, } - builder.unwrap_one().spawn().await; } } -pub async fn run_forever() { - //daemonize(); - env_logger::init(); - let arg_ip = env::args().nth(1); - let instance = ClientHandler::new(arg_ip.as_deref()); - info!("Connecting to the server"); +async fn agent_loop(client: Arc) -> ! { loop { - let job_requests: Vec = - retry_until_ok!(instance.get_personal_jobs(Some(*UID)).await).into_builtin_vec(); - process_request(job_requests, &instance).await; - let result: Vec = pop_completed().await.into_iter().collect(); - if result.len() > 0 { - retry_until_ok!(instance.report(&result).await); + match client.get_personal_jobs(get_self_uid()).await { + Ok(jobs) => { + process_request(jobs, &client).await; + } + Err(err) => ErrChan::send(err, "processing").await, + } + let result: Vec = pop_completed() + .await + .into_iter() + .map(|result| match result { + Ok(r) => Reportable::Assigned(r), + Err(e) => Reportable::Error(e), + }) + .collect(); + if !result.is_empty() { + if let Err(err) = client.report(result).await { + ErrChan::send(err, "report").await; + } } - sleep(Duration::from_secs(5)).await; + sleep(Duration::from_secs(ITERATION_LATENCY)).await; + } +} + +pub async fn run_forever() -> ! { + let env = load_env_default().unwrap(); + let client = Arc::new(ClientHandler::new(&env.u_server, None)); + tokio::spawn(error_reporting(client.clone())); + + if cfg!(debug_assertions) { + init_logger(Some(format!( + "u_agent-{}", + get_self_uid() + .hyphenated() + .to_string() + .split("-") + .next() + .unwrap() + ))); + // } else { + // if let Err(e) = Daemonize::new().start() { + // ErrChan::send(UError::Runtime(e.to_string()), "deeeemon").await + // } } + info!("Starting agent {}", get_self_uid()); + agent_loop(client).await } diff --git a/bin/u_agent/src/main.rs b/bin/u_agent/src/main.rs index 43166a8..1f3d50a 100644 --- a/bin/u_agent/src/main.rs +++ b/bin/u_agent/src/main.rs @@ -1,4 +1,3 @@ -use tokio; use u_agent::run_forever; #[tokio::main] diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index 1f15786..e6705f6 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -2,18 +2,27 @@ name = "u_panel" version = "0.1.0" authors = ["plazmoid "] -edition = "2018" +edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process"] } +actix-cors = "0.6.1" +actix-web = "4.1" +anyhow = { workspace = true } +futures-util = "0.3.21" +mime_guess = "2.0.4" +once_cell = "1.8.0" +rust-embed = { version = "6.3.0", features = ["debug-embed", "compression"] } +serde = { workspace = true } +serde_json = { workspace = true } +strum = { version = "0.22.0", features = ["derive"] } +tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +tracing-appender = { workspace = true } +shlex = "1.1.0" structopt = "0.3.21" -log = "^0.4" -env_logger = "0.7.1" -uuid = "0.6.5" -reqwest = { version = "0.11", features = ["json"] } -openssl = "*" -u_lib = { version = "*", path = "../../lib/u_lib" } -serde_json = "1.0.4" -serde = { version = "1.0.114", features = ["derive"] } +thiserror = "1.0.31" +uuid = { workspace = true } +u_lib = { version = "*", path = "../../lib/u_lib", features = ["panel"] } diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs new file mode 100644 index 0000000..4e43067 --- /dev/null +++ b/bin/u_panel/src/argparse.rs @@ -0,0 +1,129 @@ +use serde_json::{from_str, to_value, Value}; +use structopt::StructOpt; +use u_lib::{ + api::ClientHandler, + datatypes::PanelResult, + messaging::AsMsg, + models::{Agent, AssignedJob, JobMeta}, + UError, UResult, +}; +use uuid::Uuid; + +#[derive(StructOpt, Debug)] +pub struct Args { + #[structopt(subcommand)] + cmd: Cmd, +} + +#[derive(StructOpt, Debug)] +enum Cmd { + Agents(RUD), + Jobs(JobCRUD), + Map(JobMapCRUD), + Ping, + Serve, +} + +#[derive(StructOpt, Debug)] +enum JobCRUD { + Create { + job: String, + }, + #[structopt(flatten)] + RUD(RUD), +} + +#[derive(StructOpt, Debug)] +enum JobCmd { + #[structopt(external_subcommand)] + Cmd(Vec), +} + +#[derive(StructOpt, Debug)] +enum JobMapCRUD { + Create { + #[structopt(parse(try_from_str = parse_uuid))] + agent_uid: Uuid, + + job_idents: Vec, + }, + #[structopt(flatten)] + RUD(RUD), +} + +#[derive(StructOpt, Debug)] +enum RUD { + Read { + #[structopt(parse(try_from_str = parse_uuid))] + uid: Option, + }, + Update { + item: String, + }, + Delete { + #[structopt(parse(try_from_str = parse_uuid))] + uid: Uuid, + }, +} + +fn parse_uuid(src: &str) -> Result { + Uuid::parse_str(src).map_err(|e| e.to_string()) +} + +pub fn into_value(data: M) -> Value { + to_value(data).unwrap() +} + +pub async fn process_cmd(client: ClientHandler, args: Args) -> PanelResult { + let catcher: UResult = (|| async { + Ok(match args.cmd { + Cmd::Agents(action) => match action { + RUD::Read { uid } => into_value(client.get_agents(uid).await?), + RUD::Update { item } => { + let agent = from_str::(&item)?; + into_value(client.update_agent(agent).await?) + } + RUD::Delete { uid } => into_value(client.del(uid).await?), + }, + Cmd::Jobs(action) => match action { + JobCRUD::Create { job } => { + let raw_job = from_str::(&job)?; + let job = raw_job.validated()?; + into_value(client.upload_jobs(job).await?) + } + JobCRUD::RUD(RUD::Read { uid }) => into_value(client.get_jobs(uid).await?), + JobCRUD::RUD(RUD::Update { item }) => { + let raw_job = from_str::(&item)?; + let job = raw_job.validated()?; + into_value(client.update_job(job).await?) + } + JobCRUD::RUD(RUD::Delete { uid }) => into_value(client.del(uid).await?), + }, + Cmd::Map(action) => match action { + JobMapCRUD::Create { + agent_uid, + job_idents, + } => into_value(client.set_jobs(agent_uid, job_idents).await?), + JobMapCRUD::RUD(RUD::Read { uid }) => into_value(client.get_agent_jobs(uid).await?), + JobMapCRUD::RUD(RUD::Update { item }) => { + let assigned = from_str::(&item)?; + into_value(client.update_result(assigned).await?) + } + JobMapCRUD::RUD(RUD::Delete { uid }) => into_value(client.del(uid).await?), + }, + Cmd::Ping => into_value(client.ping().await?), + Cmd::Serve => { + crate::server::serve(client) + .await + .map_err(|e| UError::PanelError(format!("{e:?}")))?; + Value::Null + } + }) + })() + .await; + + match catcher { + Ok(r) => PanelResult::Ok(r), + Err(e) => PanelResult::Err(e), + } +} diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index 4b86905..595cbfa 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -1,143 +1,33 @@ -use std::env; -use std::fmt; -use structopt::StructOpt; -use u_lib::{ - api::ClientHandler, datatypes::DataResult, messaging::AsMsg, models::JobMeta, utils::init_env, - UResult, -}; -use uuid::Uuid; - -#[derive(StructOpt, Debug)] -struct Args { - #[structopt(subcommand)] - cmd: Cmd, - #[structopt(long)] - json: bool, -} - -#[derive(StructOpt, Debug)] -enum Cmd { - Agents(LD), - Jobs(JobALD), - Jobmap(JobMapALD), -} +mod argparse; +mod server; -#[derive(StructOpt, Debug)] -enum JobALD { - Add { - #[structopt(long, parse(try_from_str = parse_uuid))] - agent: Option, +#[macro_use] +extern crate tracing; - #[structopt(long)] - alias: String, - - #[structopt(subcommand)] - cmd: JobCmd, - }, - #[structopt(flatten)] - LD(LD), -} - -#[derive(StructOpt, Debug)] -enum JobCmd { - #[structopt(external_subcommand)] - Cmd(Vec), -} - -#[derive(StructOpt, Debug)] -enum JobMapALD { - Add { - #[structopt(parse(try_from_str = parse_uuid))] - agent_uid: Uuid, - - job_idents: Vec, - }, - List { - #[structopt(parse(try_from_str = parse_uuid))] - uid: Option, - }, - Delete { - #[structopt(parse(try_from_str = parse_uuid))] - uid: Uuid, - }, -} - -#[derive(StructOpt, Debug)] -enum LD { - List { - #[structopt(parse(try_from_str = parse_uuid))] - uid: Option, - }, - Delete { - #[structopt(parse(try_from_str = parse_uuid))] - uid: Uuid, - }, -} +use anyhow::Result as AnyResult; +use argparse::{process_cmd, Args}; +use serde::Deserialize; +use structopt::StructOpt; +use u_lib::api::ClientHandler; +use u_lib::logging::init_logger; +use u_lib::utils::{env::default_host, load_env}; -fn parse_uuid(src: &str) -> Result { - Uuid::parse_str(src).map_err(|e| e.to_string()) +#[derive(Deserialize)] +struct AccessEnv { + admin_auth_token: String, + #[serde(default = "default_host")] + u_server: String, } -async fn process_cmd(args: Args) { - struct Printer { - json: bool, - } +#[actix_web::main] +async fn main() -> AnyResult<()> { + let env = load_env::()?; + let client = ClientHandler::new(&env.u_server, Some(env.admin_auth_token)); + let args = Args::from_args(); - impl Printer { - pub fn print(&self, data: UResult) { - if self.json { - let data = match data { - Ok(r) => DataResult::Ok(r), - Err(e) => DataResult::Err(e), - }; - println!("{}", serde_json::to_string_pretty(&data).unwrap()); - } else { - match data { - Ok(r) => println!("{}", r), - Err(e) => eprintln!("Error: {}", e), - } - } - } - } - - let token = env::var("ADMIN_AUTH_TOKEN").expect("Authentication token is not set"); - let cli_handler = ClientHandler::new(None).password(token); - let printer = Printer { json: args.json }; - match args.cmd { - Cmd::Agents(action) => match action { - LD::List { uid } => printer.print(cli_handler.get_agents(uid).await), - LD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await), - }, - Cmd::Jobs(action) => match action { - JobALD::Add { - cmd: JobCmd::Cmd(cmd), - alias, - agent: _agent, - } => { - let job = JobMeta::builder() - .with_shell(cmd.join(" ")) - .with_alias(alias) - .build() - .unwrap(); - printer.print(cli_handler.upload_jobs(&[job]).await); - } - JobALD::LD(LD::List { uid }) => printer.print(cli_handler.get_jobs(uid).await), - JobALD::LD(LD::Delete { uid }) => printer.print(cli_handler.del(Some(uid)).await), - }, - Cmd::Jobmap(action) => match action { - JobMapALD::Add { - agent_uid, - job_idents, - } => printer.print(cli_handler.set_jobs(Some(agent_uid), &job_idents).await), - JobMapALD::List { uid } => printer.print(cli_handler.get_agent_jobs(uid).await), - JobMapALD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await), - }, - } -} + init_logger(None::<&str>); -#[tokio::main] -async fn main() { - init_env(); - let args: Args = Args::from_args(); - process_cmd(args).await; + let result = process_cmd(client, args).await.to_string(); + println!("{result}"); + Ok(()) } diff --git a/bin/u_panel/src/server/error.rs b/bin/u_panel/src/server/error.rs new file mode 100644 index 0000000..003ff2f --- /dev/null +++ b/bin/u_panel/src/server/error.rs @@ -0,0 +1,17 @@ +use actix_web::http::StatusCode; +use actix_web::ResponseError; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Arg parse error: {0}")] + ArgparseError(#[from] structopt::clap::Error), + + #[error("Just an error: {0}")] + JustError(String), +} + +impl ResponseError for Error { + fn status_code(&self) -> actix_web::http::StatusCode { + StatusCode::BAD_REQUEST + } +} diff --git a/bin/u_panel/src/server/fe/.gitignore b/bin/u_panel/src/server/fe/.gitignore new file mode 100644 index 0000000..e5c1ba1 --- /dev/null +++ b/bin/u_panel/src/server/fe/.gitignore @@ -0,0 +1,48 @@ +# See http://help.github.com/ignore-files/ for more about ignoring files. + +# compiled output +/dist +/tmp +/out-tsc +# Only exists if Bazel was run +/bazel-out + +# dependencies +/node_modules + +# profiling files +chrome-profiler-events*.json + +# IDEs and editors +/.idea +.project +.classpath +.c9/ +*.launch +.settings/ +*.sublime-workspace + +# IDE - VSCode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +.history/* + +# misc +/.angular/cache +/.sass-cache +/connect.lock +/coverage +/libpeerconnection.log +npm-debug.log +yarn-error.log +testem.log +/typings + +# System Files +.DS_Store +Thumbs.db + +package-lock.json \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/README.md b/bin/u_panel/src/server/fe/README.md new file mode 100644 index 0000000..d16d556 --- /dev/null +++ b/bin/u_panel/src/server/fe/README.md @@ -0,0 +1,27 @@ +# Fe + +This project was generated with [Angular CLI](https://github.com/angular/angular-cli) version 13.1.2. + +## Development server + +Run `ng serve` for a dev server. Navigate to `http://localhost:4200/`. The app will automatically reload if you change any of the source files. + +## Code scaffolding + +Run `ng generate component component-name` to generate a new component. You can also use `ng generate directive|pipe|service|class|guard|interface|enum|module`. + +## Build + +Run `ng build` to build the project. The build artifacts will be stored in the `dist/` directory. + +## Running unit tests + +Run `ng test` to execute the unit tests via [Karma](https://karma-runner.github.io). + +## Running end-to-end tests + +Run `ng e2e` to execute the end-to-end tests via a platform of your choice. To use this command, you need to first add a package that implements end-to-end testing capabilities. + +## Further help + +To get more help on the Angular CLI use `ng help` or go check out the [Angular CLI Overview and Command Reference](https://angular.io/cli) page. diff --git a/bin/u_panel/src/server/fe/angular.json b/bin/u_panel/src/server/fe/angular.json new file mode 100644 index 0000000..d35487b --- /dev/null +++ b/bin/u_panel/src/server/fe/angular.json @@ -0,0 +1,114 @@ +{ + "$schema": "./node_modules/@angular/cli/lib/config/schema.json", + "version": 1, + "newProjectRoot": "projects", + "projects": { + "fe": { + "projectType": "application", + "schematics": { + "@schematics/angular:component": { + "style": "less" + }, + "@schematics/angular:application": { + "strict": true + } + }, + "root": "", + "sourceRoot": "src", + "prefix": "app", + "architect": { + "build": { + "builder": "@angular-devkit/build-angular:browser", + "options": { + "outputPath": "dist/fe", + "index": "src/index.html", + "main": "src/main.ts", + "polyfills": "src/polyfills.ts", + "tsConfig": "tsconfig.app.json", + "inlineStyleLanguage": "less", + "assets": [ + "src/favicon.ico", + "src/assets" + ], + "styles": [ + "./node_modules/@angular/material/prebuilt-themes/deeppurple-amber.css", + "src/styles.less" + ], + "scripts": [] + }, + "configurations": { + "production": { + "baseHref": "/core/", + "budgets": [ + { + "type": "initial", + "maximumWarning": "500kb", + "maximumError": "1mb" + }, + { + "type": "anyComponentStyle", + "maximumWarning": "2kb", + "maximumError": "4kb" + } + ], + "fileReplacements": [ + { + "replace": "src/environments/environment.ts", + "with": "src/environments/environment.prod.ts" + } + ], + "outputHashing": "all" + }, + "development": { + "buildOptimizer": false, + "optimization": false, + "vendorChunk": true, + "extractLicenses": false, + "sourceMap": true, + "namedChunks": true + } + }, + "defaultConfiguration": "production" + }, + "serve": { + "builder": "@angular-devkit/build-angular:dev-server", + "configurations": { + "production": { + "browserTarget": "fe:build:production" + }, + "development": { + "browserTarget": "fe:build:development" + } + }, + "defaultConfiguration": "development" + }, + "extract-i18n": { + "builder": "@angular-devkit/build-angular:extract-i18n", + "options": { + "browserTarget": "fe:build" + } + }, + "test": { + "builder": "@angular-devkit/build-angular:karma", + "options": { + "main": "src/test.ts", + "polyfills": "src/polyfills.ts", + "tsConfig": "tsconfig.spec.json", + "karmaConfig": "karma.conf.js", + "inlineStyleLanguage": "less", + "assets": [ + "src/favicon.ico", + "src/assets" + ], + "styles": [ + "./node_modules/@angular/material/prebuilt-themes/deeppurple-amber.css", + "src/styles.less" + ], + "scripts": [] + } + } + } + } + }, + "defaultProject": "fe" +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/karma.conf.js b/bin/u_panel/src/server/fe/karma.conf.js new file mode 100644 index 0000000..07a1538 --- /dev/null +++ b/bin/u_panel/src/server/fe/karma.conf.js @@ -0,0 +1,44 @@ +// Karma configuration file, see link for more information +// https://karma-runner.github.io/1.0/config/configuration-file.html + +module.exports = function (config) { + config.set({ + basePath: '', + frameworks: ['jasmine', '@angular-devkit/build-angular'], + plugins: [ + require('karma-jasmine'), + require('karma-chrome-launcher'), + require('karma-jasmine-html-reporter'), + require('karma-coverage'), + require('@angular-devkit/build-angular/plugins/karma') + ], + client: { + jasmine: { + // you can add configuration options for Jasmine here + // the possible options are listed at https://jasmine.github.io/api/edge/Configuration.html + // for example, you can disable the random execution with `random: false` + // or set a specific seed with `seed: 4321` + }, + clearContext: false // leave Jasmine Spec Runner output visible in browser + }, + jasmineHtmlReporter: { + suppressAll: true // removes the duplicated traces + }, + coverageReporter: { + dir: require('path').join(__dirname, './coverage/fe'), + subdir: '.', + reporters: [ + { type: 'html' }, + { type: 'text-summary' } + ] + }, + reporters: ['progress', 'kjhtml'], + port: 9876, + colors: true, + logLevel: config.LOG_INFO, + autoWatch: true, + browsers: ['Chrome'], + singleRun: false, + restartOnFileChange: true + }); +}; diff --git a/bin/u_panel/src/server/fe/package.json b/bin/u_panel/src/server/fe/package.json new file mode 100644 index 0000000..fb86167 --- /dev/null +++ b/bin/u_panel/src/server/fe/package.json @@ -0,0 +1,43 @@ +{ + "name": "fe", + "version": "0.0.0", + "scripts": { + "ng": "ng", + "start": "ng serve", + "build": "ng build", + "watch": "ng build --watch --configuration development", + "test": "ng test" + }, + "private": true, + "dependencies": { + "@angular/animations": "~13.1.0", + "@angular/cdk": "^13.3.9", + "@angular/common": "~13.1.0", + "@angular/compiler": "~13.1.0", + "@angular/core": "~13.1.0", + "@angular/forms": "~13.1.0", + "@angular/material": "^13.3.9", + "@angular/platform-browser": "~13.1.0", + "@angular/platform-browser-dynamic": "~13.1.0", + "@angular/router": "~13.1.0", + "@types/uuid": "^8.3.4", + "rxjs": "~7.4.0", + "tslib": "^2.3.0", + "uuid": "^8.3.2", + "zone.js": "~0.11.4" + }, + "devDependencies": { + "@angular-devkit/build-angular": "^13.3.9", + "@angular/cli": "~13.1.2", + "@angular/compiler-cli": "~13.1.0", + "@types/jasmine": "~3.10.0", + "@types/node": "^12.11.1", + "jasmine-core": "~3.10.0", + "karma": "~6.3.0", + "karma-chrome-launcher": "~3.1.0", + "karma-coverage": "~2.1.0", + "karma-jasmine": "~4.0.0", + "karma-jasmine-html-reporter": "~1.7.0", + "typescript": "~4.5.2" + } +} diff --git a/bin/u_panel/src/server/fe/src/app/app-routing.module.ts b/bin/u_panel/src/server/fe/src/app/app-routing.module.ts new file mode 100644 index 0000000..0ec7bf4 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/app-routing.module.ts @@ -0,0 +1,19 @@ +import { NgModule } from '@angular/core'; +import { RouterModule, Routes } from '@angular/router'; +import { AgentComponent } from './core/tables/agent.component'; +import { JobComponent } from './core/tables/job.component'; +import { ResultComponent } from './core/tables/result.component'; +import { AgentInfoDialogComponent } from './core/tables/dialogs/agent_info.component'; + +const routes: Routes = [ + { path: '', redirectTo: 'agents', pathMatch: 'full' }, + { path: 'agents', component: AgentComponent }, + { path: 'jobs', component: JobComponent }, + { path: 'results', component: ResultComponent }, +]; + +@NgModule({ + imports: [RouterModule.forRoot(routes)], + exports: [RouterModule] +}) +export class AppRoutingModule { } diff --git a/bin/u_panel/src/server/fe/src/app/app.component.html b/bin/u_panel/src/server/fe/src/app/app.component.html new file mode 100644 index 0000000..b35dfd7 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/app.component.html @@ -0,0 +1,5 @@ + + \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/app.component.less b/bin/u_panel/src/server/fe/src/app/app.component.less new file mode 100644 index 0000000..e69de29 diff --git a/bin/u_panel/src/server/fe/src/app/app.component.spec.ts b/bin/u_panel/src/server/fe/src/app/app.component.spec.ts new file mode 100644 index 0000000..d2ec0e1 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/app.component.spec.ts @@ -0,0 +1,35 @@ +import { TestBed } from '@angular/core/testing'; +import { RouterTestingModule } from '@angular/router/testing'; +import { AppComponent } from './app.component'; + +describe('AppComponent', () => { + beforeEach(async () => { + await TestBed.configureTestingModule({ + imports: [ + RouterTestingModule + ], + declarations: [ + AppComponent + ], + }).compileComponents(); + }); + + it('should create the app', () => { + const fixture = TestBed.createComponent(AppComponent); + const app = fixture.componentInstance; + expect(app).toBeTruthy(); + }); + + it(`should have as title 'fe'`, () => { + const fixture = TestBed.createComponent(AppComponent); + const app = fixture.componentInstance; + expect(app.title).toEqual('fe'); + }); + + it('should render title', () => { + const fixture = TestBed.createComponent(AppComponent); + fixture.detectChanges(); + const compiled = fixture.nativeElement as HTMLElement; + expect(compiled.querySelector('.content span')?.textContent).toContain('fe app is running!'); + }); +}); diff --git a/bin/u_panel/src/server/fe/src/app/app.component.ts b/bin/u_panel/src/server/fe/src/app/app.component.ts new file mode 100644 index 0000000..566f46b --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/app.component.ts @@ -0,0 +1,14 @@ +import { Component, ViewChild, AfterViewInit } from '@angular/core'; + +@Component({ + selector: 'app-root', + templateUrl: './app.component.html', + styleUrls: ['./app.component.less'] +}) +export class AppComponent { + tabs = [ + { name: 'Agents', link: '/agents' }, + { name: 'Jobs', link: '/jobs' }, + { name: 'Results', link: '/results' } + ]; +} diff --git a/bin/u_panel/src/server/fe/src/app/app.module.ts b/bin/u_panel/src/server/fe/src/app/app.module.ts new file mode 100644 index 0000000..c1572f4 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/app.module.ts @@ -0,0 +1,60 @@ +import { NgModule } from '@angular/core'; +import { BrowserModule } from '@angular/platform-browser'; +import { AppRoutingModule } from './app-routing.module'; +import { AppComponent } from './app.component'; +import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; +import { MatTabsModule } from '@angular/material/tabs'; +import { MatTableModule } from '@angular/material/table'; +import { MatFormFieldModule } from '@angular/material/form-field'; +import { MatButtonModule } from '@angular/material/button' +import { MatInputModule } from '@angular/material/input'; +import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; +import { HttpClientModule } from '@angular/common/http'; +import { MatDialogModule } from '@angular/material/dialog'; +import { MatIconModule } from '@angular/material/icon'; +import { FormsModule } from '@angular/forms'; +import { AgentComponent, JobComponent, ResultComponent } from './core/tables'; +import { + AgentInfoDialogComponent, + AssignJobDialogComponent, + JobInfoDialogComponent, + ResultInfoDialogComponent +} from './core/tables/dialogs'; +import { APP_BASE_HREF } from '@angular/common'; +import { MatTooltipModule } from '@angular/material/tooltip'; +import { MatSnackBarModule } from '@angular/material/snack-bar'; +import { MatListModule } from '@angular/material/list'; + +@NgModule({ + declarations: [ + AppComponent, + AgentComponent, + JobComponent, + ResultComponent, + AgentInfoDialogComponent, + JobInfoDialogComponent, + ResultInfoDialogComponent, + AssignJobDialogComponent + ], + imports: [ + BrowserModule, + HttpClientModule, + AppRoutingModule, + MatTabsModule, + MatTableModule, + MatButtonModule, + MatFormFieldModule, + MatInputModule, + MatDialogModule, + MatProgressSpinnerModule, + MatIconModule, + MatTooltipModule, + MatSnackBarModule, + MatListModule, + FormsModule, + BrowserAnimationsModule + ], + providers: [{ provide: APP_BASE_HREF, useValue: '/' }], + bootstrap: [AppComponent] +}) +export class AppModule { } diff --git a/bin/u_panel/src/server/fe/src/app/core/index.ts b/bin/u_panel/src/server/fe/src/app/core/index.ts new file mode 100644 index 0000000..b248abc --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/index.ts @@ -0,0 +1 @@ +export * from './services'; \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts b/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts new file mode 100644 index 0000000..d798f0e --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts @@ -0,0 +1,16 @@ +import { UTCDate, ApiModel } from "."; + +export interface AgentModel extends ApiModel { + alias: string | null, + hostname: string, + host_info: string, + id: string, + is_root: boolean, + is_root_allowed: boolean, + last_active: UTCDate, + platform: string, + regtime: UTCDate, + state: "new" | "active" | "banned", + token: string | null, + username: string, +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/models/index.ts b/bin/u_panel/src/server/fe/src/app/core/models/index.ts new file mode 100644 index 0000000..dad077a --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/models/index.ts @@ -0,0 +1,14 @@ +export * from './agent.model'; +export * from './result.model'; +export * from './job.model'; + +export interface UTCDate { + secs_since_epoch: number, + nanos_since_epoch: number +} + +export abstract class ApiModel { } + +export interface Empty extends ApiModel { } + +export type Area = "agents" | "jobs" | "map"; \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/models/job.model.ts b/bin/u_panel/src/server/fe/src/app/core/models/job.model.ts new file mode 100644 index 0000000..9db303f --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/models/job.model.ts @@ -0,0 +1,12 @@ +import { ApiModel } from "."; + +export interface JobModel extends ApiModel { + alias: string, + argv: string, + id: string, + exec_type: string, + platform: string, + payload: number[] | null, + payload_path: string | null, + schedule: string | null, +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/models/result.model.ts b/bin/u_panel/src/server/fe/src/app/core/models/result.model.ts new file mode 100644 index 0000000..c699787 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/models/result.model.ts @@ -0,0 +1,13 @@ +import { UTCDate, ApiModel } from "."; + +export interface ResultModel extends ApiModel { + agent_id: string, + alias: string, + created: UTCDate, + id: string, + job_id: string, + result: number[], + state: "Queued" | "Running" | "Finished", + retcode: number | null, + updated: UTCDate, +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts b/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts new file mode 100644 index 0000000..84348e2 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/services/api.service.ts @@ -0,0 +1,53 @@ +import { environment } from 'src/environments/environment'; +import { HttpClient } from '@angular/common/http'; +import { firstValueFrom } from 'rxjs'; +import { ApiModel, Empty, Area } from '../models'; + +interface ServerResponse { + status: "ok" | "err", + data: T | string +} + +export class ApiTableService { + area: Area; + + constructor(private http: HttpClient, area: Area) { + this.area = area; + } + + requestUrl = `${environment.server}/cmd/`; + + async req(cmd: string): Promise> { + return await firstValueFrom(this.http.post>(this.requestUrl, cmd)) + } + + async getOne(id: string, area: string = this.area): Promise> { + const resp = await this.req(`${area} read ${id}`) + if (resp.data.length === 0) { + return { + status: 'err', + data: `${id} not found in ${area}` + } + } + return { + status: resp.status, + data: resp.data[0] + } + } + + async getMany(): Promise> { + return await this.req(`${this.area} read`) + } + + async update(item: T): Promise> { + return await this.req(`${this.area} update '${JSON.stringify(item)}'`) + } + + async delete(id: string): Promise> { + return await this.req(`${this.area} delete ${id}`) + } + + async create(item: string): Promise> { + return await this.req(`${this.area} create ${item}`) + } +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/services/index.ts b/bin/u_panel/src/server/fe/src/app/core/services/index.ts new file mode 100644 index 0000000..aeef56f --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/services/index.ts @@ -0,0 +1 @@ +export * from './api.service' \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html new file mode 100644 index 0000000..fbaaba9 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html @@ -0,0 +1,78 @@ +
+ +
+
+ +
+ + Filter + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ID + {{row.id}} + Alias + {{row.alias}} + User + {{row.username}} + Hostname + {{row.hostname}} + Last active + {{row.last_active.secs_since_epoch * 1000 | date:'long'}} + + + | + + | + +
No data
+
+ + +
\ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts new file mode 100644 index 0000000..c72b8dc --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts @@ -0,0 +1,52 @@ +import { Component, OnDestroy, OnInit } from '@angular/core'; +import { TablesComponent } from './table.component'; +import { AgentModel } from '../models'; +import { AgentInfoDialogComponent } from './dialogs/agent_info.component'; +import { HttpErrorResponse } from '@angular/common/http'; +import { AssignJobDialogComponent } from './dialogs'; + +@Component({ + selector: 'agent-table', + templateUrl: './agent.component.html', + styleUrls: ['./table.component.less'] +}) +export class AgentComponent extends TablesComponent implements OnInit { + + //dialogSubscr!: Subscription; + area = 'agents' as const; + + displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions'] + + show_item_dialog(id: string) { + this.data_source!.getOne(id).then(resp => { + if (resp.status === 'ok') { + const dialog = this.infoDialog.open(AgentInfoDialogComponent, { + data: resp.data as AgentModel, + width: '1000px', + }); + + const saveSub = dialog.componentInstance.onSave.subscribe(result => { + this.data_source!.update(result).then(_ => { + this.openSnackBar('Saved', false) + this.loadTableData() + }) + .catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) + }) + + dialog.afterClosed().subscribe(result => { + saveSub.unsubscribe() + this.router.navigate(['.'], { relativeTo: this.route }) + }) + } else { + this.openSnackBar(resp.data) + } + }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) + } + + assignJobs(id: string) { + const dialog = this.infoDialog.open(AssignJobDialogComponent, { + data: id, + width: '1000px', + }); + } +} diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html new file mode 100644 index 0000000..ff66ad5 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html @@ -0,0 +1,64 @@ +

Agent info

+

Editing agent info

+ +

+ + ID + + +

+

+ + Alias + + +

+

+ + Username + + +

+

+ + Hostname + + +

+

+ + Host info + + +

+

+ + Platform + + +

+

+ + Is root + + +

+

+ + Registration time + + +

+

+ + Last active time + + +

+ + + + + + \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts new file mode 100644 index 0000000..d24d3a9 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts @@ -0,0 +1,21 @@ +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { AgentModel } from '../../models/agent.model'; +import { EventEmitter } from '@angular/core'; + +@Component({ + selector: 'agent-info-dialog', + templateUrl: 'agent-info-dialog.html', + styleUrls: ['info-dialog.component.less'] +}) +export class AgentInfoDialogComponent { + is_preview = true; + onSave = new EventEmitter(); + + constructor(@Inject(MAT_DIALOG_DATA) public data: AgentModel) { } + + updateAgent() { + console.log(this.data); + this.onSave.emit(this.data); + } +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign-job-dialog.html b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign-job-dialog.html new file mode 100644 index 0000000..2ebe970 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign-job-dialog.html @@ -0,0 +1,12 @@ +

Assign job

+ + + + {{row}} + + + + + + + \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign_job.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign_job.component.ts new file mode 100644 index 0000000..cd278d6 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign_job.component.ts @@ -0,0 +1,33 @@ +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { HttpClient } from '@angular/common/http'; +import { ApiTableService } from '../../services'; +import { JobModel } from '../../models'; +import { MatListOption } from '@angular/material/list'; + +@Component({ + selector: 'assign-job-dialog', + templateUrl: 'assign-job-dialog.html', + styleUrls: [] +}) +export class AssignJobDialogComponent { + rows: string[] = []; + selected_rows: string[] = []; + + constructor(@Inject(MAT_DIALOG_DATA) public agent_id: string, private http: HttpClient) { + new ApiTableService(http, "jobs").getMany().then(result => { + if (result.status == "ok") { + const jobs = result.data as JobModel[] + this.rows = jobs.map(j => `${j.id} ${j.alias}`) + } else { + alert(result.data as string) + } + }).catch(err => alert(err)) + } + + assignSelectedJobs() { + const job_ids = this.selected_rows.map(row => row.split(' ', 1)[0]).join(' '); + const request = `${this.agent_id} ${job_ids}` + new ApiTableService(this.http, "map").create(request).catch(err => alert(err)) + } +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/index.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/index.ts new file mode 100644 index 0000000..4bdb1aa --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/index.ts @@ -0,0 +1,4 @@ +export * from './agent_info.component'; +export * from './result_info.component'; +export * from './job_info.component'; +export * from './assign_job.component'; \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/info-dialog.component.less b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/info-dialog.component.less new file mode 100644 index 0000000..23b8bc5 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/info-dialog.component.less @@ -0,0 +1,14 @@ +.info-dlg-field { + width: 100%; +} + +div.info-dialog-forms-box { + width: 100%; + margin-right: 10px; +} + +div.info-dialog-forms-box-smol { + width: 30%; + float: left; + margin-right: 10px; +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job-info-dialog.html b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job-info-dialog.html new file mode 100644 index 0000000..79108e6 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job-info-dialog.html @@ -0,0 +1,50 @@ +

Job info

+

Editing job info

+ +
+ + ID + + + + Alias + + + + Args + + +
+
+ + Type + + + + Platform + + + + Schedule + + +
+
+ + Payload path + + +
+
+ + Payload + + +
+ + + + + + \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job_info.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job_info.component.ts new file mode 100644 index 0000000..08c29da --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job_info.component.ts @@ -0,0 +1,30 @@ +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { JobModel } from '../../models/job.model'; +import { EventEmitter } from '@angular/core'; + +@Component({ + selector: 'job-info-dialog', + templateUrl: 'job-info-dialog.html', + styleUrls: ['info-dialog.component.less'] +}) +export class JobInfoDialogComponent { + is_preview = true; + decodedPayload: string; + onSave = new EventEmitter(); + + constructor(@Inject(MAT_DIALOG_DATA) public data: JobModel) { + if (data.payload !== null) { + this.decodedPayload = new TextDecoder().decode(new Uint8Array(data.payload)) + } else { + this.decodedPayload = "" + } + } + + updateJob() { + if (this.decodedPayload.length > 0) { + this.data.payload = Array.from(new TextEncoder().encode(this.decodedPayload)) + } + this.onSave.emit(this.data); + } +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result-info-dialog.html b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result-info-dialog.html new file mode 100644 index 0000000..6ce43c2 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result-info-dialog.html @@ -0,0 +1,53 @@ +

Result

+ +
+ + ID + + + + Job ID + + + + Agent ID + + +
+
+ + Alias + + + + State + + + + Return code + + +
+
+ + Created + + + + Updated + + +
+
+

+ + Result + + +

+
+ + + + \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result_info.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result_info.component.ts new file mode 100644 index 0000000..b02fae5 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result_info.component.ts @@ -0,0 +1,20 @@ +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { ResultModel } from '../../models/result.model'; + +@Component({ + selector: 'result-info-dialog', + templateUrl: 'result-info-dialog.html', + styleUrls: ['info-dialog.component.less'] +}) +export class ResultInfoDialogComponent { + decodedResult: string; + + constructor(@Inject(MAT_DIALOG_DATA) public data: ResultModel) { + if (data.result !== null) { + this.decodedResult = new TextDecoder().decode(new Uint8Array(data.result)) + } else { + this.decodedResult = "" + } + } +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/index.ts b/bin/u_panel/src/server/fe/src/app/core/tables/index.ts new file mode 100644 index 0000000..11dfaf2 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/index.ts @@ -0,0 +1,3 @@ +export * from './agent.component'; +export * from './job.component'; +export * from './result.component'; \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html new file mode 100644 index 0000000..bfd99b2 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.html @@ -0,0 +1,83 @@ +
+ +
+
+ +
+ + Filter + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ID + {{row.id}} + Alias + {{row.alias}} + Cmd-line args + {{row.argv}} + Platform + {{row.platform}} + Schedule + {{row.schedule}} + Type + {{row.exec_type}} + + + | + +
No data
+
+ + +
\ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts new file mode 100644 index 0000000..4a5e1cf --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts @@ -0,0 +1,59 @@ +import { Component, OnInit } from '@angular/core'; +import { TablesComponent } from './table.component'; +import { JobModel } from '../models'; +import { JobInfoDialogComponent } from './dialogs'; +import { HttpErrorResponse } from '@angular/common/http'; + +@Component({ + selector: 'job-table', + templateUrl: './job.component.html', + styleUrls: ['./table.component.less'] +}) +export class JobComponent extends TablesComponent { + area = 'jobs' as const; + displayedColumns = ['id', 'alias', 'platform', 'schedule', 'exec_type', 'actions'] + + show_item_dialog(id: string | null) { + const show_dlg = (id: string, edit: boolean) => { + this.data_source!.getOne(id).then(resp => { + if (resp.status === 'ok') { + var dialog = this.infoDialog.open(JobInfoDialogComponent, { + data: resp.data as JobModel, + width: '1000px', + }); + if (edit) { + dialog.componentInstance.is_preview = false + } + + const saveSub = dialog.componentInstance.onSave.subscribe(result => { + this.data_source!.update(result) + .then(_ => { + this.openSnackBar("Saved", false) + this.loadTableData() + }) + .catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) + }) + + dialog.afterClosed().subscribe(result => { + saveSub.unsubscribe() + this.router.navigate(['.'], { relativeTo: this.route }) + }) + } else { + this.openSnackBar(resp.data) + } + }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) + } + + if (id) { + show_dlg(id, false) + } else { + this.data_source!.create('"{}"').then(resp => { + if (resp.status === 'ok') { + show_dlg(resp.data[0], true) + } else { + this.openSnackBar(resp.data) + } + }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) + } + } +} diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html new file mode 100644 index 0000000..afda1cb --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.html @@ -0,0 +1,81 @@ +
+ +
+
+ +
+ + Filter + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ID + {{row.id}} + Alias + {{row.alias}} + Agent + {{row.agent_id}} + Job + {{row.job_id}} + State + {{row.state}} {{(row.state === "Finished") ? '(' + row.retcode + ')' : ''}} + ID + {{row.updated.secs_since_epoch * 1000| date:'long'}} + + + | + +
No data
+
+ + +
\ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts new file mode 100644 index 0000000..ec98ab7 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts @@ -0,0 +1,41 @@ +import { Component, OnInit } from '@angular/core'; +import { TablesComponent } from './table.component'; +import { ResultModel } from '../models'; +import { ResultInfoDialogComponent } from './dialogs'; +import { HttpErrorResponse } from '@angular/common/http'; + +@Component({ + selector: 'results-table', + templateUrl: './result.component.html', + styleUrls: ['./table.component.less'] +}) +export class ResultComponent extends TablesComponent { + area = 'map' as const; + + displayedColumns = [ + 'id', + 'alias', + 'agent_id', + 'job_id', + 'state', + 'last_updated', + 'actions' + ]; + + show_item_dialog(id: string) { + this.data_source!.getOne(id).then(resp => { + if (resp.status === 'ok') { + const dialog = this.infoDialog.open(ResultInfoDialogComponent, { + data: resp.data as ResultModel, + width: '1000px', + }); + + dialog.afterClosed().subscribe(result => { + this.router.navigate(['.'], { relativeTo: this.route }) + }) + } else { + this.openSnackBar(resp.data) + } + }).catch((err: HttpErrorResponse) => this.openSnackBar(err.message)) + } +} diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less new file mode 100644 index 0000000..167dd69 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.less @@ -0,0 +1,32 @@ +.data-table { + width: 100%; +} + +.table-container { + margin: 50px; +} + +.loading-shade { + position: absolute; + top: 0; + left: 0; + bottom: 56px; + right: 0; + //background: rgba(0, 0, 0, 0.15); + z-index: 1; + display: flex; + align-items: center; + justify-content: center; +} + +#refresh_btn { + margin-left: 10px; +} + +.data-table-row { + height: 30px; +} + +.data-table-row:hover { + background: whitesmoke; +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts new file mode 100644 index 0000000..5a7bd23 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts @@ -0,0 +1,84 @@ +import { OnInit, Directive } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { ApiTableService } from '../'; +import { MatTableDataSource } from '@angular/material/table'; +import { MatDialog } from '@angular/material/dialog'; +import { ApiModel, Area } from '../models'; +import { ActivatedRoute, Router } from '@angular/router'; +import { interval } from 'rxjs'; +import { MatSnackBar, MatSnackBarConfig } from '@angular/material/snack-bar'; + +@Directive() +export abstract class TablesComponent implements OnInit { + abstract area: Area; + data_source!: ApiTableService; + table_data!: MatTableDataSource; + + isLoadingResults = true; + + constructor( + public httpClient: HttpClient, + public infoDialog: MatDialog, + public route: ActivatedRoute, + public router: Router, + public snackBar: MatSnackBar + ) { + this.table_data = new MatTableDataSource; + } + + ngOnInit() { + this.data_source = new ApiTableService(this.httpClient, this.area); + this.loadTableData(); + this.route.queryParams.subscribe(params => { + const id = params['id'] + const new_agent = params['new'] + if (id) { + this.show_item_dialog(id); + } + if (new_agent) { + this.show_item_dialog(null); + } + }) + //interval(10000).subscribe(_ => this.loadTableData()); + } + + async loadTableData() { + this.isLoadingResults = true; + //possibly needs try/catch + const data = await this.data_source!.getMany(); + this.isLoadingResults = false; + + if (typeof data.data !== 'string') { + this.table_data.data = data.data + } else { + alert(`Error: ${data}`) + }; + } + + apply_filter(event: Event) { + const filterValue = (event.target as HTMLInputElement).value; + this.table_data.filter = filterValue.trim().toLowerCase(); + } + + deleteItem(id: string) { + if (confirm(`Delete ${id}?`)) { + this.data_source!.delete(id).catch(this.openSnackBar) + } + } + + openSnackBar(message: any, error: boolean = true) { + const msg = JSON.stringify(message) + const _config = (duration: number): MatSnackBarConfig => { + return { + horizontalPosition: 'right', + verticalPosition: 'bottom', + duration + } + } + const cfg = error ? _config(0) : _config(2000) + this.snackBar.open(msg, 'Ok', cfg); + } + + abstract displayedColumns: string[]; + abstract show_item_dialog(id: string | null): void; +} \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/app/core/utils.ts b/bin/u_panel/src/server/fe/src/app/core/utils.ts new file mode 100644 index 0000000..bc6c422 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/app/core/utils.ts @@ -0,0 +1,3 @@ +export function epochToStr(epoch: number): string { + return new Date(epoch * 1000).toLocaleString('en-GB') +} diff --git a/bin/u_panel/src/server/fe/src/environments/environment.prod.ts b/bin/u_panel/src/server/fe/src/environments/environment.prod.ts new file mode 100644 index 0000000..cc43895 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/environments/environment.prod.ts @@ -0,0 +1,4 @@ +export const environment = { + production: true, + server: "", +}; diff --git a/bin/u_panel/src/server/fe/src/environments/environment.ts b/bin/u_panel/src/server/fe/src/environments/environment.ts new file mode 100644 index 0000000..1b3b824 --- /dev/null +++ b/bin/u_panel/src/server/fe/src/environments/environment.ts @@ -0,0 +1,17 @@ +// This file can be replaced during build by using the `fileReplacements` array. +// `ng build` replaces `environment.ts` with `environment.prod.ts`. +// The list of file replacements can be found in `angular.json`. + +export const environment = { + production: false, + server: "http://127.0.0.1:8080", +}; + +/* + * For easier debugging in development mode, you can import the following file + * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`. + * + * This import should be commented out in production mode because it will have a negative impact + * on performance if an error is thrown. + */ +// import 'zone.js/plugins/zone-error'; // Included with Angular CLI. diff --git a/bin/u_panel/src/server/fe/src/favicon.ico b/bin/u_panel/src/server/fe/src/favicon.ico new file mode 100644 index 0000000..997406a Binary files /dev/null and b/bin/u_panel/src/server/fe/src/favicon.ico differ diff --git a/bin/u_panel/src/server/fe/src/index.html b/bin/u_panel/src/server/fe/src/index.html new file mode 100644 index 0000000..e299aec --- /dev/null +++ b/bin/u_panel/src/server/fe/src/index.html @@ -0,0 +1,18 @@ + + + + + + Fe + + + + + + + + + + + + \ No newline at end of file diff --git a/bin/u_panel/src/server/fe/src/main.ts b/bin/u_panel/src/server/fe/src/main.ts new file mode 100644 index 0000000..c7b673c --- /dev/null +++ b/bin/u_panel/src/server/fe/src/main.ts @@ -0,0 +1,12 @@ +import { enableProdMode } from '@angular/core'; +import { platformBrowserDynamic } from '@angular/platform-browser-dynamic'; + +import { AppModule } from './app/app.module'; +import { environment } from './environments/environment'; + +if (environment.production) { + enableProdMode(); +} + +platformBrowserDynamic().bootstrapModule(AppModule) + .catch(err => console.error(err)); diff --git a/bin/u_panel/src/server/fe/src/polyfills.ts b/bin/u_panel/src/server/fe/src/polyfills.ts new file mode 100644 index 0000000..429bb9e --- /dev/null +++ b/bin/u_panel/src/server/fe/src/polyfills.ts @@ -0,0 +1,53 @@ +/** + * This file includes polyfills needed by Angular and is loaded before the app. + * You can add your own extra polyfills to this file. + * + * This file is divided into 2 sections: + * 1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers. + * 2. Application imports. Files imported after ZoneJS that should be loaded before your main + * file. + * + * The current setup is for so-called "evergreen" browsers; the last versions of browsers that + * automatically update themselves. This includes recent versions of Safari, Chrome (including + * Opera), Edge on the desktop, and iOS and Chrome on mobile. + * + * Learn more in https://angular.io/guide/browser-support + */ + +/*************************************************************************************************** + * BROWSER POLYFILLS + */ + +/** + * By default, zone.js will patch all possible macroTask and DomEvents + * user can disable parts of macroTask/DomEvents patch by setting following flags + * because those flags need to be set before `zone.js` being loaded, and webpack + * will put import in the top of bundle, so user need to create a separate file + * in this directory (for example: zone-flags.ts), and put the following flags + * into that file, and then add the following code before importing zone.js. + * import './zone-flags'; + * + * The flags allowed in zone-flags.ts are listed here. + * + * The following flags will work for all browsers. + * + * (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame + * (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick + * (window as any).__zone_symbol__UNPATCHED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames + * + * in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js + * with the following flag, it will bypass `zone.js` patch for IE/Edge + * + * (window as any).__Zone_enable_cross_context_check = true; + * + */ + +/*************************************************************************************************** + * Zone JS is required by default for Angular itself. + */ +import 'zone.js'; // Included with Angular CLI. + + +/*************************************************************************************************** + * APPLICATION IMPORTS + */ diff --git a/bin/u_panel/src/server/fe/src/styles.less b/bin/u_panel/src/server/fe/src/styles.less new file mode 100644 index 0000000..7e7239a --- /dev/null +++ b/bin/u_panel/src/server/fe/src/styles.less @@ -0,0 +1,4 @@ +/* You can add global styles to this file, and also import other style files */ + +html, body { height: 100%; } +body { margin: 0; font-family: Roboto, "Helvetica Neue", sans-serif; } diff --git a/bin/u_panel/src/server/fe/src/test.ts b/bin/u_panel/src/server/fe/src/test.ts new file mode 100644 index 0000000..00025da --- /dev/null +++ b/bin/u_panel/src/server/fe/src/test.ts @@ -0,0 +1,26 @@ +// This file is required by karma.conf.js and loads recursively all the .spec and framework files + +import 'zone.js/testing'; +import { getTestBed } from '@angular/core/testing'; +import { + BrowserDynamicTestingModule, + platformBrowserDynamicTesting +} from '@angular/platform-browser-dynamic/testing'; + +declare const require: { + context(path: string, deep?: boolean, filter?: RegExp): { + (id: string): T; + keys(): string[]; + }; +}; + +// First, initialize the Angular testing environment. +getTestBed().initTestEnvironment( + BrowserDynamicTestingModule, + platformBrowserDynamicTesting(), +); + +// Then we find all the tests. +const context = require.context('./', true, /\.spec\.ts$/); +// And load the modules. +context.keys().map(context); diff --git a/bin/u_panel/src/server/fe/tsconfig.app.json b/bin/u_panel/src/server/fe/tsconfig.app.json new file mode 100644 index 0000000..82d91dc --- /dev/null +++ b/bin/u_panel/src/server/fe/tsconfig.app.json @@ -0,0 +1,15 @@ +/* To learn more about this file see: https://angular.io/config/tsconfig. */ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./out-tsc/app", + "types": [] + }, + "files": [ + "src/main.ts", + "src/polyfills.ts" + ], + "include": [ + "src/**/*.d.ts" + ] +} diff --git a/bin/u_panel/src/server/fe/tsconfig.json b/bin/u_panel/src/server/fe/tsconfig.json new file mode 100644 index 0000000..f531992 --- /dev/null +++ b/bin/u_panel/src/server/fe/tsconfig.json @@ -0,0 +1,32 @@ +/* To learn more about this file see: https://angular.io/config/tsconfig. */ +{ + "compileOnSave": false, + "compilerOptions": { + "baseUrl": "./", + "outDir": "./dist/out-tsc", + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "sourceMap": true, + "declaration": false, + "downlevelIteration": true, + "experimentalDecorators": true, + "moduleResolution": "node", + "importHelpers": true, + "target": "es2017", + "module": "es2020", + "lib": [ + "es2020", + "dom" + ] + }, + "angularCompilerOptions": { + "enableI18nLegacyMessageIdFormat": false, + "strictInjectionParameters": true, + "strictInputAccessModifiers": true, + "strictTemplates": true + } +} diff --git a/bin/u_panel/src/server/fe/tsconfig.spec.json b/bin/u_panel/src/server/fe/tsconfig.spec.json new file mode 100644 index 0000000..092345b --- /dev/null +++ b/bin/u_panel/src/server/fe/tsconfig.spec.json @@ -0,0 +1,18 @@ +/* To learn more about this file see: https://angular.io/config/tsconfig. */ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./out-tsc/spec", + "types": [ + "jasmine" + ] + }, + "files": [ + "src/test.ts", + "src/polyfills.ts" + ], + "include": [ + "src/**/*.spec.ts", + "src/**/*.d.ts" + ] +} diff --git a/bin/u_panel/src/server/mod.rs b/bin/u_panel/src/server/mod.rs new file mode 100644 index 0000000..897be93 --- /dev/null +++ b/bin/u_panel/src/server/mod.rs @@ -0,0 +1,98 @@ +mod error; + +use crate::{process_cmd, Args}; +use actix_cors::Cors; +use actix_web::{get, middleware::Logger, post, web, App, HttpResponse, HttpServer, Responder}; +use error::Error; +use futures_util::StreamExt; +use rust_embed::RustEmbed; +use std::borrow::Cow; +use structopt::StructOpt; +use u_lib::{api::ClientHandler, unwrap_enum}; + +#[derive(RustEmbed)] +#[folder = "./src/server/fe/dist/fe/"] +struct Files; + +impl Files { + pub fn get_static(path: impl AsRef) -> Option<&'static [u8]> { + let file = Self::get(path.as_ref())?.data; + Some(unwrap_enum!(file, Cow::Borrowed)) + } +} + +async fn spa_main() -> impl Responder { + let index = Files::get_static("index.html").unwrap(); + HttpResponse::Ok().body(index) +} + +#[get("/core/{path}")] +async fn resources_adapter(path: web::Path<(String,)>) -> impl Responder { + let path = path.into_inner().0; + let mimetype = mime_guess::from_path(&path).first_or_octet_stream(); + + match Files::get_static(path) { + Some(data) => HttpResponse::Ok() + .content_type(mimetype.to_string()) + .body(data), + None => HttpResponse::NotFound().finish(), + } +} + +#[post("/cmd/")] +async fn send_cmd( + mut body: web::Payload, + client: web::Data, +) -> Result { + let mut bytes = web::BytesMut::new(); + + while let Some(item) = body.next().await { + bytes.extend_from_slice( + &item.map_err(|e| Error::JustError(format!("payload loading failure: {e}")))?, + ); + } + + let cmd = String::from_utf8(bytes.to_vec()) + .map_err(|_| Error::JustError("cmd contains non-utf8 data".to_string()))?; + let mut cmd = shlex::split(&cmd).ok_or(Error::JustError("argparse failed".to_string()))?; + + info!("cmd: {:?}", cmd); + cmd.insert(0, String::from("u_panel")); + + let parsed_cmd = Args::from_iter_safe(cmd)?; + let result = process_cmd(client.as_ref().clone(), parsed_cmd).await; + let result_string = result.to_string(); + + let response = if result.is_ok() { + HttpResponse::Ok().body(result_string) + } else if result.is_err() { + HttpResponse::BadRequest().body(result_string) + } else { + unreachable!() + }; + + Ok(response) +} + +pub async fn serve(client: ClientHandler) -> anyhow::Result<()> { + info!("Connecting to u_server..."); + client.ping().await?; + + let addr = "127.0.0.1:8080"; + info!("Connected, instanciating u_panel at http://{}", addr); + + HttpServer::new(move || { + App::new() + .wrap(Logger::default()) + .wrap(Cors::permissive()) + .app_data(web::Data::new(client.clone())) + .service(send_cmd) + .service(resources_adapter) + .service(web::resource("/").to(spa_main)) + .service(web::resource("/{_}").to(spa_main)) + }) + .bind(addr)? + .run() + .await?; + Ok(()) +} diff --git a/bin/u_run/Cargo.toml b/bin/u_run/Cargo.toml index 9756ea2..1c3b145 100644 --- a/bin/u_run/Cargo.toml +++ b/bin/u_run/Cargo.toml @@ -7,5 +7,5 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -nix = "0.17" libc = "^0.2" +nix = "0.17" diff --git a/bin/u_server/Cargo.toml b/bin/u_server/Cargo.toml index 043022a..1209d12 100644 --- a/bin/u_server/Cargo.toml +++ b/bin/u_server/Cargo.toml @@ -1,39 +1,27 @@ [package] authors = ["plazmoid "] -edition = "2018" +edition = "2021" name = "u_server" version = "0.1.0" [dependencies] -log = "0.4.11" -simplelog = "0.10" -thiserror = "*" -warp = { version = "0.3.1", features = ["tls"] } -uuid = { version = "0.6.5", features = ["serde", "v4"] } -once_cell = "1.7.2" +anyhow = { workspace = true } +diesel = { version = "1.4.5", features = ["postgres", "uuid"] } hyper = "0.14" -mockall = "0.9.1" -mockall_double = "0.2" +once_cell = "1.7.2" openssl = "*" +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +tokio = { workspace = true, features = ["macros"] } +uuid = { workspace = true, features = ["serde", "v4"] } +u_lib = { path = "../../lib/u_lib", features = ["server"] } +warp = { version = "0.3.1", features = ["tls"] } -[dependencies.diesel] -features = ["postgres", "uuid"] -version = "1.4.5" - -[dependencies.serde] -features = ["derive"] -version = "1.0.114" - -[dependencies.tokio] -features = ["macros"] -version = "1.9" - -[dependencies.u_lib] -path = "../../lib/u_lib" -version = "*" [dev-dependencies] -test-case = "1.1.0" +rstest = "0.12" [lib] name = "u_server_lib" @@ -41,4 +29,4 @@ path = "src/u_server.rs" [[bin]] name = "u_server" -path = "src/main.rs" \ No newline at end of file +path = "src/main.rs" diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index fa39ad4..5bee420 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -1,109 +1,130 @@ +use crate::error::Error as ServerError; use diesel::{pg::PgConnection, prelude::*, result::Error as DslError}; use once_cell::sync::OnceCell; -use std::{ - env, - sync::{Arc, Mutex, MutexGuard}, -}; +use serde::Deserialize; +use std::sync::{Mutex, MutexGuard}; use u_lib::{ models::{schema, Agent, AssignedJob, JobMeta, JobState}, - ULocalError, ULocalResult, + utils::load_env, }; use uuid::Uuid; +type Result = std::result::Result; + pub struct UDB { - pub conn: PgConnection, + conn: PgConnection, } -static DB: OnceCell>> = OnceCell::new(); +static DB: OnceCell> = OnceCell::new(); + +#[derive(Deserialize)] +struct DBEnv { + db_host: String, + db_name: String, + db_user: String, + db_password: String, +} -#[cfg_attr(test, automock)] impl UDB { - pub fn lock_db() -> MutexGuard<'static, UDB> { + pub fn lock_db() -> MutexGuard<'static, Self> { DB.get_or_init(|| { - let _getenv = |v| env::var(v).unwrap(); - let db_host = _getenv("DB_HOST"); - let db_name = _getenv("DB_NAME"); - let db_user = _getenv("DB_USER"); - let db_password = _getenv("DB_PASSWORD"); + let env = load_env::().unwrap(); let db_url = format!( "postgres://{}:{}@{}/{}", - db_user, db_password, db_host, db_name + env.db_user, env.db_password, env.db_host, env.db_name ); - let conn = PgConnection::establish(&db_url).unwrap(); - let instance = UDB { conn }; - Arc::new(Mutex::new(instance)) + let instance = UDB { + conn: PgConnection::establish(&db_url).unwrap(), + }; + Mutex::new(instance) }) .lock() .unwrap() } - pub fn insert_jobs(&self, job_metas: &[JobMeta]) -> ULocalResult<()> { + pub fn insert_jobs(&self, job_metas: &[JobMeta]) -> Result> { use schema::jobs; + diesel::insert_into(jobs::table) .values(job_metas) - .execute(&self.conn)?; - Ok(()) + .get_results(&self.conn) + .map(|rows| rows.iter().map(|job: &JobMeta| job.id).collect()) + .map_err(with_err_ctx("Can't insert jobs")) } - pub fn get_jobs(&self, uid: Option) -> ULocalResult> { + pub fn get_jobs(&self, ouid: Option) -> Result> { use schema::jobs; - let result = if uid.is_some() { - jobs::table - .filter(jobs::id.eq(uid.unwrap())) - .get_results::(&self.conn)? - } else { - jobs::table.load::(&self.conn)? - }; - Ok(result) + + match ouid { + Some(uid) => jobs::table + .filter(jobs::id.eq(uid)) + .get_results::(&self.conn), + None => jobs::table.load::(&self.conn), + } + .map_err(with_err_ctx("Can't get exact jobs")) } - pub fn find_job_by_alias(&self, alias: &str) -> ULocalResult { + pub fn find_job_by_alias(&self, alias: &str) -> Result> { use schema::jobs; + let result = jobs::table .filter(jobs::alias.eq(alias)) - .first::(&self.conn)?; + .first::(&self.conn) + .optional() + .map_err(with_err_ctx(format!("Can't find job by alias {alias}")))?; Ok(result) } - pub fn insert_agent(&self, agent: &Agent) -> ULocalResult<()> { + pub fn insert_agent(&self, agent: &Agent) -> Result<()> { use schema::agents; + diesel::insert_into(agents::table) .values(agent) .on_conflict(agents::id) .do_update() .set(agent) - .execute(&self.conn)?; + .execute(&self.conn) + .map_err(with_err_ctx(format!("Can't insert agent {agent:x?}")))?; Ok(()) } - pub fn get_agents(&self, uid: Option) -> ULocalResult> { + pub fn insert_result(&self, result: &AssignedJob) -> Result<()> { + use schema::results; + + diesel::insert_into(results::table) + .values(result) + .execute(&self.conn) + .map_err(with_err_ctx(format!("Can't insert result {result:x?}")))?; + Ok(()) + } + + pub fn get_agents(&self, ouid: Option) -> Result> { use schema::agents; - let result = if uid.is_some() { - agents::table - .filter(agents::id.eq(uid.unwrap())) - .load::(&self.conn)? - } else { - agents::table.load::(&self.conn)? - }; - Ok(result) + + match ouid { + Some(uid) => agents::table + .filter(agents::id.eq(uid)) + .load::(&self.conn), + None => agents::table.load::(&self.conn), + } + .map_err(with_err_ctx(format!("Can't get agent(s) {ouid:?}"))) } - pub fn update_job_status(&self, uid: Uuid, status: JobState) -> ULocalResult<()> { + pub fn update_job_status(&self, uid: Uuid, status: JobState) -> Result<()> { use schema::results; + diesel::update(results::table) .filter(results::id.eq(uid)) .set(results::state.eq(status)) - .execute(&self.conn)?; + .execute(&self.conn) + .map_err(with_err_ctx(format!("Can't update status of job {uid}")))?; Ok(()) } //TODO: filters possibly could work in a wrong way, check - pub fn get_exact_jobs( - &self, - uid: Option, - personal: bool, - ) -> ULocalResult> { + pub fn get_exact_jobs(&self, uid: Option, personal: bool) -> Result> { use schema::results; + let mut q = results::table.into_boxed(); /*if uid.is_some() { q = q.filter(results::agent_id.eq(uid.unwrap())) @@ -120,115 +141,101 @@ impl UDB { .or_filter(results::job_id.eq(uid.unwrap())) .or_filter(results::id.eq(uid.unwrap())) } - let result = q.load::(&self.conn)?; + let result = q + .load::(&self.conn) + .map_err(with_err_ctx("Can't get exact jobs"))?; Ok(result) } - pub fn set_jobs_for_agent( - &self, - agent_uid: &Uuid, - job_uids: &[Uuid], - ) -> ULocalResult> { - use schema::{agents::dsl::agents, jobs::dsl::jobs, results}; - if let Err(DslError::NotFound) = agents.find(agent_uid).first::(&self.conn) { - return Err(ULocalError::NotFound(agent_uid.to_string())); - } - let not_found_jobs = job_uids - .iter() - .filter_map(|job_uid| { - if let Err(DslError::NotFound) = jobs.find(job_uid).first::(&self.conn) { - Some(job_uid.to_string()) - } else { - None - } - }) - .collect::>(); - if not_found_jobs.len() > 0 { - return Err(ULocalError::NotFound(not_found_jobs.join(", "))); - } + pub fn set_jobs_for_agent(&self, agent_uid: &Uuid, job_uids: &[Uuid]) -> Result> { + use schema::results; + let job_requests = job_uids .iter() - .map(|job_uid| { - info!("set_jobs_for_agent: set {} for {}", job_uid, agent_uid); - AssignedJob { - job_id: *job_uid, - agent_id: *agent_uid, - ..Default::default() - } + .map(|job_uid| AssignedJob { + job_id: *job_uid, + agent_id: *agent_uid, + ..Default::default() }) .collect::>(); + diesel::insert_into(results::table) .values(&job_requests) - .execute(&self.conn)?; - let assigned_uids = job_requests.iter().map(|aj| aj.id).collect(); - Ok(assigned_uids) + .execute(&self.conn) + .map_err(with_err_ctx(format!( + "Can't setup jobs {job_uids:?} for agent {agent_uid:?}" + )))?; + + Ok(job_requests.iter().map(|aj| aj.id).collect()) } - pub fn del_jobs(&self, uids: &Vec) -> ULocalResult { + pub fn del_jobs(&self, uids: &[Uuid]) -> Result { use schema::jobs; + let mut affected = 0; for &uid in uids { let deleted = diesel::delete(jobs::table) .filter(jobs::id.eq(uid)) - .execute(&self.conn)?; + .execute(&self.conn) + .map_err(with_err_ctx("Can't delete jobs"))?; affected += deleted; } Ok(affected) } - pub fn del_results(&self, uids: &Vec) -> ULocalResult { + pub fn del_results(&self, uids: &[Uuid]) -> Result { use schema::results; + let mut affected = 0; for &uid in uids { let deleted = diesel::delete(results::table) .filter(results::id.eq(uid)) - .execute(&self.conn)?; + .execute(&self.conn) + .map_err(with_err_ctx("Can't delete results"))?; affected += deleted; } Ok(affected) } - pub fn del_agents(&self, uids: &Vec) -> ULocalResult { + pub fn del_agents(&self, uids: &[Uuid]) -> Result { use schema::agents; + let mut affected = 0; for &uid in uids { let deleted = diesel::delete(agents::table) .filter(agents::id.eq(uid)) - .execute(&self.conn)?; + .execute(&self.conn) + .map_err(with_err_ctx("Can't delete agents"))?; affected += deleted; } Ok(affected) } -} -/* -#[cfg(test)] -mod tests { - use super::*; - - fn setup_db() -> Storage { - return UDB::new().unwrap(); - } - - #[tokio::test] - async fn test_add_agent() { - let db = setup_db(); - let agent = IAgent { - alias: None, - id: "000-000".to_string(), - hostname: "test".to_string(), - is_root: false, - is_root_allowed: false, - platform: "linux".to_string(), - status: None, - token: None, - username: "test".to_string() - }; - db.lock().unwrap().new_agent(agent).unwrap(); - let result = db.lock().unwrap().get_agents().unwrap(); - assert_eq!( - result[0].username, - "test".to_string() - ) + + pub fn update_agent(&self, agent: &Agent) -> Result<()> { + agent + .save_changes::(&self.conn) + .map_err(with_err_ctx(format!("Can't update agent {agent:x?}")))?; + Ok(()) + } + + pub fn update_job(&self, job: &JobMeta) -> Result<()> { + job.save_changes::(&self.conn) + .map_err(with_err_ctx(format!("Can't update job {job:x?}")))?; + Ok(()) } + + pub fn update_result(&self, result: &AssignedJob) -> Result<()> { + debug!( + "updating result: id = {}, job_id = {}, agent_id = {}", + result.id, result.job_id, result.agent_id + ); + result + .save_changes::(&self.conn) + .map_err(with_err_ctx(format!("Can't update result {result:x?}")))?; + Ok(()) + } +} + +fn with_err_ctx(msg: impl AsRef) -> impl Fn(DslError) -> ServerError { + move |err| ServerError::DBErrorCtx(format!("{}, reason: {err}", msg.as_ref())) } -*/ diff --git a/bin/u_server/src/error.rs b/bin/u_server/src/error.rs new file mode 100644 index 0000000..6f07b72 --- /dev/null +++ b/bin/u_server/src/error.rs @@ -0,0 +1,59 @@ +use diesel::result::Error as DslError; +use thiserror::Error; +use warp::{ + http::StatusCode, + reject::Reject, + reply::{with_status, Response}, + Reply, +}; + +#[derive(Error, Debug)] +pub enum Error { + #[error("Error processing {0}")] + ProcessingError(String), + + #[error(transparent)] + DBError(#[from] DslError), + + #[error("DB error: {0}")] + DBErrorCtx(String), + + #[error("General error: {0}")] + Other(String), +} + +impl Reject for Error {} + +pub struct RejResponse { + message: String, + status: StatusCode, +} + +impl RejResponse { + pub fn not_found(msg: impl Into) -> Self { + Self { + message: msg.into(), + status: StatusCode::NOT_FOUND, + } + } + + pub fn bad_request(msg: impl Into) -> Self { + Self { + message: msg.into(), + status: StatusCode::BAD_REQUEST, + } + } + + pub fn internal() -> Self { + Self { + message: "INTERNAL_SERVER_ERROR".to_string(), + status: StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +impl Reply for RejResponse { + fn into_response(self) -> Response { + with_status(self.message, self.status).into_response() + } +} diff --git a/bin/u_server/src/filters.rs b/bin/u_server/src/filters.rs deleted file mode 100644 index 59527a0..0000000 --- a/bin/u_server/src/filters.rs +++ /dev/null @@ -1,87 +0,0 @@ -use crate::handlers::Endpoints; -use serde::de::DeserializeOwned; -use std::env; -use u_lib::{ - messaging::{AsMsg, BaseMessage}, - models::*, -}; -use uuid::Uuid; -use warp::{body, Filter, Rejection, Reply}; - -fn get_content() -> impl Filter,), Error = Rejection> + Clone -where - M: AsMsg + Sync + Send + DeserializeOwned + 'static, -{ - body::content_length_limit(1024 * 64).and(body::json::>()) -} - -pub fn make_filters() -> impl Filter + Clone { - let infallible_none = |_| async { Ok::<(Option,), std::convert::Infallible>((None,)) }; - - let get_agents = warp::get() - .and(warp::path("get_agents")) - .and( - warp::path::param::() - .map(Some) - .or_else(infallible_none), - ) - .and_then(Endpoints::get_agents); - - let upload_jobs = warp::post() - .and(warp::path("upload_jobs")) - .and(get_content::>()) - .and_then(Endpoints::upload_jobs); - - let get_jobs = warp::get() - .and(warp::path("get_jobs")) - .and( - warp::path::param::() - .map(Some) - .or_else(infallible_none), - ) - .and_then(Endpoints::get_jobs); - - let get_agent_jobs = warp::get() - .and(warp::path("get_agent_jobs")) - .and( - warp::path::param::() - .map(Some) - .or_else(infallible_none), - ) - .and_then(|uid| Endpoints::get_agent_jobs(uid)); - - let get_personal_jobs = warp::get() - .and(warp::path("get_personal_jobs")) - .and(warp::path::param::().map(Some)) - .and_then(|uid| Endpoints::get_personal_jobs(uid)); - - let del = warp::get() - .and(warp::path("del")) - .and(warp::path::param::()) - .and_then(Endpoints::del); - - let set_jobs = warp::post() - .and(warp::path("set_jobs")) - .and(warp::path::param::()) - .and(get_content::>()) - .and_then(Endpoints::set_jobs); - - let report = warp::post() - .and(warp::path("report")) - .and(get_content::>().and_then(Endpoints::report)); - - let auth_token = format!("Bearer {}", env::var("ADMIN_AUTH_TOKEN").unwrap()).into_boxed_str(); - let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); - - let auth_zone = (get_agents - .or(get_jobs) - .or(upload_jobs) - .or(del) - .or(set_jobs) - .or(get_agent_jobs)) - .and(auth_header); - - let agent_zone = get_jobs.clone().or(get_personal_jobs).or(report); - - auth_zone.or(agent_zone) -} diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index db8ff4c..c0907c9 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -1,171 +1,162 @@ use crate::db::UDB; -use diesel::SaveChangesDsl; -use hyper::Body; -use serde::Serialize; +use crate::error::Error; use u_lib::{ - messaging::{AsMsg, BaseMessage}, - models::{Agent, AgentState, AssignedJob, ExecResult, JobMeta, JobState}, - ULocalError, + messaging::{AsMsg, BaseMessage, Reportable}, + models::*, + utils::OneOrVec, }; use uuid::Uuid; -use warp::{ - http::{Response, StatusCode}, - Rejection, Reply, -}; - -pub fn build_response>(code: StatusCode, body: S) -> Response { - Response::builder().status(code).body(body.into()).unwrap() -} - -pub fn build_ok>(body: S) -> Response { - build_response(StatusCode::OK, body) -} - -pub fn build_err(body: S) -> Response { - build_response(StatusCode::BAD_REQUEST, body.to_string()) -} +use warp::Rejection; -pub fn build_message(m: M) -> Response { - warp::reply::json(&m.as_message()).into_response() -} +type EndpResult = Result; pub struct Endpoints; -#[cfg_attr(test, automock)] impl Endpoints { - pub async fn add_agent(msg: Agent) -> Result, Rejection> { - info!("hnd: add_agent"); - UDB::lock_db() - .insert_agent(&msg) - .map(|_| build_ok("")) - .or_else(|e| Ok(build_err(e))) + pub async fn add_agent(msg: Agent) -> EndpResult<()> { + UDB::lock_db().insert_agent(&msg).map_err(From::from) } - pub async fn get_agents(uid: Option) -> Result, Rejection> { - info!("hnd: get_agents"); - UDB::lock_db() - .get_agents(uid) - .map(|m| build_message(m)) - .or_else(|e| Ok(build_err(e))) + pub async fn get_agents(uid: Option) -> EndpResult> { + UDB::lock_db().get_agents(uid).map_err(From::from) } - pub async fn get_jobs(uid: Option) -> Result, Rejection> { - info!("hnd: get_jobs"); - UDB::lock_db() - .get_jobs(uid) - .map(|m| build_message(m)) - .or_else(|e| Ok(build_err(e))) + pub async fn get_jobs(uid: Option) -> EndpResult> { + UDB::lock_db().get_jobs(uid).map_err(From::from) } - pub async fn get_agent_jobs(uid: Option) -> Result, Rejection> { - info!("hnd: get_agent_jobs"); + pub async fn get_agent_jobs(uid: Option) -> EndpResult> { UDB::lock_db() .get_exact_jobs(uid, false) - .map(|m| build_message(m)) - .or_else(|e| Ok(build_err(e))) + .map_err(From::from) } - pub async fn get_personal_jobs(uid: Option) -> Result, Rejection> { - info!("hnd: get_personal_jobs"); - let agents = UDB::lock_db().get_agents(uid).unwrap(); - if agents.len() == 0 { - let db = UDB::lock_db(); - db.insert_agent(&Agent::with_id(uid.unwrap())).unwrap(); - let job = db.find_job_by_alias("agent_hello").unwrap(); - if let Err(e) = db.set_jobs_for_agent(&uid.unwrap(), &[job.id]) { - return Ok(build_err(e)); - } + pub async fn get_personal_jobs(uid: Uuid) -> EndpResult> { + let db = UDB::lock_db(); + let mut agents = db.get_agents(Some(uid))?; + if agents.is_empty() { + let new_agent = Agent::with_id(uid); + db.insert_agent(&new_agent)?; + let job = db + .find_job_by_alias("agent_hello")? + .expect("agent_hello job not found"); + db.set_jobs_for_agent(&uid, &[job.id])?; + } else { + let mut agent = agents.pop().unwrap(); + agent.touch(); + db.update_agent(&agent)?; } - let result = UDB::lock_db().get_exact_jobs(uid, true); - match result { - Ok(r) => { - let db = UDB::lock_db(); - for j in r.iter() { - db.update_job_status(j.id, JobState::Running).unwrap(); - } - Ok(build_message(r)) - } - Err(e) => Ok(build_err(e)), + let result = db.get_exact_jobs(Some(uid), true)?; + + for j in result.iter() { + db.update_job_status(j.id, JobState::Running)?; } + Ok(result) } - pub async fn upload_jobs( - msg: BaseMessage<'static, Vec>, - ) -> Result, Rejection> { - info!("hnd: upload_jobs"); + pub async fn upload_jobs(msg: BaseMessage<'static, Vec>) -> EndpResult> { UDB::lock_db() .insert_jobs(&msg.into_inner()) - .map(|_| build_ok("")) - .or_else(|e| Ok(build_err(e))) + .map_err(From::from) } - pub async fn del(uid: Uuid) -> Result, Rejection> { - info!("hnd: del"); + pub async fn del(uid: Uuid) -> EndpResult { let db = UDB::lock_db(); let del_fns = &[UDB::del_agents, UDB::del_jobs, UDB::del_results]; for del_fn in del_fns { - let affected = del_fn(&db, &vec![uid]).unwrap(); + let affected = del_fn(&db, &[uid])?; if affected > 0 { - return Ok(build_message(affected as i32)); + return Ok(affected); } } - Ok(build_message(0)) + Ok(0) } pub async fn set_jobs( agent_uid: Uuid, msg: BaseMessage<'static, Vec>, - ) -> Result, Rejection> { - info!("hnd: set_jobs_by_alias, agent: {}", agent_uid); - let jobs: Result, ULocalError> = msg - .into_inner() + ) -> EndpResult> { + msg.into_inner() .into_iter() .map(|ident| { - info!("hnd: set_jobs_by_alias, job: {}", ident); - Uuid::parse_str(&ident) - .or_else(|_| UDB::lock_db().find_job_by_alias(&ident).map(|j| j.id)) + Uuid::parse_str(&ident).or_else(|_| { + let job_from_db = UDB::lock_db().find_job_by_alias(&ident); + match job_from_db { + Ok(job) => match job { + Some(j) => Ok(j.id), + None => Err(Error::ProcessingError(format!("unknown ident {ident}"))), + }, + Err(e) => Err(e), + } + }) }) - .collect(); - match jobs { - Ok(j) => UDB::lock_db() - .set_jobs_for_agent(&agent_uid, &j) - .map(|assigned_uids| build_message(assigned_uids)) - .or_else(|e| Ok(build_err(e))), - Err(e) => Ok(build_err(e)), - } + .collect::, Error>>() + .and_then(|j| UDB::lock_db().set_jobs_for_agent(&agent_uid, &j)) + .map_err(From::from) } - pub async fn report( - msg: BaseMessage<'static, Vec>, - ) -> Result, Rejection> { - info!("hnd: report"); + pub async fn report + AsMsg + 'static>( + msg: BaseMessage<'static, Data>, + ) -> EndpResult<()> { let id = msg.id; - let mut failed = vec![]; - for entry in msg.into_inner() { + for entry in msg.into_inner().into_vec() { match entry { - ExecResult::Assigned(res) => { - if id != res.agent_id { + Reportable::Assigned(mut result) => { + let result_agent_id = &result.agent_id; + if id != *result_agent_id { + warn!("Ids are not equal! actual id: {id}, id from job: {result_agent_id}"); continue; } - let db = UDB::lock_db(); - if let Err(e) = res - .save_changes::(&db.conn) - .map_err(ULocalError::from) - { - failed.push(e.to_string()) + result.state = JobState::Finished; + result.touch(); + match result.exec_type { + JobType::Init => match &result.result { + Some(rbytes) => { + let mut agent: Agent = match serde_json::from_slice(&rbytes) { + Ok(a) => a, + Err(e) => { + warn!("Error deserializing agent from {id}: {e}"); + continue; + } + }; + agent.state = AgentState::Active; + Self::add_agent(agent).await?; + } + None => warn!("Empty agent data"), + }, + JobType::Shell => (), + JobType::Terminate => todo!(), + JobType::Update => todo!(), } + UDB::lock_db().update_result(&result)?; } - ExecResult::Agent(mut a) => { - a.state = AgentState::Active; - Self::add_agent(a).await?; + Reportable::Error(e) => { + warn!("{} reported an error: {}", id, e); } - ExecResult::Dummy => (), + Reportable::Dummy => (), } } - if failed.len() > 0 { - let err_msg = ULocalError::ProcessingError(failed.join(", ")); - return Ok(build_err(err_msg)); - } - Ok(build_ok("")) + Ok(()) + } + + pub async fn update_agent(agent: BaseMessage<'static, Agent>) -> EndpResult<()> { + UDB::lock_db().update_agent(&agent.into_inner())?; + Ok(()) + } + + pub async fn update_job(job: BaseMessage<'static, JobMeta>) -> EndpResult<()> { + UDB::lock_db().update_job(&job.into_inner())?; + Ok(()) + } + + pub async fn update_assigned_job( + assigned: BaseMessage<'static, AssignedJob>, + ) -> EndpResult<()> { + UDB::lock_db().update_result(&assigned.into_inner())?; + Ok(()) + } + + pub async fn download(_file_uid: String) -> EndpResult> { + todo!() } } diff --git a/bin/u_server/src/main.rs b/bin/u_server/src/main.rs index 05ec4e0..77877ce 100644 --- a/bin/u_server/src/main.rs +++ b/bin/u_server/src/main.rs @@ -1,6 +1,11 @@ use u_server_lib::serve; +#[macro_use] +extern crate tracing; + #[tokio::main] async fn main() { - serve().await; + if let Err(e) = serve().await { + error!("U_SERVER error: {}", e); + } } diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index cbba6b1..4a6d2db 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -1,117 +1,267 @@ #[macro_use] -extern crate log; +extern crate tracing; +#[cfg(test)] #[macro_use] -extern crate mockall; -#[macro_use] -extern crate mockall_double; +extern crate rstest; -// because of linking errors +// due to linking errors extern crate openssl; -#[macro_use] +// don't touch anything extern crate diesel; -// +// in this block + mod db; -mod filters; +mod error; mod handlers; -use db::UDB; -use filters::make_filters; -use u_lib::{config::MASTER_PORT, models::*, utils::init_env}; -use warp::Filter; +use error::{Error as ServerError, RejResponse}; +use serde::{de::DeserializeOwned, Deserialize}; +use std::{convert::Infallible, path::PathBuf}; +use u_lib::{ + config::MASTER_PORT, + logging::init_logger, + messaging::{AsMsg, BaseMessage, Reportable}, + models::*, + utils::load_env, +}; +use uuid::Uuid; +use warp::{ + body, + log::{custom, Info}, + reply::{json, reply, Json, Response}, + Filter, Rejection, Reply, +}; -const LOGFILE: &str = "u_server.log"; +use crate::db::UDB; +use crate::handlers::Endpoints; -fn prefill_jobs() { - let agent_hello = JobMeta::builder() - .with_type(misc::JobType::Manage) - .with_alias("agent_hello") - .build() - .unwrap(); - UDB::lock_db().insert_jobs(&[agent_hello]).ok(); +#[derive(Deserialize)] +struct ServEnv { + admin_auth_token: String, } -fn init_logger() { - use simplelog::*; - use std::fs::OpenOptions; - let log_cfg = ConfigBuilder::new() - .set_time_format_str("%x %X") - .set_time_to_local(true) - .build(); - let logfile = OpenOptions::new() - .append(true) - .create(true) - .open(LOGFILE) - .unwrap(); - let level = LevelFilter::Info; - let loggers = vec![ - WriteLogger::new(level, log_cfg.clone(), logfile) as Box, - TermLogger::new(level, log_cfg, TerminalMode::Stderr, ColorChoice::Auto), - ]; - CombinedLogger::init(loggers).unwrap(); +fn get_content() -> impl Filter,), Error = Rejection> + Clone +where + M: AsMsg + Sync + Send + DeserializeOwned + 'static, +{ + body::content_length_limit(1024 * 64).and(body::json::>()) +} + +fn into_message(msg: M) -> Json { + json(&msg.as_message()) +} + +pub fn init_endpoints( + auth_token: &str, +) -> impl Filter + Clone { + let path = |p: &'static str| warp::post().and(warp::path(p)); + let infallible_none = |_| async { Ok::<_, Infallible>((None::,)) }; + + let get_agents = path("get_agents") + .and( + warp::path::param::() + .map(Some) + .or_else(infallible_none), + ) + .and_then(Endpoints::get_agents) + .map(into_message); + + let upload_jobs = path("upload_jobs") + .and(get_content::>()) + .and_then(Endpoints::upload_jobs) + .map(into_message); + + let get_jobs = path("get_jobs") + .and( + warp::path::param::() + .map(Some) + .or_else(infallible_none), + ) + .and_then(Endpoints::get_jobs) + .map(into_message); + + let get_agent_jobs = path("get_agent_jobs") + .and( + warp::path::param::() + .map(Some) + .or_else(infallible_none), + ) + .and_then(Endpoints::get_agent_jobs) + .map(into_message); + + let get_personal_jobs = path("get_personal_jobs") + .and(warp::path::param::()) + .and_then(Endpoints::get_personal_jobs) + .map(into_message); + + let del = path("del") + .and(warp::path::param::()) + .and_then(Endpoints::del) + .map(ok); + + let set_jobs = path("set_jobs") + .and(warp::path::param::()) + .and(get_content::>()) + .and_then(Endpoints::set_jobs) + .map(into_message); + + let report = path("report") + .and(get_content::>()) + .and_then(Endpoints::report) + .map(ok); + + let update_agent = path("update_agent") + .and(get_content::()) + .and_then(Endpoints::update_agent) + .map(ok); + + let update_job = path("update_job") + .and(get_content::()) + .and_then(Endpoints::update_job) + .map(ok); + + let update_assigned_job = path("update_result") + .and(get_content::()) + .and_then(Endpoints::update_assigned_job) + .map(ok); + + let download = path("download") + .and(warp::path::param::()) + .and_then(Endpoints::download) + .map(ok); + + let ping = path("ping").map(reply); + + let auth_token = format!("Bearer {auth_token}",).into_boxed_str(); + let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); + + let auth_zone = (get_agents + .or(get_jobs) + .or(upload_jobs) + .or(del) + .or(set_jobs) + .or(get_agent_jobs) + .or(update_agent.or(update_job).or(update_assigned_job)) + .or(download) + .or(ping)) + .and(auth_header); + + let agent_zone = get_jobs.or(get_personal_jobs).or(report).or(download); + + auth_zone.or(agent_zone) } -fn init_all() { - init_logger(); - init_env(); - prefill_jobs(); +pub fn preload_jobs() -> Result<(), ServerError> { + let job_alias = "agent_hello"; + let if_job_exists = UDB::lock_db().find_job_by_alias(job_alias)?; + if if_job_exists.is_none() { + let agent_hello = JobMeta::builder() + .with_type(JobType::Init) + .with_alias(job_alias) + .build() + .unwrap(); + UDB::lock_db().insert_jobs(&[agent_hello])?; + } + Ok(()) } -pub async fn serve() { - init_all(); - let routes = make_filters(); - warp::serve(routes.with(warp::log("warp"))) +pub async fn serve() -> Result<(), ServerError> { + init_logger(Some("u_server")); + preload_jobs()?; + + let certs_dir = PathBuf::from("certs"); + let env = load_env::().map_err(|e| ServerError::Other(e.to_string()))?; + let routes = init_endpoints(&env.admin_auth_token) + .recover(handle_rejection) + .with(custom(logger)); + + warp::serve(routes) .tls() - .cert_path("./certs/server.crt") - .key_path("./certs/server.key") - .client_auth_required_path("./certs/ca.crt") + .cert_path(certs_dir.join("server.crt")) + .key_path(certs_dir.join("server.key")) + .client_auth_required_path(certs_dir.join("ca.crt")) .run(([0, 0, 0, 0], MASTER_PORT)) .await; + Ok(()) } +async fn handle_rejection(rej: Rejection) -> Result { + let resp = if let Some(err) = rej.find::() { + error!("{:x?}", err); + RejResponse::bad_request(err.to_string()) + } else if rej.is_not_found() { + RejResponse::not_found("not found placeholder") + } else { + error!("{:?}", rej); + RejResponse::internal() + }; + Ok(resp.into_response()) +} + +fn logger(info: Info<'_>) { + info!(target: "warp", + "{raddr} {agent_uid} \"{path}\" {status}", + raddr = info.remote_addr().unwrap_or(([0, 0, 0, 0], 0).into()), + path = info.path(), + agent_uid = info.user_agent() + .map(|uid: &str| uid.splitn(3, '-') + .take(2) + .collect::() + ) + .unwrap_or_else(|| "NO_AGENT".to_string()), + status = info.status() + ); +} + +fn ok(_: T) -> impl Reply { + reply() +} + +/* #[cfg(test)] mod tests { use super::*; - #[double] use crate::handlers::Endpoints; use handlers::build_ok; - use mockall::predicate::*; - use test_case::test_case; - use u_lib::messaging::{AsMsg, BaseMessage}; + use u_lib::messaging::{AsMsg, BaseMessage, Reportable}; use uuid::Uuid; - use warp::test::request; + use warp::test; - #[test_case(Some(Uuid::new_v4()))] - #[test_case(None => panics)] + #[rstest] + #[case(Some(Uuid::new_v4()))] + #[should_panic] + #[case(None)] #[tokio::test] - async fn test_get_agent_jobs_unauthorized(uid: Option) { - let mock = Endpoints::get_agent_jobs_context(); - mock.expect().with(eq(uid)).returning(|_| Ok(build_ok(""))); - request() + async fn test_get_agent_jobs_unauthorized(#[case] uid: Option) { + let mock = Endpoints::faux(); + when!(mock.get_agent_jobs).then_return(Ok(build_ok(""))); + //mock.expect().with(eq(uid)).returning(|_| Ok(build_ok(""))); + test::request() .path(&format!( "/get_agent_jobs/{}", uid.map(|u| u.simple().to_string()).unwrap_or(String::new()) )) .method("GET") - .filter(&make_filters()) + .filter(&init_filters("")) .await .unwrap(); - mock.checkpoint(); } #[tokio::test] async fn test_report_unauth_successful() { - let mock = Endpoints::report_context(); + let mock = Endpoints::report(); mock.expect() - .withf(|msg: &BaseMessage<'_, Vec>| msg.inner_ref()[0] == ExecResult::Dummy) + .withf(|msg: &BaseMessage<'_, Vec>| msg.inner_ref()[0] == Reportable::Dummy) .returning(|_| Ok(build_ok(""))); - request() + test::request() .path("/report/") .method("POST") - .json(&vec![ExecResult::Dummy].as_message()) - .filter(&make_filters()) + .json(&vec![Reportable::Dummy].as_message()) + .filter(&init_filters("")) .await .unwrap(); mock.checkpoint(); } } +*/ diff --git a/images/integration-tests/tests_runner.Dockerfile b/images/integration-tests/tests_runner.Dockerfile new file mode 100644 index 0000000..cefab17 --- /dev/null +++ b/images/integration-tests/tests_runner.Dockerfile @@ -0,0 +1,5 @@ +FROM rust:1.64 + +RUN rustup target add x86_64-unknown-linux-musl +RUN mkdir -p /tests && chmod 777 /tests +CMD ["sleep", "3600"] \ No newline at end of file diff --git a/integration/images/u_agent.Dockerfile b/images/integration-tests/u_agent.Dockerfile similarity index 100% rename from integration/images/u_agent.Dockerfile rename to images/integration-tests/u_agent.Dockerfile diff --git a/images/integration-tests/u_db.Dockerfile b/images/integration-tests/u_db.Dockerfile new file mode 100644 index 0000000..5f1812f --- /dev/null +++ b/images/integration-tests/u_db.Dockerfile @@ -0,0 +1,17 @@ +FROM postgres:14.5 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt update && apt upgrade -y +RUN apt install -y curl build-essential libpq-dev iproute2 +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal +ENV PATH /root/.cargo/bin:$PATH +RUN rustup target add x86_64-unknown-linux-musl +RUN cargo install diesel_cli --no-default-features --features postgres + +RUN mkdir -p /unki +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 +RUN apt install -y locales locales-all +COPY u_db_entrypoint.sh /unki/ diff --git a/images/integration-tests/u_db_entrypoint.sh b/images/integration-tests/u_db_entrypoint.sh new file mode 100755 index 0000000..f6f560c --- /dev/null +++ b/images/integration-tests/u_db_entrypoint.sh @@ -0,0 +1,7 @@ +set -m + +export DATABASE_URL=postgres://${DB_USER}:${DB_PASSWORD}@127.0.0.1/${DB_NAME} +touch /unki/Cargo.toml +/usr/local/bin/docker-entrypoint.sh postgres & +sleep 10 && diesel setup && diesel migration run +[[ $1 == "svc" ]] && fg %1 \ No newline at end of file diff --git a/images/integration-tests/u_server.Dockerfile b/images/integration-tests/u_server.Dockerfile new file mode 100644 index 0000000..691a618 --- /dev/null +++ b/images/integration-tests/u_server.Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:latest + +RUN apk add iproute2 bash \ No newline at end of file diff --git a/images/musl-libs.Dockerfile b/images/musl-libs.Dockerfile new file mode 100644 index 0000000..c2e0b01 --- /dev/null +++ b/images/musl-libs.Dockerfile @@ -0,0 +1,96 @@ +FROM ubuntu:xenial +LABEL maintainer="Eirik Albrigtsen " + +# Required packages: +# - musl-dev, musl-tools - the musl toolchain +# - curl, g++, make, pkgconf, cmake - for fetching and building third party libs +# - ca-certificates - openssl + curl + peer verification of downloads +# - xutils-dev - for openssl makedepend +# - libssl-dev and libpq-dev - for dynamic linking during diesel_codegen build process +# - git - cargo builds in user projects +# - linux-headers-amd64 - needed for building openssl 1.1 (stretch only) +# - file - needed by rustup.sh install +# - automake autoconf libtool - support crates building C deps as part cargo build +# recently removed: +# cmake (not used), nano, zlib1g-dev +RUN apt-get update && apt-get install -y \ + musl-dev \ + musl-tools \ + git \ + file \ + openssh-client \ + make \ + g++ \ + curl \ + pkgconf \ + ca-certificates \ + xutils-dev \ + libssl-dev \ + libpq-dev \ + automake \ + autoconf \ + libtool \ + python3 \ + --no-install-recommends && \ + rm -rf /var/lib/apt/lists/* + +# Convenience list of versions and variables for compilation later on +# This helps continuing manually if anything breaks. +ENV SSL_VER="1.0.2u" \ + CURL_VER="7.77.0" \ + ZLIB_VER="1.2.13" \ + PQ_VER="11.12" \ + SQLITE_VER="3350500" \ + CC=musl-gcc \ + PREFIX=/musl \ + PATH=/usr/local/bin:/root/.cargo/bin:$PATH \ + PKG_CONFIG_PATH=/usr/local/lib/pkgconfig \ + LD_LIBRARY_PATH=$PREFIX + +# Set up a prefix for musl build libraries, make the linker's job of finding them easier +# Primarily for the benefit of postgres. +# Lastly, link some linux-headers for openssl 1.1 (not used herein) +RUN mkdir $PREFIX && \ + echo "$PREFIX/lib" >> /etc/ld-musl-x86_64.path && \ + ln -s /usr/include/x86_64-linux-gnu/asm /usr/include/x86_64-linux-musl/asm && \ + ln -s /usr/include/asm-generic /usr/include/x86_64-linux-musl/asm-generic && \ + ln -s /usr/include/linux /usr/include/x86_64-linux-musl/linux + +# Build zlib (used in openssl and pq) +RUN curl -sSL https://zlib.net/zlib-$ZLIB_VER.tar.gz | tar xz && \ + cd zlib-$ZLIB_VER && \ + CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure --static --prefix=$PREFIX && \ + make -j$(nproc) && make install && \ + cd .. && rm -rf zlib-$ZLIB_VER + +# Build openssl (used in curl and pq) +# Would like to use zlib here, but can't seem to get it to work properly +# TODO: fix so that it works +RUN curl -sSL https://www.openssl.org/source/old/1.0.2/openssl-$SSL_VER.tar.gz | tar xz && \ + cd openssl-$SSL_VER && \ + ./Configure no-zlib no-shared -fPIC --prefix=$PREFIX --openssldir=$PREFIX/ssl linux-x86_64 && \ + env C_INCLUDE_PATH=$PREFIX/include make depend 2> /dev/null && \ + make -j$(nproc) && make install && \ + cd .. && rm -rf openssl-$SSL_VER + +# Build curl (needs with-zlib and all this stuff to allow https) +# curl_LDFLAGS needed on stretch to avoid fPIC errors - though not sure from what +RUN curl -sSL https://curl.se/download/curl-$CURL_VER.tar.gz | tar xz && \ + cd curl-$CURL_VER && \ + CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \ + --enable-shared=no --with-zlib --enable-static=ssl --enable-optimize --prefix=$PREFIX \ + --with-ca-path=/etc/ssl/certs/ --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt --without-ca-fallback \ + --with-openssl && \ + make -j$(nproc) curl_LDFLAGS="-all-static" && make install && \ + cd .. && rm -rf curl-$CURL_VER + +# Build libpq +RUN curl -sSL https://ftp.postgresql.org/pub/source/v$PQ_VER/postgresql-$PQ_VER.tar.gz | tar xz && \ + cd postgresql-$PQ_VER && \ + CC="musl-gcc -fPIE -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \ + --without-readline \ + --with-openssl \ + --prefix=$PREFIX --host=x86_64-unknown-linux-musl && \ + cd src/interfaces/libpq make -s -j$(nproc) all-static-lib && make -s install install-lib-static && \ + cd ../../bin/pg_config && make -j $(nproc) && make install && \ + cd .. && rm -rf postgresql-$PQ_VER diff --git a/integration/Cargo.toml b/integration/Cargo.toml index 28028a2..706d97d 100644 --- a/integration/Cargo.toml +++ b/integration/Cargo.toml @@ -2,26 +2,23 @@ name = "integration" version = "0.1.0" authors = ["plazmoid "] -edition = "2018" +edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process", "time"] } -log = "^0.4" -env_logger = "0.8.3" -uuid = { version = "0.6.5", features = ["serde", "v4"] } -reqwest = { version = "0.11", features = ["json"] } -serde_json = "1.0" -serde = { version = "1.0.114", features = ["derive"] } -futures = "0.3.5" +once_cell = "1.10.0" +reqwest = { workspace = true } +rstest = "0.12" +serde = { workspace = true } +serde_json = { workspace = true } shlex = "1.0.0" - -[dependencies.u_lib] -path = "../lib/u_lib" -version = "*" +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "time"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["serde", "v4"] } +u_lib = { path = "../lib/u_lib", features = ["panel"] } [[test]] name = "integration" -path = "tests/tests.rs" \ No newline at end of file +path = "tests/lib.rs" diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml index c0381e3..f988509 100644 --- a/integration/docker-compose.yml +++ b/integration/docker-compose.yml @@ -1,4 +1,8 @@ -version: "2.1" +version: "3.4" + +x-global: + user: &user + "${DOCKER_UID:-1000}:${DOCKER_GID:-1000}" networks: u_net: @@ -6,28 +10,28 @@ networks: services: u_server: + user: *user image: unki/u_server networks: - u_net volumes: - - ../target/x86_64-unknown-linux-musl/release/u_server:/u_server - - ../:/unki/ + - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_server:/unki/u_server + - ../certs:/unki/certs + - ../logs:/unki/logs:rw working_dir: /unki - command: bash -c " - export DATABASE_URL=postgres://$${DB_USER}:$${DB_PASSWORD}@$${DB_HOST}/$${DB_NAME} && - diesel setup && diesel migration run && /u_server" + command: /unki/u_server depends_on: u_db: condition: service_healthy - expose: - - '63714' + ports: + - 63714:63714 env_file: - ../.env - ../.env.private environment: - RUST_LOG: trace + RUST_LOG: warp=info,u_server_lib=debug healthcheck: - test: /bin/ss -tlpn | grep 63714 + test: ss -tlpn | grep 63714 interval: 5s timeout: 2s retries: 2 @@ -36,64 +40,57 @@ services: image: unki/u_db networks: - u_net - expose: - - '5432' + ports: + - 54321:5432 env_file: - ../.env - ../.env.private + working_dir: /unki + volumes: + - ../migrations:/unki/migrations + command: /unki/u_db_entrypoint.sh svc healthcheck: - test: /bin/ss -tlpn | grep 5432 + # test if db's port is open and db is created + test: ss -tlpn | grep 5432 && psql -lqt -U $${DB_USER} | grep -qw $${DB_NAME} interval: 5s - timeout: 2s - retries: 2 - - u_agent_1: - image: unki/u_agent - networks: - - u_net - volumes: - - ../target/x86_64-unknown-linux-musl/release/u_agent:/u_agent - command: /u_agent u_server - env_file: - - ../.env - environment: - RUST_LOG: u_agent=debug - depends_on: - u_server: - condition: service_healthy + timeout: 5s + retries: 3 - u_agent_2: + u_agent: + user: *user image: unki/u_agent networks: - u_net volumes: - - ../target/x86_64-unknown-linux-musl/release/u_agent:/u_agent - command: /u_agent u_server + - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_agent:/unki/u_agent + - ../logs:/unki/logs:rw + working_dir: /unki + command: /unki/u_agent u_server env_file: - ../.env environment: - RUST_LOG: u_agent=debug + RUST_LOG: u_agent=debug,u_lib=debug depends_on: u_server: condition: service_healthy tests_runner: + user: *user image: unki/tests_runner networks: - u_net volumes: - - ~/.cargo/registry:/root/.cargo/registry - - ./:/tests/ - - ../certs:/certs - - ../target/x86_64-unknown-linux-musl/release/u_panel:/u_panel - - ../lib/u_lib:/lib/u_lib - - ../lib/u_api_proc_macro:/lib/u_api_proc_macro + - ${HOME}/.cargo/registry/:/usr/local/cargo/registry/ + - ../__Cargo_integration.toml:/tests/Cargo.toml + - ./:/tests/integration/ + - ../certs:/tests/certs + - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_panel:/u_panel + - ../lib/u_lib:/tests/lib/u_lib + - ../logs:/tests/integration/logs:rw working_dir: - /tests/ + /tests/integration/ depends_on: - u_agent_1: - condition: service_started - u_agent_2: + u_agent: condition: service_started u_server: condition: service_healthy diff --git a/integration/docker.py b/integration/docker.py index 68abd69..36035a3 100644 --- a/integration/docker.py +++ b/integration/docker.py @@ -1,26 +1,28 @@ import subprocess +import shlex + from utils import * -BASE_IMAGE_DIR = 'images' +BASE_IMAGE_DIR = '../images/integration-tests' -DOCKERFILES = { - 'u_agent': { +DOCKERFILES = [ + { + 'name': 'u_agent', 'ctx': BASE_IMAGE_DIR, - 'dockerfile_prefix': 'u_agent' }, - 'u_server': { + { + 'name': 'u_server', 'ctx': BASE_IMAGE_DIR, - 'dockerfile_prefix': 'u_server' }, - 'u_db': { + { + 'name': 'u_db', 'ctx': BASE_IMAGE_DIR, - 'dockerfile_prefix': 'u_db' }, - 'tests_runner': { + { + 'name': 'tests_runner', 'ctx': BASE_IMAGE_DIR, - 'dockerfile_prefix': 'tests_runner' }, -} +] def docker(args): @@ -62,20 +64,96 @@ def check_state(containers): def rebuild_images_if_needed(force_rebuild=False): - for img_name, data in DOCKERFILES.items(): - ctx = data['ctx'] - df_prefix = data.get('dockerfile_prefix') + for img in DOCKERFILES: + ctx = img['ctx'] + name = img.get('name') df_suffix = 'Dockerfile' - img_name = f'unki/{img_name}' + img_name = f'unki/{name}' log(f'Building docker image {img_name}') cmd = [ 'build', - '-t', - img_name, + '-t', img_name, + '-f', f'{BASE_IMAGE_DIR}/{name}.{df_suffix}', ctx, ] - if df_prefix: - cmd += ['-f', f'{ctx}/{df_prefix}.{df_suffix}'] if force_rebuild: cmd += ['--no-cache'] docker(cmd) + + +class Compose: + ALL_IMAGES = [ + 'u_agent', + 'u_server', + 'u_db', + 'tests_runner', + ] + + def __init__(self): + self.container_tpl = 'integration-%s-%d' + self.cmd_container = self.container_tpl % ('tests_runner', 1) + self.ALL_CONTAINERS = [self.container_tpl % + (c, 1) for c in self.ALL_IMAGES] + self.scaled_svc = {} + self.scale("u_agent", 2) + + def scale(self, svc, count): + for c in range(1, count): + new_container = self.container_tpl % (svc, c + 1) + self.ALL_CONTAINERS.append(new_container) + self.scaled_svc[svc] = count + + def _call(self, *args): + cmd = [ + 'docker-compose', + '--ansi=never', + ] + list(args) + log(f'Running docker-compose command: {cmd}') + subprocess.check_call(cmd) + + def up(self): + log(f'Instanciating cluster: {self.ALL_CONTAINERS}') + scaled = [f"{k}={v}" for k, v in self.scaled_svc.items()] + if len(scaled) > 0: + scaled.insert(0, '--scale') + self._call('up', '-d', *scaled) + + def down(self): + log('Shutting down cluster') + self._call('down') + + def stop(self): + log('Stopping cluster') + self._call('stop') + + def run(self, cmd): + container = self.cmd_container + if isinstance(cmd, str): + cmd = shlex.split(cmd) + result = docker([ + 'exec', + '-ti', + container + ] + cmd) + return result + + def is_alive(self): + log('Check if all containers are alive') + + errors = check_state(self.ALL_CONTAINERS) + + if errors: + print_errors(errors) + raise TestsError('Error during `is_alive` check') + else: + log('All containers are alive') + + def print_containers_logs(self): + for container in self.ALL_CONTAINERS: + try: + docker([ + 'logs', + container + ]) + except Exception: + pass diff --git a/integration/docker_compose.py b/integration/docker_compose.py deleted file mode 100644 index 47e3916..0000000 --- a/integration/docker_compose.py +++ /dev/null @@ -1,66 +0,0 @@ -import subprocess -import shlex -from utils import * -from docker import docker, check_state, print_errors - - -class Compose: - ALL_CONTAINERS = [ - 'u_agent_1', - 'u_agent_2', - 'u_server', - 'u_db', - 'tests_runner', - ] - - def __init__(self): - self.container_tpl = 'integration_%s_1' - self.cmd_container = self.container_tpl % 'tests_runner' - self.ALL_CONTAINERS = [self.container_tpl % c for c in self.ALL_CONTAINERS] - - def _call(self, *args): - subprocess.check_call([ - 'docker-compose', - '--no-ansi', - ] + list(args) - ) - - def up(self): - log('Instanciating cluster') - self._call('up', '-d') - log('Ok') - - def down(self): - log('Shutting down cluster') - self._call('down') - log('Ok') - - def stop(self): - log('Stopping cluster') - self._call('stop') - log('Ok') - - def run(self, cmd): - container = self.cmd_container - if isinstance(cmd, str): - cmd = shlex.split(cmd) - log(f'Running command "{cmd}" in container {container}') - result = docker([ - 'exec', - '-ti', - container - ] + cmd) - log('Ok') - return result - - def is_alive(self): - log('Check if all containers are alive') - - errors = check_state(self.ALL_CONTAINERS) - log('Check done') - - if errors: - print_errors(errors) - raise TestsError('Error during `is_alive` check') - else: - log('All containers are alive') diff --git a/integration/images/tests_runner.Dockerfile b/integration/images/tests_runner.Dockerfile deleted file mode 100644 index 2411bd6..0000000 --- a/integration/images/tests_runner.Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM rust:1.53 - -RUN rustup target add x86_64-unknown-linux-musl -CMD ["sleep", "3600"] \ No newline at end of file diff --git a/integration/images/u_db.Dockerfile b/integration/images/u_db.Dockerfile deleted file mode 100644 index 8577c8d..0000000 --- a/integration/images/u_db.Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM postgres:13.3 - -RUN apt update && apt -y upgrade && apt install -y iproute2 \ No newline at end of file diff --git a/integration/images/u_server.Dockerfile b/integration/images/u_server.Dockerfile deleted file mode 100644 index ea82ccd..0000000 --- a/integration/images/u_server.Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM rust:1.53 - -RUN cargo install diesel_cli --no-default-features --features postgres \ No newline at end of file diff --git a/integration/integration_tests.py b/integration/integration_tests.py index e4fe460..512ea98 100644 --- a/integration/integration_tests.py +++ b/integration/integration_tests.py @@ -1,35 +1,73 @@ import signal import sys +import toml + +from docker import rebuild_images_if_needed, Compose +from pathlib import Path from utils import * -from docker import rebuild_images_if_needed -from docker_compose import Compose + +CARGO_INTEGRATION_TOML = Path('../__Cargo_integration.toml') +CLUSTER = Compose() + + +def fail(msg): + err(msg) + sys.exit(1) -cluster = Compose() +def usage_exit(): + usage = f"""Usage: + python {__file__.split('/')[-1]} [--rebuild] [--preserve] [--no-run] [--down]""" + fail(usage) -def abort_handler(s, _): - warn(f'Received signal: {s}') - warn(f'Gracefully stopping...') - cluster.down() + +def create_integration_workspace(): + if CARGO_INTEGRATION_TOML.exists(): + CARGO_INTEGRATION_TOML.unlink() + workspace = toml.load('../Cargo.toml') + workspace['workspace']['members'] = ['integration'] + with open(CARGO_INTEGRATION_TOML, 'w') as fo: + toml.dump(workspace, fo) def run_tests(): - force_rebuild = '--rebuild' in sys.argv - preserve_containers = '--preserve' in sys.argv + allowed_args = set(["--rebuild", "--preserve", "--no-run", "--release", "--down"]) + args = sys.argv[1:] + if not set(args).issubset(allowed_args): + usage_exit() + force_rebuild = '--rebuild' in args + preserve_containers = '--preserve' in args + only_setup_cluster = '--no-run' in args + down_cluster = "--down" in args + + def _cleanup(): + if not preserve_containers and not only_setup_cluster: + CLUSTER.down() + CARGO_INTEGRATION_TOML.unlink(missing_ok=True) + + def abort_handler(s, _): + warn(f'Received signal: {s}, gracefully stopping...') + _cleanup() + + if down_cluster: + _cleanup() + return + for s in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): signal.signal(s, abort_handler) rebuild_images_if_needed(force_rebuild) + create_integration_workspace() try: - cluster.up() - cluster.is_alive() - cluster.run('cargo test --test integration') + CLUSTER.up() + CLUSTER.is_alive() + if not only_setup_cluster: + CLUSTER.run('cargo test --test integration') except Exception as e: - err(e) - sys.exit(1) + CLUSTER.print_containers_logs() + fail(e) finally: - if not preserve_containers: - cluster.down() + _cleanup() if __name__ == '__main__': diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 3f3eca1..b8efc7e 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -1,3 +1,6 @@ #!/bin/bash set -e +export DOCKER_UID=$(id -u) +export DOCKER_GID=$(id -g) +[[ "$@" =~ "--release" ]] && export PROFILE=release || export PROFILE=debug python integration_tests.py $@ diff --git a/integration/tests/behaviour.rs b/integration/tests/behaviour.rs deleted file mode 100644 index 8b13789..0000000 --- a/integration/tests/behaviour.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs new file mode 100644 index 0000000..2e6cdce --- /dev/null +++ b/integration/tests/fixtures/agent.rs @@ -0,0 +1,35 @@ +use crate::helpers::ENV; +use u_lib::{api::ClientHandler, messaging::Reportable, models::*}; +use uuid::Uuid; + +pub struct RegisteredAgent { + pub uid: Uuid, +} + +impl RegisteredAgent { + pub async fn unregister(self) { + let cli = ClientHandler::new(&ENV.u_server, None); + cli.del(self.uid).await.unwrap(); + } +} + +#[fixture] +pub async fn register_agent() -> RegisteredAgent { + let cli = ClientHandler::new(&ENV.u_server, None); + let agent_uid = Uuid::new_v4(); + println!("registering agent {agent_uid}"); + let resp = cli + .get_personal_jobs(agent_uid) + .await + .unwrap() + .pop() + .unwrap(); + let job_id = resp.job_id; + let job = cli.get_jobs(Some(job_id)).await.unwrap().pop().unwrap(); + assert_eq!(job.alias, Some("agent_hello".to_string())); + let mut agent_data = AssignedJob::from(&job); + agent_data.agent_id = agent_uid; + agent_data.set_result(&Agent::with_id(agent_uid)); + cli.report(Reportable::Assigned(agent_data)).await.unwrap(); + RegisteredAgent { uid: agent_uid } +} diff --git a/integration/tests/fixtures/mod.rs b/integration/tests/fixtures/mod.rs new file mode 100644 index 0000000..f17bc55 --- /dev/null +++ b/integration/tests/fixtures/mod.rs @@ -0,0 +1 @@ +pub mod agent; diff --git a/integration/tests/helpers/client.rs b/integration/tests/helpers/client.rs deleted file mode 100644 index 01c5c89..0000000 --- a/integration/tests/helpers/client.rs +++ /dev/null @@ -1,48 +0,0 @@ -use reqwest::{Client, RequestBuilder, Url}; -use serde::Serialize; -use serde_json::{from_str, json, Value}; - -const SERVER: &str = "u_server"; -const PORT: &str = "63714"; - -pub struct AgentClient { - client: Client, - base_url: Url, -} - -impl AgentClient { - pub fn new() -> Self { - Self { - client: Client::new(), - base_url: Url::parse(&format!("http://{}:{}", SERVER, PORT)).unwrap(), - } - } - - async fn process_request(&self, req: RequestBuilder, resp_needed: bool) -> Value { - let resp = req.send().await.unwrap(); - if let Err(e) = resp.error_for_status_ref() { - panic!( - "Server responded with code {}\nError: {}", - e.status() - .map(|s| s.to_string()) - .unwrap_or(String::from("")), - e.to_string() - ); - } - if !resp_needed { - return json!([]); - } - let resp: Value = from_str(&resp.text().await.unwrap()).unwrap(); - resp.get("inner").unwrap().get(0).unwrap().clone() - } - - pub async fn get>(&self, url: S) -> Value { - let req = self.client.get(self.base_url.join(url.as_ref()).unwrap()); - self.process_request(req, true).await - } - - pub async fn post, B: Serialize>(&self, url: S, body: &B) -> Value { - let req = self.client.post(self.base_url.join(url.as_ref()).unwrap()); - self.process_request(req.json(body), false).await - } -} diff --git a/integration/tests/helpers/mod.rs b/integration/tests/helpers/mod.rs index 783b365..91d7eb7 100644 --- a/integration/tests/helpers/mod.rs +++ b/integration/tests/helpers/mod.rs @@ -1,3 +1,8 @@ pub mod panel; pub use panel::Panel; + +use once_cell::sync::Lazy; +use u_lib::utils::{env::DefaultEnv, load_env_default}; + +pub static ENV: Lazy = Lazy::new(|| load_env_default().unwrap()); diff --git a/integration/tests/helpers/panel.rs b/integration/tests/helpers/panel.rs index 41008f9..f8fa576 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration/tests/helpers/panel.rs @@ -1,54 +1,67 @@ use serde::de::DeserializeOwned; -use serde_json::from_slice; -use shlex::split; +use serde_json::{from_slice, Value}; +use std::fmt::{Debug, Display}; use std::process::{Command, Output}; -use u_lib::{datatypes::DataResult, messaging::AsMsg}; +use u_lib::{ + datatypes::PanelResult, + utils::{bytes_to_string, ProcOutput}, +}; const PANEL_BINARY: &str = "/u_panel"; -type PanelResult = Result, String>; - pub struct Panel; impl Panel { fn run(args: &[&str]) -> Output { - Command::new(PANEL_BINARY) - .arg("--json") - .args(args) - .output() - .unwrap() + Command::new(PANEL_BINARY).args(args).output().unwrap() } - pub fn output_argv(args: &[&str]) -> PanelResult { - let result = Self::run(args); - assert!(result.status.success()); - from_slice(&result.stdout).map_err(|e| e.to_string()) + pub fn output_argv(argv: &[&str]) -> PanelResult { + let result = Self::run(argv); + let output = ProcOutput::from_output(&result).into_vec(); + from_slice(&output) + .map_err(|e| { + eprintln!( + "Failed to decode panel response: '{}'", + bytes_to_string(&output) + ); + e.to_string() + }) + .unwrap() } - pub fn output(args: impl Into) -> PanelResult { - let splitted = split(args.into().as_ref()).unwrap(); - Self::output_argv( + pub fn output( + args: impl Into + Display, + ) -> PanelResult { + eprintln!(">>> {PANEL_BINARY} {}", &args); + let splitted = shlex::split(args.into().as_ref()).unwrap(); + let result = Self::output_argv( splitted .iter() .map(|s| s.as_ref()) .collect::>() .as_ref(), - ) + ); + match &result { + PanelResult::Ok(r) => eprintln!("<<<+ {r:02x?}"), + PanelResult::Err(e) => eprintln!("<<(data: PanelResult) -> T { - match data.unwrap() { - DataResult::Ok(r) => r, - DataResult::Err(e) => panic!("Panel failed with erroneous status: {}", e), + fn status_is_ok(data: PanelResult) -> T { + match data { + PanelResult::Ok(r) => r, + PanelResult::Err(e) => panic!("Panel failed: {}", e), } } - pub fn check_status<'s, T: AsMsg + DeserializeOwned>(args: &'s str) { - let result: PanelResult = Self::output(args); + pub fn check_status(args: impl Into + Display) { + let result: PanelResult = Self::output(args); Self::status_is_ok(result); } - pub fn check_output(args: impl Into) -> T { + pub fn check_output(args: impl Into + Display) -> T { let result = Self::output(args); Self::status_is_ok(result) } diff --git a/integration/tests/integration/behaviour.rs b/integration/tests/integration/behaviour.rs new file mode 100644 index 0000000..932cb03 --- /dev/null +++ b/integration/tests/integration/behaviour.rs @@ -0,0 +1,51 @@ +use crate::fixtures::agent::*; +use crate::helpers::Panel; + +use rstest::rstest; +use serde_json::{json, to_string}; +use std::error::Error; +use std::time::Duration; +use tokio::time::sleep; +use u_lib::models::*; +use uuid::Uuid; + +type TestResult = Result>; + +#[rstest] +#[tokio::test] +async fn test_registration(#[future] register_agent: RegisteredAgent) -> TestResult { + let agent = register_agent.await; + let agents: Vec = Panel::check_output("agents read"); + let found = agents.iter().find(|v| v.id == agent.uid); + assert!(found.is_some()); + Panel::check_status(format!("agents delete {}", agent.uid)); + Ok(()) +} + +#[tokio::test] +async fn test_setup_tasks() -> TestResult { + let agents: Vec = Panel::check_output("agents read"); + let agent_uid = match agents.get(0) { + Some(a) => a.id, + None => panic!("Some independent agents should present"), + }; + let job_alias = "passwd_contents"; + let job = json!( + {"alias": job_alias, "payload": b"cat /etc/passwd", "argv": "/bin/bash {}" } + ); + let cmd = format!("jobs create '{}'", to_string(&job).unwrap()); + Panel::check_status(cmd); + let cmd = format!("map create {} {}", agent_uid, job_alias); + let assigned_uids: Vec = Panel::check_output(cmd); + for _ in 0..3 { + let result: Vec = + Panel::check_output(format!("map read {}", assigned_uids[0])); + if result[0].state == JobState::Finished { + return Ok(()); + } else { + sleep(Duration::from_secs(5)).await; + eprintln!("waiting for task"); + } + } + panic!("Job didn't appear in the job map"); +} diff --git a/integration/tests/integration/connection.rs b/integration/tests/integration/connection.rs new file mode 100644 index 0000000..78a4c2c --- /dev/null +++ b/integration/tests/integration/connection.rs @@ -0,0 +1,22 @@ +use crate::helpers::ENV; +use u_lib::config::MASTER_PORT; + +#[tokio::test] +async fn test_non_auth_connection_dropped() { + let client = reqwest::ClientBuilder::new() + .danger_accept_invalid_certs(true) + .build() + .unwrap(); + match client + .get(format!("https://{}:{}", &ENV.u_server, MASTER_PORT)) + .send() + .await + { + Err(e) => { + let err = e.to_string(); + println!("captured err: {err}"); + assert!(err.contains("certificate required")); + } + _ => panic!("no error occured on foreign client connection"), + } +} diff --git a/integration/tests/integration/mod.rs b/integration/tests/integration/mod.rs new file mode 100644 index 0000000..0b76512 --- /dev/null +++ b/integration/tests/integration/mod.rs @@ -0,0 +1,2 @@ +mod behaviour; +mod connection; diff --git a/integration/tests/lib.rs b/integration/tests/lib.rs new file mode 100644 index 0000000..826b82d --- /dev/null +++ b/integration/tests/lib.rs @@ -0,0 +1,6 @@ +mod fixtures; +mod helpers; +mod integration; + +#[macro_use] +extern crate rstest; diff --git a/integration/tests/tests.rs b/integration/tests/tests.rs deleted file mode 100644 index 636a4be..0000000 --- a/integration/tests/tests.rs +++ /dev/null @@ -1,67 +0,0 @@ -mod helpers; - -use helpers::Panel; - -use std::error::Error; -use std::thread::sleep; -use std::time::Duration; -use u_lib::{api::ClientHandler, models::*}; -use uuid::Uuid; - -type TestResult = Result>; - -async fn register_agent() -> Uuid { - let cli = ClientHandler::new(None); - let agent_uid = Uuid::new_v4(); - let resp = cli - .get_personal_jobs(Some(agent_uid)) - .await - .unwrap() - .pop() - .unwrap(); - let job_id = resp.job_id; - let resp = cli.get_jobs(Some(job_id)).await.unwrap().pop().unwrap(); - assert_eq!(resp.alias, Some("agent_hello".to_string())); - let agent_data = Agent { - id: agent_uid, - ..Default::default() - }; - cli.report(&vec![ExecResult::Agent(agent_data)]) - .await - .unwrap(); - agent_uid -} - -#[tokio::test] -async fn test_registration() -> TestResult { - let agent_uid = register_agent().await; - let agents: Vec = Panel::check_output("agents list"); - let found = agents.iter().find(|v| v.id == agent_uid); - assert!(found.is_some()); - //teardown - Panel::check_status::(&format!("agents delete {}", agent_uid)); - Ok(()) -} - -#[tokio::test] -async fn test_setup_tasks() -> TestResult { - //some independent agents should present - let agents: Vec = Panel::check_output("agents list"); - let agent_uid = agents[0].id; - let job_alias = "passwd_contents"; - let cmd = format!("jobs add --alias {} 'cat /etc/passwd'", job_alias); - Panel::check_status::(&cmd); - let cmd = format!("jobmap add {} {}", agent_uid, job_alias); - let assigned_uids: Vec = Panel::check_output(cmd); - for _ in 0..3 { - let result: Vec = - Panel::check_output(format!("jobmap list {}", assigned_uids[0])); - if result[0].state == JobState::Finished { - return Ok(()); - } else { - sleep(Duration::from_secs(5)); - eprintln!("waiting for task"); - } - } - panic!() -} diff --git a/lib/u_api_proc_macro/Cargo.toml b/lib/u_api_proc_macro/Cargo.toml deleted file mode 100644 index c0e8456..0000000 --- a/lib/u_api_proc_macro/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "u_api_proc_macro" -version = "0.1.0" -authors = ["plazmoid "] -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[lib] -proc-macro = true - -[dependencies] -syn = { version = "1.0", features = ["full", "extra-traits"] } -quote = "1.0" -strum = { version = "0.20", features = ["derive"] } -proc-macro2 = "1.0" \ No newline at end of file diff --git a/lib/u_api_proc_macro/src/lib.rs b/lib/u_api_proc_macro/src/lib.rs deleted file mode 100644 index 46f1efa..0000000 --- a/lib/u_api_proc_macro/src/lib.rs +++ /dev/null @@ -1,179 +0,0 @@ -use proc_macro::TokenStream; -use proc_macro2::{Ident, TokenStream as TokenStream2}; -use quote::quote; -use std::{collections::HashMap, str::FromStr}; -use strum::EnumString; -use syn::{ - parse_macro_input, punctuated::Punctuated, AttributeArgs, FnArg, ItemFn, Lit, NestedMeta, - ReturnType, Signature, Token, Type, -}; - -#[derive(EnumString, Debug)] -enum ReqMethod { - GET, - POST, -} - -#[derive(Debug)] -struct Endpoint { - method: ReqMethod, -} - -#[derive(Debug)] -struct FnArgs { - url_param: Option, - payload: Option, -} - -#[proc_macro_attribute] -pub fn api_route(args: TokenStream, item: TokenStream) -> TokenStream { - let args: AttributeArgs = parse_macro_input!(args); - let input: ItemFn = parse_macro_input!(item); - let Signature { - ident, - inputs, - generics, - output, - .. - } = input.sig; - let (impl_generics, _, _) = generics.split_for_impl(); - let FnArgs { url_param, payload } = parse_fn_args(inputs); - let Endpoint { method } = parse_attr_args(args); - let url_path = build_url_path(&ident, &url_param); - let return_ty = match output { - ReturnType::Type(_, ty) => quote!(#ty), - ReturnType::Default => quote!(()), - }; - let request = match method { - ReqMethod::GET => build_get(url_path), - ReqMethod::POST => build_post(url_path, &payload), - }; - let url_param = match url_param { - Some(p) => quote!(, param: #p), - None => TokenStream2::new(), - }; - let payload = match payload { - Some(p) => quote!(, payload: #p), - None => TokenStream2::new(), - }; - let q = quote! { - pub async fn #ident #impl_generics( - &self #url_param #payload - ) -> UResult<#return_ty> { - let request = { - #request - }; - let response = request.send().await?; - let content_len = response.content_length(); - let is_success = match response.error_for_status_ref() { - Ok(_) => Ok(()), - Err(e) => Err(UError::from(e)) - }; - match is_success { - Ok(_) => response.json::>() - .await - .map(|msg| msg.into_inner()) - .or_else(|e| { - match content_len { - Some(0) => Ok(Default::default()), - _ => Err(UError::from(e)) - } - }), - Err(UError::NetError(err_src, _)) => Err( - UError::NetError( - err_src, - response.text().await.unwrap() - ) - ), - _ => unreachable!() - } - } - }; - //eprintln!("#!#! RESULT:\n{}", q); - q.into() -} - -fn parse_fn_args(raw: Punctuated) -> FnArgs { - let mut arg: HashMap = raw - .into_iter() - .filter_map(|arg| { - if let FnArg::Typed(argt) = arg { - let mut arg_name = String::new(); - // did you think I won't overplay you? won't destroy? - |arg_ident| -> TokenStream { - let q: TokenStream = quote!(#arg_ident).into(); - arg_name = parse_macro_input!(q as Ident).to_string(); - TokenStream::new() - }(argt.pat); - if &arg_name != "url_param" && &arg_name != "payload" { - panic!("Wrong arg name: {}", &arg_name) - } - let arg_type = *argt.ty.clone(); - Some((arg_name, arg_type)) - } else { - None - } - }) - .collect(); - FnArgs { - url_param: arg.remove("url_param"), - payload: arg.remove("payload"), - } -} - -fn build_get(url: TokenStream2) -> TokenStream2 { - quote! { - let request = self.build_get(#url); - request - } -} - -fn build_post(url: TokenStream2, payload: &Option) -> TokenStream2 { - let pld = match payload { - Some(_) => quote! { - .json(&payload.as_message()) - }, - None => TokenStream2::new(), - }; - quote! { - let request = self.build_post(#url); - request #pld - } -} - -fn build_url_path(path: &Ident, url_param: &Option) -> TokenStream2 { - let url_param = match url_param { - Some(_) => quote! { - + &opt_to_string(param) - }, - None => TokenStream2::new(), - }; - quote! { - &format!( - "{}/{}", - stringify!(#path), - String::new() #url_param - ) - } -} - -fn parse_attr_args(args: AttributeArgs) -> Endpoint { - let mut args = args.into_iter(); - let method = match args.next() { - Some(method) => match method { - NestedMeta::Lit(l) => { - if let Lit::Str(s) = l { - match ReqMethod::from_str(&s.value()) { - Ok(v) => v, - Err(_) => panic!("Unknown method"), - } - } else { - panic!("Method must be a str") - } - } - _ => panic!("Method must be on the first place"), - }, - None => panic!("Method required"), - }; - Endpoint { method } -} diff --git a/lib/u_api_proc_macro/tests/tests.rs b/lib/u_api_proc_macro/tests/tests.rs deleted file mode 100644 index 7c4b404..0000000 --- a/lib/u_api_proc_macro/tests/tests.rs +++ /dev/null @@ -1,15 +0,0 @@ -/* -use std::fmt::Display; -use u_api_proc_macro::api_route; - -type UResult = Result; - -struct ClientHandler; -struct Paths; - -#[test] -fn test1() { - #[api_route("GET", Uuid)] - fn list(url_param: T) {} -} -*/ diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index 5d56eff..1696f34 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -2,36 +2,39 @@ name = "u_lib" version = "0.1.0" authors = ["plazmoid "] -edition = "2018" +edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = { workspace = true } +chrono = "0.4.19" +diesel = { version = "1.4.5", features = ["postgres", "uuid"], optional = true } +diesel-derive-enum = { version = "1", features = ["postgres"], optional = true } dotenv = "0.15.0" -serde = { version = "1.0.114", features = ["derive"] } -uuid = { version = "0.6.5", features = ["serde", "v4"] } -nix = "0.17" -libc = "^0.2" -lazy_static = "1.4.0" -tokio = { version = "1.2.0", features = ["rt-multi-thread", "sync", "macros", "process", "time"] } -reqwest = { version = "0.11", features = ["json", "native-tls"] } -openssl = "*" +envy = "0.4.2" futures = "0.3.5" guess_host_triple = "0.1.2" -thiserror = "*" +libc = "^0.2" +lazy_static = "1.4.0" log = "*" -mockall = "0.9.1" -env_logger = "0.8.3" -diesel-derive-enum = { version = "1", features = ["postgres"] } -chrono = "0.4.19" -strum = { version = "0.20", features = ["derive"] } +nix = "0.17" once_cell = "1.7.2" +platforms = "3.0.1" +reqwest = { workspace = true, features = ["native-tls"] } shlex = "1.0.0" -u_api_proc_macro = { version = "*", path = "../u_api_proc_macro" } +serde = { workspace = true } +serde_json = { workspace = true } +strum = { version = "0.20", features = ["derive"] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "sync", "macros", "process", "time"] } +tracing-appender = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } +uuid = { workspace = true, features = ["serde", "v4"] } -[dependencies.diesel] -version = "1.4.5" -features = ["postgres", "uuid"] +[features] +panel = [] +server = ["dep:diesel", "dep:diesel-derive-enum"] [dev-dependencies] -test-case = "1.1.0" \ No newline at end of file +rstest = "0.12" diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 82ba4bf..28463d4 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -1,94 +1,164 @@ -#[allow(non_upper_case_globals)] +use std::collections::HashMap; +use std::fmt::Debug; + use crate::{ - config::{MASTER_PORT, MASTER_SERVER}, - messaging::{AsMsg, BaseMessage}, - models, - utils::{opt_to_string, VecDisplay}, - UError, UResult, + config::{get_self_uid, MASTER_PORT}, + messaging::{self, AsMsg, BaseMessage}, + models::{self}, + utils::{opt_to_string, OneOrVec}, + UError, }; -use reqwest::{Certificate, Client, Identity, RequestBuilder, Url}; -use std::env; -use u_api_proc_macro::api_route; +use anyhow::{Context, Result}; +use reqwest::{header::HeaderMap, Certificate, Client, Identity, Url}; +use serde::de::DeserializeOwned; +use serde_json::from_str; use uuid::Uuid; const AGENT_IDENTITY: &[u8] = include_bytes!("../../../certs/alice.p12"); const ROOT_CA_CERT: &[u8] = include_bytes!("../../../certs/ca.crt"); +#[derive(Clone, Debug)] pub struct ClientHandler { base_url: Url, client: Client, - password: Option, } impl ClientHandler { - pub fn new(server: Option<&str>) -> Self { - let env_server = env::var("U_SERVER").unwrap_or(String::from(MASTER_SERVER)); - let master_server = server.unwrap_or(env_server.as_str()); + pub fn new(server: &str, password: Option) -> Self { let identity = Identity::from_pkcs12_der(AGENT_IDENTITY, "").unwrap(); + let mut default_headers = HashMap::from([( + "user-agent".to_string(), + get_self_uid().hyphenated().to_string(), + )]); + + if let Some(pwd) = password { + default_headers.insert("authorization".to_string(), format!("Bearer {pwd}")); + } + let client = Client::builder() .identity(identity) + .default_headers(HeaderMap::try_from(&default_headers).unwrap()) .add_root_certificate(Certificate::from_pem(ROOT_CA_CERT).unwrap()) .build() .unwrap(); + Self { client, - base_url: Url::parse(&format!("https://{}:{}", master_server, MASTER_PORT)).unwrap(), - password: None, + base_url: Url::parse(&format!("https://{}:{}", server, MASTER_PORT)).unwrap(), } } - pub fn password(mut self, password: String) -> ClientHandler { - self.password = Some(password); - self + async fn req(&self, url: impl AsRef) -> Result { + self.req_with_payload(url, ()).await } - fn set_pwd(&self, rb: RequestBuilder) -> RequestBuilder { - match &self.password { - Some(p) => rb.bearer_auth(p), - None => rb, + async fn req_with_payload( + &self, + url: impl AsRef, + payload: P, + ) -> Result { + let request = self + .client + .post(self.base_url.join(url.as_ref()).unwrap()) + .json(&payload.as_message()); + + let response = request + .send() + .await + .context("error while sending request")?; + let content_len = response.content_length(); + let is_success = match response.error_for_status_ref() { + Ok(_) => Ok(()), + Err(e) => Err(UError::from(e)), + }; + let resp = response.text().await.context("resp")?; + debug!("url = {}, resp = {}", url.as_ref(), resp); + match is_success { + Ok(_) => from_str::>(&resp) + .map(|msg| msg.into_inner()) + .or_else(|e| match content_len { + Some(0) => Ok(Default::default()), + _ => Err(UError::NetError(e.to_string(), resp)), + }), + Err(UError::NetError(err, _)) => Err(UError::NetError(err, resp)), + _ => unreachable!(), } + .map_err(From::from) } - fn build_get(&self, url: &str) -> RequestBuilder { - let rb = self.client.get(self.base_url.join(url).unwrap()); - self.set_pwd(rb) + // get jobs for client + pub async fn get_personal_jobs(&self, url_param: Uuid) -> Result> { + self.req(format!("get_personal_jobs/{}", url_param)).await } - fn build_post(&self, url: &str) -> RequestBuilder { - let rb = self.client.post(self.base_url.join(url).unwrap()); - self.set_pwd(rb) - } - // - // get jobs for client - #[api_route("GET")] - fn get_personal_jobs(&self, url_param: Option) -> VecDisplay {} - // // send something to server - #[api_route("POST")] - fn report(&self, payload: &M) -> models::Empty {} - - //##########// Admin area //##########// - /// client listing - #[api_route("GET")] - fn get_agents(&self, url_param: Option) -> VecDisplay {} - // - // get all available jobs - #[api_route("GET")] - fn get_jobs(&self, url_param: Option) -> VecDisplay {} - // - // create and upload job - #[api_route("POST")] - fn upload_jobs(&self, payload: &[models::JobMeta]) -> models::Empty {} - // - // delete something - #[api_route("GET")] - fn del(&self, url_param: Option) -> i32 {} - // - // set jobs for any client - #[api_route("POST")] - fn set_jobs(&self, url_param: Option, payload: &[String]) -> VecDisplay {} - // - // get jobs for any client - #[api_route("GET")] - fn get_agent_jobs(&self, url_param: Option) -> VecDisplay {} + pub async fn report(&self, payload: impl OneOrVec) -> Result<()> { + self.req_with_payload("report", payload.into_vec()).await + } + + // download file + pub async fn dl(&self, file: String) -> Result> { + self.req(format!("dl/{file}")).await + } + + /// get all available jobs + pub async fn get_jobs(&self, job: Option) -> Result> { + self.req(format!("get_jobs/{}", opt_to_string(job))).await + } +} + +//##########// Admin area //##########// +#[cfg(feature = "panel")] +impl ClientHandler { + /// agent listing + pub async fn get_agents(&self, agent: Option) -> Result> { + self.req(format!("get_agents/{}", opt_to_string(agent))) + .await + } + + /// update agent + pub async fn update_agent(&self, agent: models::Agent) -> Result<()> { + self.req_with_payload("update_agent", agent).await + } + + /// update job + pub async fn update_job(&self, job: models::JobMeta) -> Result<()> { + self.req_with_payload("update_job", job).await + } + + /// update result + pub async fn update_result(&self, result: models::AssignedJob) -> Result<()> { + self.req_with_payload("update_result", result).await + } + + /// create and upload job + pub async fn upload_jobs(&self, payload: impl OneOrVec) -> Result> { + self.req_with_payload("upload_jobs", payload.into_vec()) + .await + } + + /// delete something + pub async fn del(&self, item: Uuid) -> Result { + self.req(format!("del/{item}")).await + } + + /// set jobs for any agent + pub async fn set_jobs( + &self, + agent: Uuid, + job_idents: impl OneOrVec, + ) -> Result> { + self.req_with_payload(format!("set_jobs/{agent}"), job_idents.into_vec()) + .await + } + + /// get jobs for any agent + pub async fn get_agent_jobs(&self, agent: Option) -> Result> { + self.req(format!("get_agent_jobs/{}", opt_to_string(agent))) + .await + } + + pub async fn ping(&self) -> Result<()> { + self.req("ping").await + } } diff --git a/lib/u_lib/src/builder.rs b/lib/u_lib/src/builder.rs deleted file mode 100644 index da94225..0000000 --- a/lib/u_lib/src/builder.rs +++ /dev/null @@ -1,316 +0,0 @@ -use crate::{ - cache::JobCache, - executor::{Waiter, DynFut}, - models::{Agent, AssignedJob, JobMeta, JobType, ExecResult}, - utils::{CombinedResult, OneOrVec}, - UError, -}; -use guess_host_triple::guess_host_triple; -use std::collections::HashMap; - -pub struct JobBuilder { - jobs: Waiter, -} - -impl JobBuilder { - pub fn from_request(job_requests: impl OneOrVec) -> CombinedResult { - let job_requests = job_requests.into_vec(); - let mut prepared: Vec = vec![]; - let mut result = CombinedResult::new(); - for req in job_requests { - let job_meta = JobCache::get(&req.job_id); - if job_meta.is_none() { - result.err(UError::NoJob(req.job_id)); - continue; - } - let job_meta = job_meta.unwrap(); - //waiting for try-blocks stabilization - let built_req = (|| { - Ok(match job_meta.exec_type { - JobType::Shell => { - let meta = JobCache::get(&req.job_id).ok_or(UError::NoJob(req.job_id))?; - let curr_platform = guess_host_triple().unwrap_or("unknown").to_string(); - if meta.platform != curr_platform { - return Err(UError::InsuitablePlatform( - meta.platform.clone(), - curr_platform, - )); - } - let job = AssignedJob::new(req.job_id, Some(&req)); - prepared.push(Box::pin(job.run())) - } - JobType::Manage => prepared.push(Box::pin(Agent::run())), - _ => todo!(), - }) - })(); - if let Err(e) = built_req { - result.err(e) - } - } - result.ok(Self { - jobs: Waiter::new(prepared), - }); - result - } - - pub fn from_meta(job_metas: impl OneOrVec) -> CombinedResult { - let job_requests = job_metas - .into_vec() - .into_iter() - .map(|jm| { - let j_uid = jm.id; - JobCache::insert(jm); - AssignedJob::new(j_uid, None) - }) - .collect::>(); - JobBuilder::from_request(job_requests) - } - - /// Spawn jobs and pop results later - pub async fn spawn(mut self) -> Self { - self.jobs = self.jobs.spawn().await; - self - } - - /// Spawn jobs and wait for result - pub async fn wait(self) -> Vec { - self.jobs.spawn().await.wait().await - } - - /// Spawn one job and wait for result - pub async fn wait_one(self) -> ExecResult { - self.jobs.spawn().await.wait().await.pop().unwrap() - } -} - -/// Store jobs and get results by name -pub struct NamedJobBuilder { - builder: Option, - job_names: Vec<&'static str>, - results: HashMap<&'static str, ExecResult>, -} - -impl NamedJobBuilder { - pub fn from_shell( - named_jobs: impl OneOrVec<(&'static str, &'static str)>, - ) -> CombinedResult { - let mut result = CombinedResult::new(); - let jobs: Vec<(&'static str, JobMeta)> = named_jobs - .into_vec() - .into_iter() - .filter_map( - |(alias, cmd)| match JobMeta::builder().with_shell(cmd).build() { - Ok(meta) => Some((alias, meta)), - Err(e) => { - result.err(e); - None - } - }, - ) - .collect(); - result.ok(Self::from_meta(jobs)); - result - } - - pub fn from_meta(named_jobs: impl OneOrVec<(&'static str, JobMeta)>) -> Self { - let mut job_names = vec![]; - let job_metas: Vec = named_jobs - .into_vec() - .into_iter() - .map(|(alias, meta)| { - job_names.push(alias); - meta - }) - .collect(); - Self { - builder: Some(JobBuilder::from_meta(job_metas).unwrap_one()), - job_names, - results: HashMap::new(), - } - } - - pub async fn wait(mut self) -> Self { - let results = self.builder.take().unwrap().wait().await; - for (name, result) in self.job_names.iter().zip(results.into_iter()) { - self.results.insert(name, result); - } - self - } - - pub fn pop_opt(&mut self, name: &'static str) -> Option { - self.results.remove(name) - } - - pub fn pop(&mut self, name: &'static str) -> ExecResult { - self.pop_opt(name).unwrap() - } -} - -#[cfg(test)] -mod tests { - - use test_case::test_case; - use std::{time::SystemTime}; - use crate::{ - errors::UError, - models::{ - jobs::{JobMeta}, - ExecResult, - misc::JobType - }, - builder::{JobBuilder, NamedJobBuilder}, - unwrap_enum, - }; - - type TestResult = Result>; - - #[tokio::test] - async fn test_is_really_async() { - const SLEEP_SECS: u64 = 1; - let job = JobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); - let sleep_jobs: Vec = (0..50).map(|_| job.clone()).collect(); - let now = SystemTime::now(); - JobBuilder::from_meta(sleep_jobs).unwrap_one().wait().await; - assert!(now.elapsed().unwrap().as_secs() < SLEEP_SECS + 2) - } - - #[test_case( - "/bin/sh {}", - Some(b"echo test01 > /tmp/asd; cat /tmp/asd"), - "test01" - ;"sh payload" - )] - #[test_case( - r#"/usr/bin/python3 -c 'print("test02")'"#, - None, - "test02" - ;"python cmd" - )] - #[test_case( - "/{}", - Some( - br#"#!/bin/sh - TMPPATH=/tmp/lol - mkdir -p $TMPPATH - echo test03 > $TMPPATH/t - cat $TMPPATH/t"# - ), - "test03" - ;"sh multiline payload" - )] - #[test_case( - "/{} 'some msg as arg'", - Some(include_bytes!("../tests/fixtures/echoer")), - "some msg as arg" - ;"standalone binary with args" - )] - #[tokio::test] - async fn test_shell_job(cmd: &str, payload: Option<&[u8]>, expected_result: &str) -> TestResult { - let mut job = JobMeta::builder().with_shell(cmd); - if let Some(p) = payload { - job = job.with_payload(p); - } - let job = job.build().unwrap(); - let job_result = JobBuilder::from_meta(job).unwrap_one().wait_one().await; - let result = unwrap_enum!(job_result, ExecResult::Assigned); - let result = result.to_string_result().unwrap(); - assert_eq!(result.trim(), expected_result); - Ok(()) - } - - #[tokio::test] - async fn test_complex_load() -> TestResult { - const SLEEP_SECS: u64 = 1; - let now = SystemTime::now(); - let longest_job = JobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); - let longest_job = JobBuilder::from_meta(longest_job).unwrap_one().spawn().await; - let ls = JobBuilder::from_meta(JobMeta::from_shell("ls")?).unwrap_one() - .wait_one() - .await; - let ls = unwrap_enum!(ls, ExecResult::Assigned); - assert_eq!(ls.retcode.unwrap(), 0); - let folders = ls.to_string_result().unwrap(); - let subfolders_jobs: Vec = folders - .lines() - .map(|f| JobMeta::from_shell(format!("ls {}", f)).unwrap()) - .collect(); - let ls_subfolders = JobBuilder::from_meta(subfolders_jobs) - .unwrap_one() - .wait() - .await; - for result in ls_subfolders { - let result = unwrap_enum!(result, ExecResult::Assigned); - assert_eq!(result.retcode.unwrap(), 0); - } - longest_job.wait().await; - assert_eq!(now.elapsed().unwrap().as_secs(), SLEEP_SECS); - Ok(()) - } - /* - #[tokio::test] - async fn test_exec_multiple_jobs_nowait() -> UResult<()> { - const REPEATS: usize = 10; - let job = JobMeta::from_shell("whoami"); - let sleep_jobs: Vec = (0..=REPEATS).map(|_| job.clone()).collect(); - build_jobs(sleep_jobs).spawn().await; - let mut completed = 0; - while completed < REPEATS { - let c = pop_completed().await.len(); - if c > 0 { - completed += c; - println!("{}", c); - } - } - Ok(()) - } - */ - #[tokio::test] - async fn test_failing_shell_job() -> TestResult { - let job = JobMeta::from_shell("lol_kek_puk")?; - let job_result = JobBuilder::from_meta(job) - .unwrap_one() - .wait_one() - .await; - let job_result = unwrap_enum!(job_result, ExecResult::Assigned); - let output = job_result.to_string_result().unwrap(); - assert!(output.contains("No such file")); - assert!(job_result.retcode.is_none()); - Ok(()) - } - - #[test_case( - "/bin/bash {}", - None, - "contains executable" - ; "no binary" - )] - #[test_case( - "/bin/bash", - Some(b"whoami"), - "contains no executable" - ; "no path to binary" - )] - #[tokio::test] - async fn test_job_building_failed(cmd: &str, payload: Option<&[u8]>, err_str: &str) -> TestResult { - let mut job = JobMeta::builder().with_shell(cmd); - if let Some(p) = payload { - job = job.with_payload(p); - } - let err = job.build().unwrap_err(); - let err_msg = unwrap_enum!(err, UError::JobArgsError); - assert!(err_msg.contains(err_str)); - Ok(()) - } - - #[tokio::test] - async fn test_different_job_types() -> TestResult { - let mut jobs = NamedJobBuilder::from_meta(vec![ - ("sleeper", JobMeta::from_shell("sleep 3")?), - ("gatherer", JobMeta::builder().with_type(JobType::Manage).build()?) - ]).wait().await; - let gathered = jobs.pop("gatherer"); - assert_eq!(unwrap_enum!(gathered, ExecResult::Agent).alias, None); - Ok(()) - } - -} \ No newline at end of file diff --git a/lib/u_lib/src/cache.rs b/lib/u_lib/src/cache.rs index a486906..9d173ee 100644 --- a/lib/u_lib/src/cache.rs +++ b/lib/u_lib/src/cache.rs @@ -1,4 +1,5 @@ use crate::models::JobMeta; +use lazy_static::lazy_static; use std::{ collections::HashMap, ops::Deref, @@ -19,11 +20,11 @@ impl JobCache { JOB_CACHE.write().unwrap().insert(job_meta.id, job_meta); } - pub fn contains(uid: &Uuid) -> bool { - JOB_CACHE.read().unwrap().contains_key(uid) + pub fn contains(uid: Uuid) -> bool { + JOB_CACHE.read().unwrap().contains_key(&uid) } - pub fn get(uid: &Uuid) -> Option { + pub fn get<'jh>(uid: Uuid) -> Option> { if !Self::contains(uid) { return None; } @@ -31,17 +32,17 @@ impl JobCache { Some(JobCacheHolder(lock, uid)) } - pub fn remove(uid: &Uuid) { - JOB_CACHE.write().unwrap().remove(uid); + pub fn remove(uid: Uuid) { + JOB_CACHE.write().unwrap().remove(&uid); } } -pub struct JobCacheHolder<'jm>(pub RwLockReadGuard<'jm, Cache>, pub &'jm Uuid); +pub struct JobCacheHolder<'jh>(pub RwLockReadGuard<'jh, Cache>, pub Uuid); -impl<'jm> Deref for JobCacheHolder<'jm> { +impl<'jh> Deref for JobCacheHolder<'jh> { type Target = JobMeta; fn deref(&self) -> &Self::Target { - self.0.get(self.1).unwrap() + self.0.get(&self.1).unwrap() } } diff --git a/lib/u_lib/src/config.rs b/lib/u_lib/src/config.rs index 88fc795..3ab8fce 100644 --- a/lib/u_lib/src/config.rs +++ b/lib/u_lib/src/config.rs @@ -1,8 +1,13 @@ +use lazy_static::lazy_static; use uuid::Uuid; -pub const MASTER_SERVER: &str = "127.0.0.1"; //Ipv4Addr::new(3,9,16,40) pub const MASTER_PORT: u16 = 63714; lazy_static! { - pub static ref UID: Uuid = Uuid::new_v4(); + static ref UID: Uuid = Uuid::new_v4(); +} + +#[inline] +pub fn get_self_uid() -> Uuid { + *UID } diff --git a/lib/u_lib/src/datatypes.rs b/lib/u_lib/src/datatypes.rs index d980734..25b954a 100644 --- a/lib/u_lib/src/datatypes.rs +++ b/lib/u_lib/src/datatypes.rs @@ -4,7 +4,23 @@ use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] #[serde(rename_all = "lowercase")] #[serde(tag = "status", content = "data")] -pub enum DataResult { +pub enum PanelResult { Ok(M), Err(UError), } + +impl PanelResult { + pub fn is_ok(&self) -> bool { + matches!(self, PanelResult::Ok(_)) + } + + pub fn is_err(&self) -> bool { + matches!(self, PanelResult::Err(_)) + } +} + +impl ToString for PanelResult { + fn to_string(&self) -> String { + serde_json::to_string(self).unwrap() + } +} diff --git a/lib/u_lib/src/errors.rs b/lib/u_lib/src/errors.rs deleted file mode 100644 index 206e363..0000000 --- a/lib/u_lib/src/errors.rs +++ /dev/null @@ -1,56 +0,0 @@ -use diesel::result::Error as DslError; -use reqwest::Error as ReqError; -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use uuid::Uuid; - -pub type UResult = std::result::Result; -pub type ULocalResult = std::result::Result; - -#[derive(Error, Debug, Serialize, Deserialize, Clone)] -pub enum UError { - #[error("Error: {0}")] - Raw(String), - - #[error("Connection error: {0}. Body: {1}")] - NetError(String, String), - - #[error("Parse error")] - ParseError, - - #[error("Job error: {0}")] - JobError(String), - - #[error("Argument parsing failed: {0}")] - JobArgsError(String), - - #[error("Job is uncompleted yet")] - JobUncompleted, - - #[error("Job cannot be ran on this platform. Expected: {0}, got: {1}")] - InsuitablePlatform(String, String), - - #[error("Job {0} doesn't exist")] - NoJob(Uuid), - - #[error("Error opening {0}: {1}")] - FSError(String, String), -} - -impl From for UError { - fn from(e: ReqError) -> Self { - UError::NetError(e.to_string(), String::new()) - } -} - -#[derive(Error, Debug)] -pub enum ULocalError { - #[error("{0} is not found")] - NotFound(String), - - #[error("Error processing {0}")] - ProcessingError(String), - - #[error(transparent)] - DBError(#[from] DslError), -} diff --git a/lib/u_lib/src/errors/chan.rs b/lib/u_lib/src/errors/chan.rs new file mode 100644 index 0000000..ef3b7ea --- /dev/null +++ b/lib/u_lib/src/errors/chan.rs @@ -0,0 +1,39 @@ +use crate::UError; +use anyhow::Error; +use once_cell::sync::OnceCell; +use tokio::sync::mpsc::{channel, error::TryRecvError, Receiver, Sender}; +use tokio::sync::{Mutex, MutexGuard}; + +type ChanError = Error; +static ERR_CHAN: OnceCell> = OnceCell::new(); + +pub struct ErrChan { + tx: Sender, + rx: Receiver, +} + +impl ErrChan { + async fn get() -> MutexGuard<'static, Self> { + ERR_CHAN + .get_or_init(|| { + let (tx, rx) = channel(20); + Mutex::new(Self { tx, rx }) + }) + .lock() + .await + } + + pub async fn send(err: impl Into, ctx: impl AsRef) { + let err = err.into(); + error!("Encountered an error at '{}': {:?}", ctx.as_ref(), err); + Self::get().await.tx.try_send(err).unwrap(); + } + + pub async fn recv() -> Option { + match Self::get().await.rx.try_recv() { + Ok(err) => Some(UError::from(err)), + Err(TryRecvError::Disconnected) => panic!("err chan disconnected"), + Err(TryRecvError::Empty) => None, + } + } +} diff --git a/lib/u_lib/src/errors/mod.rs b/lib/u_lib/src/errors/mod.rs new file mode 100644 index 0000000..c910a5a --- /dev/null +++ b/lib/u_lib/src/errors/mod.rs @@ -0,0 +1,5 @@ +mod chan; +mod variants; + +pub use chan::*; +pub use variants::*; diff --git a/lib/u_lib/src/errors/variants.rs b/lib/u_lib/src/errors/variants.rs new file mode 100644 index 0000000..ff65f45 --- /dev/null +++ b/lib/u_lib/src/errors/variants.rs @@ -0,0 +1,71 @@ +#[cfg(not(target_arch = "wasm32"))] +use reqwest::Error as ReqError; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use uuid::Uuid; + +pub type UResult = std::result::Result; + +#[derive(PartialEq, Eq, Error, Debug, Serialize, Deserialize, Clone)] +pub enum UError { + #[error("Runtime error: {0}")] + Runtime(String), + + #[error("Connection error: {0}. Body: {1}")] + NetError(String, String), + + #[error("Parse error")] + ParseError, + + #[error("Job error: {0}")] + JobError(String), + + #[error("Argument parsing failed: {0}")] + JobArgsError(String), + + #[error("Job is uncompleted yet")] + JobUncompleted, + + #[error("Job cannot be ran on this platform. Expected: {0}, got: {1}")] + InsuitablePlatform(String, String), + + #[error("Job {0} doesn't exist")] + NoJob(Uuid), + + #[error("FS error while processing {0}: {1}")] + FSError(String, String), + + #[error("Wrong auth token")] + WrongToken, + + #[error("Panicked: {0}")] + Panic(String), + + #[error("Panel error: {0}")] + PanelError(String), + + #[error("Deserialize from json error: {0}")] + DeserializeError(String), +} + +#[cfg(not(target_arch = "wasm32"))] +impl From for UError { + fn from(e: ReqError) -> Self { + UError::NetError(e.to_string(), String::new()) + } +} + +impl From for UError { + fn from(e: serde_json::Error) -> Self { + UError::DeserializeError(e.to_string()) + } +} + +impl From for UError { + fn from(e: anyhow::Error) -> Self { + match e.downcast::() { + Ok(err) => err, + Err(err) => UError::Runtime(err.to_string()), + } + } +} diff --git a/lib/u_lib/src/executor.rs b/lib/u_lib/src/executor.rs index 1c128e7..169002f 100644 --- a/lib/u_lib/src/executor.rs +++ b/lib/u_lib/src/executor.rs @@ -1,15 +1,16 @@ -use crate::{models::ExecResult, utils::OneOrVec}; +use crate::{models::AssignedJob, UResult}; use futures::{future::BoxFuture, lock::Mutex}; use lazy_static::lazy_static; use std::collections::HashMap; +use std::future::Future; use tokio::{ - spawn, + runtime::Handle, sync::mpsc::{channel, Receiver, Sender}, - task::JoinHandle, + task::{spawn, spawn_blocking, JoinHandle}, }; use uuid::Uuid; -pub type DynFut = BoxFuture<'static, ExecResult>; +pub type ExecResult = UResult; lazy_static! { static ref FUT_RESULTS: Mutex> = Mutex::new(HashMap::new()); @@ -21,33 +22,45 @@ lazy_static! { } struct JoinInfo { - handle: JoinHandle, + handle: JoinHandle>, completed: bool, collectable: bool, // indicates if future can be popped from pool via pop_task_if_completed } +impl JoinInfo { + async fn wait_result(self) -> ExecResult { + self.handle.await.unwrap().await.unwrap() + } +} + fn get_sender() -> Sender { FUT_CHANNEL.0.clone() } pub struct Waiter { - tasks: Vec, + tasks: Vec>, fids: Vec, } impl Waiter { - pub fn new(tasks: impl OneOrVec) -> Self { + pub fn new() -> Self { Self { - tasks: tasks.into_vec(), + tasks: vec![], fids: vec![], } } + pub fn push(&mut self, task: impl Future + Send + 'static) { + self.tasks.push(Box::pin(task)); + } + + /// Spawn prepared tasks pub async fn spawn(mut self) -> Self { let collectable = true; //TODO: self.tasks.len() != 1; for f in self.tasks.drain(..) { - let tx = get_sender(); + let handle = Handle::current(); let fid = Uuid::new_v4(); + let tx = get_sender(); self.fids.push(fid); let task_wrapper = async move { debug!("inside wrapper (started): {}", fid); @@ -55,12 +68,12 @@ impl Waiter { tx.send(fid).await.unwrap(); result }; - let handle = JoinInfo { - handle: spawn(task_wrapper), + let handler = JoinInfo { + handle: spawn_blocking(move || handle.spawn(task_wrapper)), completed: false, collectable, }; - FUT_RESULTS.lock().await.insert(fid, handle); + FUT_RESULTS.lock().await.insert(fid, handler); } self } @@ -72,8 +85,7 @@ impl Waiter { let mut result = vec![]; for fid in self.fids { if let Some(task) = pop_task(fid).await { - let r = task.handle.await; - result.push(r.unwrap()); + result.push(task.wait_result().await); } } result @@ -95,18 +107,17 @@ async fn pop_task(fid: Uuid) -> Option { } pub async fn pop_task_if_completed(fid: Uuid) -> Option { - let &mut JoinInfo { + let &JoinInfo { handle: _, collectable, completed, - } = match FUT_RESULTS.lock().await.get_mut(&fid) { + } = match FUT_RESULTS.lock().await.get(&fid) { Some(t) => t, None => return None, }; if collectable && completed { let task = pop_task(fid).await.unwrap(); - let result = task.handle.await.unwrap(); - Some(result) + Some(task.wait_result().await) } else { None } @@ -118,7 +129,7 @@ pub async fn pop_completed() -> Vec { .lock() .await .keys() - .map(|k| *k) + .copied() .collect::>(); for fid in fids { if let Some(r) = pop_task_if_completed(fid).await { @@ -147,9 +158,9 @@ mod tests { }) }; assert_eq!(0, *val.lock().await); - spawn(async {}).await.ok(); + spawn(async {}).await.unwrap(); assert_eq!(5, *val.lock().await); - t.await.ok(); + t.await.unwrap(); assert_eq!(5, *val.lock().await); } } diff --git a/lib/u_lib/src/lib.rs b/lib/u_lib/src/lib.rs index 55f4e1e..62b1f0b 100644 --- a/lib/u_lib/src/lib.rs +++ b/lib/u_lib/src/lib.rs @@ -1,32 +1,46 @@ #![allow(non_upper_case_globals)] -pub mod api; -pub mod builder; -pub mod cache; -pub mod config; -pub mod datatypes; -pub mod errors; -pub mod executor; -pub mod messaging; -pub mod models; -pub mod utils; +#[cfg(not(target_arch = "wasm32"))] +#[path = "."] +pub mod exports { + pub mod api; + pub mod cache; + pub mod config; + pub mod datatypes; + pub mod errors; + pub mod executor; + pub mod logging; + pub mod messaging; + pub mod models; + pub mod runner; + pub mod utils; +} + +#[cfg(target_arch = "wasm32")] +#[path = "."] +pub mod exports { + pub mod config; + pub mod errors; + pub mod messaging; + pub mod models; + pub mod utils; +} -pub use config::UID; -pub use errors::{UError, ULocalError, ULocalResult, UResult}; +pub use errors::{UError, UResult}; +pub use exports::*; +#[cfg(feature = "server")] pub mod schema_exports { pub use crate::models::{Agentstate, Jobstate, Jobtype}; pub use diesel::sql_types::*; } -#[macro_use] -extern crate lazy_static; - +#[cfg(feature = "server")] #[macro_use] extern crate diesel; #[macro_use] extern crate log; -extern crate env_logger; +#[cfg(test)] #[macro_use] -extern crate mockall; +extern crate rstest; diff --git a/lib/u_lib/src/logging.rs b/lib/u_lib/src/logging.rs new file mode 100644 index 0000000..9c4f1db --- /dev/null +++ b/lib/u_lib/src/logging.rs @@ -0,0 +1,28 @@ +use std::env; +use std::path::Path; + +use tracing_appender::rolling; +use tracing_subscriber::{fmt, prelude::*, registry, EnvFilter}; + +pub fn init_logger(logfile: Option + Send + Sync + 'static>) { + if env::var("RUST_LOG").is_err() { + env::set_var("RUST_LOG", "info") + } + + let reg = registry() + .with(EnvFilter::from_default_env()) + .with(fmt::layer()); + + match logfile { + Some(file) => reg + .with( + fmt::layer() + .with_writer(move || { + rolling::never("logs", file.as_ref().with_extension("log")) + }) + .with_ansi(false), + ) + .init(), + None => reg.init(), + }; +} diff --git a/lib/u_lib/src/messaging.rs b/lib/u_lib/src/messaging/base.rs similarity index 80% rename from lib/u_lib/src/messaging.rs rename to lib/u_lib/src/messaging/base.rs index 5330927..862f1ff 100644 --- a/lib/u_lib/src/messaging.rs +++ b/lib/u_lib/src/messaging/base.rs @@ -1,17 +1,14 @@ -use crate::utils::VecDisplay; -use crate::UID; +use crate::config::get_self_uid; +//use crate::utils::VecDisplay; use serde::{Deserialize, Serialize}; use std::borrow::Cow; -use std::fmt::Display; +//use std::fmt::Display; use uuid::Uuid; pub struct Moo<'cow, T: AsMsg + Clone>(pub Cow<'cow, T>); pub trait AsMsg: Clone + Serialize { - fn as_message<'m>(&'m self) -> BaseMessage<'m, Self> - where - Moo<'m, Self>: From<&'m Self>, - { + fn as_message(&self) -> BaseMessage<'_, Self> { BaseMessage::new(self) } } @@ -31,7 +28,6 @@ impl<'cow, M: AsMsg> From<&'cow M> for Moo<'cow, M> { } impl AsMsg for Vec {} -impl AsMsg for VecDisplay {} impl<'msg, M: AsMsg> AsMsg for &'msg [M] {} #[derive(Serialize, Deserialize, Debug)] @@ -47,7 +43,7 @@ impl<'cow, I: AsMsg> BaseMessage<'cow, I> { { let Moo(inner) = inner.into(); Self { - id: UID.clone(), + id: get_self_uid(), inner, } } diff --git a/lib/u_lib/src/messaging/files.rs b/lib/u_lib/src/messaging/files.rs new file mode 100644 index 0000000..4cdb804 --- /dev/null +++ b/lib/u_lib/src/messaging/files.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +pub struct DownloadInfo { + hashsum: String, + dl_fid: Uuid, +} diff --git a/lib/u_lib/src/messaging/mod.rs b/lib/u_lib/src/messaging/mod.rs new file mode 100644 index 0000000..f1e82e5 --- /dev/null +++ b/lib/u_lib/src/messaging/mod.rs @@ -0,0 +1,28 @@ +mod base; +mod files; + +use crate::models::*; +use crate::UError; +pub use base::{AsMsg, BaseMessage}; +pub use files::*; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +impl AsMsg for Agent {} +impl AsMsg for AssignedJob {} +impl AsMsg for AssignedJobById {} +impl AsMsg for DownloadInfo {} +impl AsMsg for Reportable {} +impl AsMsg for JobMeta {} +impl AsMsg for String {} +impl AsMsg for Uuid {} +impl AsMsg for i32 {} +impl AsMsg for u8 {} +impl AsMsg for () {} + +#[derive(Serialize, Deserialize, Clone, PartialEq, Debug)] +pub enum Reportable { + Assigned(AssignedJob), + Dummy, + Error(UError), +} diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index 16a78db..ba28da5 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -1,23 +1,25 @@ +#[cfg(feature = "server")] use diesel::{AsChangeset, Identifiable, Insertable, Queryable}; +#[cfg(feature = "server")] use diesel_derive_enum::DbEnum; use serde::{Deserialize, Serialize}; -use std::{fmt, time::SystemTime}; +use std::time::SystemTime; use strum::Display; -use crate::{ - builder::NamedJobBuilder, - models::{schema::*, ExecResult}, - unwrap_enum, - utils::systime_to_string, - UID, -}; +#[cfg(feature = "server")] +use crate::models::schema::*; + +use crate::{config::get_self_uid, executor::ExecResult, runner::NamedJobRunner, utils::Platform}; -use guess_host_triple::guess_host_triple; use uuid::Uuid; -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, DbEnum, Display)] -#[PgType = "AgentState"] -#[DieselType = "Agentstate"] +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Display)] +#[cfg_attr( + feature = "server", + derive(DbEnum), + PgType = "AgentState", + DieselType = "Agentstate" +)] pub enum AgentState { New, Active, @@ -25,22 +27,19 @@ pub enum AgentState { } //belongs_to -#[derive( - Clone, - Debug, - Serialize, - Deserialize, - Identifiable, - Queryable, - Insertable, - AsChangeset, - PartialEq, +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[cfg_attr( + feature = "server", + derive(Identifiable, Queryable, Insertable, AsChangeset), + table_name = "agents" )] -#[table_name = "agents"] pub struct Agent { pub alias: Option, pub hostname: String, + pub host_info: String, pub id: Uuid, + pub ip_gray: Option, + pub ip_white: Option, pub is_root: bool, pub is_root_allowed: bool, pub last_active: SystemTime, @@ -51,23 +50,7 @@ pub struct Agent { pub username: String, } -impl fmt::Display for Agent { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut out = format!("Agent: {}", self.id); - if self.alias.is_some() { - out += &format!(" ({})", self.alias.as_ref().unwrap()) - } - out += &format!("\nUsername: {}", self.username); - out += &format!("\nHostname: {}", self.hostname); - out += &format!("\nIs root: {}", self.is_root); - out += &format!("\nRoot allowed: {}", self.is_root_allowed); - out += &format!("\nLast active: {}", systime_to_string(&self.last_active)); - out += &format!("\nPlatform: {}", self.platform); - out += &format!("\nState: {}", self.state); - write!(f, "{}", out) - } -} - +#[cfg(not(target_arch = "wasm32"))] impl Agent { pub fn with_id(uid: Uuid) -> Self { Self { @@ -76,32 +59,36 @@ impl Agent { } } + pub fn touch(&mut self) { + self.last_active = SystemTime::now(); + } + + #[cfg(unix)] pub async fn gather() -> Self { - let mut builder = NamedJobBuilder::from_shell(vec![ - ("hostname", "hostname"), + let mut builder = NamedJobRunner::from_shell(vec![ + ("hostname", "uname -a"), + ("host_info", "hostnamectl --json=pretty"), ("is_root", "id -u"), ("username", "id -un"), ]) .unwrap_one() .wait() .await; - let decoder = |job_result: ExecResult| { - let assoc_job = unwrap_enum!(job_result, ExecResult::Assigned); - assoc_job.to_string_result().unwrap().trim().to_string() - }; + let decoder = + |job_result: ExecResult| job_result.unwrap().to_str_result().trim().to_string(); Self { hostname: decoder(builder.pop("hostname")), + host_info: decoder(builder.pop("host_info")), is_root: &decoder(builder.pop("is_root")) == "0", username: decoder(builder.pop("username")), - platform: guess_host_triple().unwrap_or("unknown").to_string(), + platform: Platform::current_as_string(), ..Default::default() } } - pub async fn run() -> ExecResult { - #[cfg(unix)] - ExecResult::Agent(Agent::gather().await) + pub async fn run() -> Agent { + Agent::gather().await } } @@ -109,8 +96,9 @@ impl Default for Agent { fn default() -> Self { Self { alias: None, - id: UID.clone(), + id: get_self_uid(), hostname: String::new(), + host_info: String::new(), is_root: false, is_root_allowed: false, last_active: SystemTime::now(), @@ -119,17 +107,8 @@ impl Default for Agent { state: AgentState::New, token: None, username: String::new(), + ip_gray: None, + ip_white: None, } } } - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_gather() { - let cli_info = Agent::gather().await; - assert_eq!(cli_info.alias, None) - } -} diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index b11c50b..112671d 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -1,28 +1,20 @@ -use super::JobState; -use crate::{ - cache::JobCache, - models::{schema::*, ExecResult, JobOutput}, - utils::{systime_to_string, TempFile}, - UID, -}; +use super::{JobMeta, JobState, JobType}; +#[cfg(not(target_arch = "wasm32"))] +use crate::config::get_self_uid; +#[cfg(feature = "server")] +use crate::models::schema::*; +#[cfg(feature = "server")] use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; -use std::{fmt, process::Output, string::FromUtf8Error, time::SystemTime}; -use tokio::process::Command; +use std::{borrow::Cow, time::SystemTime}; use uuid::Uuid; -#[derive( - Serialize, - Deserialize, - Clone, - Debug, - Queryable, - Identifiable, - Insertable, - AsChangeset, - PartialEq, +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] +#[cfg_attr( + feature = "server", + derive(Queryable, Identifiable, Insertable, AsChangeset), + table_name = "results" )] -#[table_name = "results"] pub struct AssignedJob { pub agent_id: Uuid, pub alias: Option, @@ -31,32 +23,50 @@ pub struct AssignedJob { pub job_id: Uuid, pub result: Option>, pub state: JobState, + pub exec_type: JobType, pub retcode: Option, pub updated: SystemTime, } -impl fmt::Display for AssignedJob { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut out = format!("Result: {}", self.id); - out += &format!("\nAgent: {}", self.agent_id); - out += &format!("\nJob: {}", self.job_id); - if self.alias.is_some() { - out += &format!("\nAlias: {}", self.alias.as_ref().unwrap()); +#[derive(Serialize, Deserialize, Clone, Copy)] +pub struct AssignedJobById { + pub agent_id: Uuid, + pub id: Uuid, + pub job_id: Uuid, +} + +impl From<(&JobMeta, AssignedJobById)> for AssignedJob { + fn from((meta, pj): (&JobMeta, AssignedJobById)) -> Self { + AssignedJob { + id: pj.id, + agent_id: pj.agent_id, + job_id: pj.job_id, + alias: meta.alias.clone(), + exec_type: meta.exec_type, + ..Default::default() } - out += &format!("\nUpdated: {}", systime_to_string(&self.updated)); - out += &format!("\nState: {}", self.state); - if self.state == JobState::Finished { - if self.retcode.is_some() { - out += &format!("\nReturn code: {}", self.retcode.unwrap()); - } - if self.result.is_some() { - out += &format!( - "\nResult: {}", - String::from_utf8_lossy(self.result.as_ref().unwrap()) - ); - } + } +} + +impl From<&JobMeta> for AssignedJob { + fn from(meta: &JobMeta) -> Self { + AssignedJob { + job_id: meta.id, + agent_id: get_self_uid(), + alias: meta.alias.clone(), + exec_type: meta.exec_type, + ..Default::default() + } + } +} + +impl Default for AssignedJobById { + fn default() -> Self { + Self { + agent_id: get_self_uid(), + id: Uuid::new_v4(), + job_id: Uuid::nil(), } - write!(f, "{}", out) } } @@ -72,80 +82,28 @@ impl Default for AssignedJob { state: JobState::Queued, retcode: None, updated: SystemTime::now(), + exec_type: JobType::default(), } } } impl AssignedJob { - pub async fn run(mut self) -> ExecResult { - let (argv, _payload) = { - let meta = JobCache::get(&self.job_id).unwrap(); - let extracted_payload = meta - .payload - .as_ref() - .and_then(|p| TempFile::write_exec(p).ok()); - let argv = if let Some(ref p) = &extracted_payload { - meta.argv.replace("{}", &p.get_path()) - } else { - meta.argv.clone() - }; - (argv, extracted_payload) - }; - let mut split_cmd = shlex::split(&argv).unwrap().into_iter(); - let cmd = split_cmd.nth(0).unwrap(); - let args = split_cmd.collect::>(); - let cmd_result = Command::new(cmd).args(args).output().await; - let (data, retcode) = match cmd_result { - Ok(Output { - status, - stdout, - stderr, - }) => ( - JobOutput::new() - .stdout(stdout) - .stderr(stderr) - .into_combined(), - status.code(), - ), - Err(e) => ( - JobOutput::new() - .stderr(e.to_string().into_bytes()) - .into_combined(), - None, - ), - }; - self.result = Some(data); - self.retcode = retcode; - self.updated = SystemTime::now(); - self.state = JobState::Finished; - ExecResult::Assigned(self) - } - - pub fn new(job_id: Uuid, other: Option<&Self>) -> Self { - Self { - agent_id: *UID, - job_id, - ..other.unwrap_or(&Default::default()).clone() + pub fn to_raw_result(&self) -> &[u8] { + match self.result.as_ref() { + Some(r) => r, + None => b"No data yet", } } - pub fn as_job_output(&self) -> Option { - self.result - .as_ref() - .and_then(|r| JobOutput::from_combined(r)) + pub fn to_str_result(&self) -> Cow<'_, str> { + String::from_utf8_lossy(self.to_raw_result()) } - pub fn to_raw_result(&self) -> Vec { - match self.result.as_ref() { - Some(r) => match JobOutput::from_combined(r) { - Some(o) => o.to_appropriate(), - None => r.clone(), - }, - None => b"No data".to_vec(), - } + pub fn set_result(&mut self, result: &S) { + self.result = Some(serde_json::to_vec(result).unwrap()); } - pub fn to_string_result(&self) -> Result { - String::from_utf8(self.to_raw_result()) + pub fn touch(&mut self) { + self.updated = SystemTime::now() } } diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 50e1b65..c8980b7 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -1,23 +1,49 @@ use super::JobType; -use crate::{models::schema::*, UError, UResult}; +#[cfg(feature = "server")] +use crate::models::schema::*; +use crate::utils::Platform; +use crate::{UError, UResult}; +#[cfg(feature = "server")] use diesel::{Identifiable, Insertable, Queryable}; -use guess_host_triple::guess_host_triple; use serde::{Deserialize, Serialize}; -use std::fmt; +use std::fs; use uuid::Uuid; -#[derive(Serialize, Deserialize, Clone, Debug, Queryable, Identifiable, Insertable)] -#[table_name = "jobs"] +#[derive(Serialize, Deserialize, Clone, Debug)] +#[cfg_attr( + feature = "server", + derive(Queryable, Identifiable, Insertable, AsChangeset), + table_name = "jobs" +)] pub struct JobMeta { + #[serde(default)] pub alias: Option, + /// string like `bash -c {} -a 1 --arg2`, /// where {} is replaced by executable's tmp path + #[serde(default)] pub argv: String, + + #[serde(default = "Uuid::new_v4")] pub id: Uuid, + + #[serde(default)] pub exec_type: JobType, - //pub schedule: JobSchedule, + + ///target triple + #[serde(default = "Platform::current_as_string")] pub platform: String, + + #[serde(default)] pub payload: Option>, + + /// if payload should be read from external resource + #[serde(default)] + pub payload_path: Option, + + ///cron-like string + #[serde(default)] + pub schedule: Option, } impl JobMeta { @@ -25,42 +51,12 @@ impl JobMeta { JobMetaBuilder::default() } - pub fn from_shell>(cmd: S) -> UResult { - Self::builder().with_shell(cmd).build() + pub fn validated(self) -> UResult { + JobMetaBuilder { inner: self }.build() } -} -impl fmt::Display for JobMeta { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut out = format!("Job: {}", self.id); - if self.alias.is_some() { - out += &format!(" ({})", self.alias.as_ref().unwrap()); - } - out += &format!("\nArgv: {}", self.argv); - out += &format!("\nExecutable type: {}", self.exec_type); - out += &format!("\nPlatform: {}", self.platform); - if self.exec_type == JobType::Shell && self.payload.is_some() { - let payload = self.payload.as_ref().unwrap(); - let pld_len = { - let pl = payload.len(); - if pl > 20 { - 20 - } else { - pl - } - }; - let pld_beginning = payload - .iter() - .take(pld_len) - .map(|u| *u) - .collect::>(); - out += &format!( - "\nPayload: {}{}", - String::from_utf8_lossy(&pld_beginning), - if pld_len <= 20 { "" } else { " <...>" } - ); - } - write!(f, "{}", out) + pub fn from_shell(cmd: impl Into) -> UResult { + Self::builder().with_shell(cmd).build() } } @@ -69,38 +65,39 @@ impl Default for JobMeta { Self { id: Uuid::new_v4(), alias: None, - argv: String::from("/bin/bash -c {}"), + argv: String::new(), exec_type: JobType::Shell, - platform: guess_host_triple().unwrap_or("unknown").to_string(), + platform: Platform::current_as_string(), payload: None, + schedule: None, + payload_path: None, } } } +#[derive(Default)] pub struct JobMetaBuilder { inner: JobMeta, } -impl Default for JobMetaBuilder { - fn default() -> Self { - Self { - inner: JobMeta::default(), - } - } -} - impl JobMetaBuilder { - pub fn with_shell>(mut self, shell_cmd: S) -> Self { + pub fn with_shell(mut self, shell_cmd: impl Into) -> Self { self.inner.argv = shell_cmd.into(); + self.inner.exec_type = JobType::Shell; self } - pub fn with_payload>>(mut self, payload: C) -> Self { + pub fn with_payload(mut self, payload: impl Into>) -> Self { self.inner.payload = Some(payload.into()); self } - pub fn with_alias>(mut self, alias: S) -> Self { + pub fn with_payload_src(mut self, path: impl Into) -> Self { + self.inner.payload_path = Some(path.into()); + self + } + + pub fn with_alias(mut self, alias: impl Into) -> Self { self.inner.alias = Some(alias.into()); self } @@ -111,23 +108,31 @@ impl JobMetaBuilder { } pub fn build(self) -> UResult { - let inner = self.inner; + let mut inner = self.inner; match inner.exec_type { JobType::Shell => { + if inner.argv.is_empty() { + // TODO: fix detecting + inner.argv = String::from("echo 'hello, world!'") + } let argv_parts = shlex::split(&inner.argv).ok_or(UError::JobArgsError("Shlex failed".into()))?; let empty_err = UError::JobArgsError("Empty argv".into()); - if argv_parts.get(0).ok_or(empty_err.clone())?.len() == 0 { - return Err(empty_err); + if argv_parts.get(0).ok_or(empty_err.clone())?.is_empty() { + return Err(empty_err.into()); + } + if let Some(path) = &inner.payload_path { + let data = fs::read(path) + .map_err(|e| UError::FSError(path.to_string(), e.to_string()))?; + inner.payload = Some(data) } match inner.payload.as_ref() { - Some(_) => { + Some(p) if p.len() > 0 => { if !inner.argv.contains("{}") { return Err(UError::JobArgsError( "Argv contains no executable placeholder".into(), - )); - } else { - () + ) + .into()); } } None => { @@ -135,26 +140,21 @@ impl JobMetaBuilder { return Err(UError::JobArgsError( "No payload provided, but argv contains executable placeholder" .into(), - )); - } else { - () + ) + .into()); } } + _ => (), }; - Ok(inner) + if !Platform::new(&inner.platform).check() { + return Err(UError::JobArgsError(format!( + "Unknown platform {}", + inner.platform + ))); + } + Ok(inner.into()) } - JobType::Manage => Ok(inner), - _ => todo!(), + _ => Ok(inner.into()), } } - /* - pub fn from_file(path: PathBuf) -> UResult { - let data = fs::read(path) - .map_err(|e| UError::FilesystemError( - path.to_string_lossy().to_string(), - e.to_string() - ))?; - let filename = path.file_name().unwrap().to_str().unwrap(); - - }*/ } diff --git a/lib/u_lib/src/models/jobs/misc.rs b/lib/u_lib/src/models/jobs/misc.rs index 7451548..ec9b866 100644 --- a/lib/u_lib/src/models/jobs/misc.rs +++ b/lib/u_lib/src/models/jobs/misc.rs @@ -1,186 +1,37 @@ +#[cfg(feature = "server")] use diesel_derive_enum::DbEnum; use serde::{Deserialize, Serialize}; use strum::Display; -#[derive(Serialize, Deserialize, Clone, Debug)] -pub enum ManageAction { - Ping, - UpdateAvailable, - JobsResultsRequest, - Terminate, -} - -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] -pub enum JobSchedule { - Once, - Permanent, - //Scheduled -} - -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, DbEnum, Display)] -#[PgType = "JobState"] -#[DieselType = "Jobstate"] +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Display)] +#[cfg_attr( + feature = "server", + derive(DbEnum), + PgType = "JobState", + DieselType = "Jobstate" +)] pub enum JobState { - Queued, // server created a job, but client didn't get it yet - //Pending, // client got a job, but not running yet - Running, // client is currently running a job + /// server created a job, but client didn't get it yet + Queued, + + // client got a job, but not running yet + //Pending, + /// client is currently running a job + Running, Finished, } -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, DbEnum, Display)] -#[PgType = "JobType"] -#[DieselType = "Jobtype"] +#[derive(Default, Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Display)] +#[cfg_attr( + feature = "server", + derive(DbEnum), + PgType = "JobType", + DieselType = "Jobtype" +)] pub enum JobType { - Manage, + Init, + #[default] Shell, - Python, -} - -#[derive(Clone, Debug)] -pub struct JobOutput { - pub stdout: Vec, - pub stderr: Vec, -} - -impl JobOutput { - const STREAM_BORDER: &'static str = "***"; - const STDOUT: &'static str = "STDOUT"; - const STDERR: &'static str = "STDERR"; - - #[inline] - fn create_delim(header: &'static str) -> Vec { - format!( - "<{border}{head}{border}>", - border = Self::STREAM_BORDER, - head = header - ) - .into_bytes() - } - - pub fn new() -> Self { - Self { - stdout: vec![], - stderr: vec![], - } - } - - pub fn stdout(mut self, data: Vec) -> Self { - self.stdout = data; - self - } - - pub fn stderr(mut self, data: Vec) -> Self { - self.stderr = data; - self - } - - pub fn into_combined(self) -> Vec { - let mut result: Vec = vec![]; - if self.stdout.len() > 0 { - result.extend(Self::create_delim(Self::STDOUT)); - result.extend(self.stdout); - } - - if self.stderr.len() > 0 { - result.extend(Self::create_delim(Self::STDERR)); - result.extend(self.stderr); - } - result - } - - pub fn from_combined(raw: &[u8]) -> Option { - enum ParseFirst { - Stdout, - Stderr, - } - fn split_by_subslice<'s>(slice: &'s [u8], subslice: &[u8]) -> Option<(&'s [u8], &'s [u8])> { - slice - .windows(subslice.len()) - .position(|w| w == subslice) - .map(|split_pos| { - let splitted = slice.split_at(split_pos); - (&splitted.0[..split_pos], &splitted.1[subslice.len()..]) - }) - } - let splitter = |p: ParseFirst| { - let (first_hdr, second_hdr) = match p { - ParseFirst::Stdout => (Self::STDOUT, Self::STDERR), - ParseFirst::Stderr => (Self::STDERR, Self::STDOUT), - }; - let first_hdr = Self::create_delim(first_hdr); - let second_hdr = Self::create_delim(second_hdr); - split_by_subslice(raw, &first_hdr).map(|(_, p2)| { - match split_by_subslice(p2, &second_hdr) { - Some((p2_1, p2_2)) => Self::new().stdout(p2_1.to_vec()).stderr(p2_2.to_vec()), - None => Self::new().stdout(p2.to_vec()), - } - }) - }; - splitter(ParseFirst::Stdout).or(splitter(ParseFirst::Stderr)) - } - - pub fn to_appropriate(&self) -> Vec { - let mut result: Vec = vec![]; - if self.stdout.len() > 0 { - result.extend(&self.stdout); - } - if self.stderr.len() > 0 { - if result.len() > 0 { - result.push(b'\n'); - } - result.extend(&self.stderr); - } - if result.len() == 0 { - result.extend(b"No data"); - } - result - } -} - -#[cfg(test)] -mod tests { - use crate::{models::JobOutput, utils::bytes_to_string}; - use test_case::test_case; - - const STDOUT: &str = "<***STDOUT***>"; - const STDERR: &str = "<***STDERR***>"; - - #[test_case( - "lol", - "kek", - &format!("{}lol{}kek", STDOUT, STDERR) - ;"stdout stderr" - )] - #[test_case( - "", - "kek", - &format!("{}kek", STDERR) - ;"stderr" - )] - fn test_to_combined(stdout: &str, stderr: &str, result: &str) { - let output = JobOutput::new() - .stdout(stdout.as_bytes().to_vec()) - .stderr(stderr.as_bytes().to_vec()); - assert_eq!(&bytes_to_string(&output.into_combined()), result) - } - - #[test_case( - &format!("{}lal{}kik", STDOUT, STDERR), - "lal\nkik" - ;"stdout stderr" - )] - #[test_case( - &format!("{}qeq", STDOUT), - "qeq" - ;"stdout" - )] - #[test_case( - &format!("{}vev", STDERR), - "vev" - ;"stderr" - )] - fn test_from_combined(src: &str, result: &str) { - let output = JobOutput::from_combined(src.as_bytes()).unwrap(); - assert_eq!(bytes_to_string(&output.to_appropriate()).trim(), result); - } + Terminate, + Update, } diff --git a/lib/u_lib/src/models/mod.rs b/lib/u_lib/src/models/mod.rs index 29b14de..bb6e50e 100644 --- a/lib/u_lib/src/models/mod.rs +++ b/lib/u_lib/src/models/mod.rs @@ -1,29 +1,6 @@ mod agent; -pub mod jobs; -mod result; +mod jobs; +#[cfg(feature = "server")] pub mod schema; -use crate::messaging::AsMsg; -pub use crate::models::result::ExecResult; pub use crate::models::{agent::*, jobs::*}; -use serde::{Deserialize, Serialize}; -use std::fmt; -use uuid::Uuid; - -impl AsMsg for Agent {} -impl AsMsg for AssignedJob {} -impl AsMsg for ExecResult {} -impl AsMsg for JobMeta {} -impl AsMsg for String {} -impl AsMsg for Uuid {} -impl AsMsg for Empty {} -impl AsMsg for i32 {} - -#[derive(Serialize, Deserialize, Clone, Default, Debug)] -pub struct Empty; - -impl fmt::Display for Empty { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "") - } -} diff --git a/lib/u_lib/src/models/result.rs b/lib/u_lib/src/models/result.rs deleted file mode 100644 index 849405a..0000000 --- a/lib/u_lib/src/models/result.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::models::{Agent, AssignedJob}; -use serde::{Deserialize, Serialize}; - -#[derive(Serialize, Deserialize, Clone, PartialEq)] -pub enum ExecResult { - Assigned(AssignedJob), - Agent(Agent), - Dummy, -} diff --git a/lib/u_lib/src/models/schema.rs b/lib/u_lib/src/models/schema.rs index be2edf1..11da1d0 100644 --- a/lib/u_lib/src/models/schema.rs +++ b/lib/u_lib/src/models/schema.rs @@ -1,10 +1,15 @@ -table! { +// @generated automatically by Diesel CLI. + +diesel::table! { use crate::schema_exports::*; agents (id) { alias -> Nullable, hostname -> Text, + host_info -> Text, id -> Uuid, + ip_gray -> Nullable, + ip_white -> Nullable, is_root -> Bool, is_root_allowed -> Bool, last_active -> Timestamp, @@ -16,7 +21,7 @@ table! { } } -table! { +diesel::table! { use crate::schema_exports::*; certificates (id) { @@ -26,22 +31,7 @@ table! { } } -table! { - use crate::schema_exports::*; - - ip_addrs (id) { - agent_id -> Uuid, - check_ts -> Timestamp, - gateway -> Nullable, - id -> Uuid, - iface -> Text, - ip_addr -> Text, - is_gray -> Bool, - netmask -> Text, - } -} - -table! { +diesel::table! { use crate::schema_exports::*; jobs (id) { @@ -51,10 +41,12 @@ table! { exec_type -> Jobtype, platform -> Text, payload -> Nullable, + payload_path -> Nullable, + schedule -> Nullable, } } -table! { +diesel::table! { use crate::schema_exports::*; results (id) { @@ -65,20 +57,14 @@ table! { job_id -> Uuid, result -> Nullable, state -> Jobstate, + exec_type -> Jobtype, retcode -> Nullable, updated -> Timestamp, } } -joinable!(certificates -> agents (agent_id)); -joinable!(ip_addrs -> agents (agent_id)); -joinable!(results -> agents (agent_id)); -joinable!(results -> jobs (job_id)); +diesel::joinable!(certificates -> agents (agent_id)); +diesel::joinable!(results -> agents (agent_id)); +diesel::joinable!(results -> jobs (job_id)); -allow_tables_to_appear_in_same_query!( - agents, - certificates, - ip_addrs, - jobs, - results, -); +diesel::allow_tables_to_appear_in_same_query!(agents, certificates, jobs, results,); diff --git a/lib/u_lib/src/runner.rs b/lib/u_lib/src/runner.rs new file mode 100644 index 0000000..2025abd --- /dev/null +++ b/lib/u_lib/src/runner.rs @@ -0,0 +1,362 @@ +use crate::{ + cache::JobCache, + executor::{ExecResult, Waiter}, + models::{Agent, AssignedJob, AssignedJobById, JobMeta, JobType}, + utils::{CombinedResult, OneOrVec, Platform}, + utils::{ProcOutput, TempFile}, + UError, UResult, +}; +use std::collections::HashMap; +use std::process::exit; +use tokio::process::Command; + +pub struct JobRunner { + waiter: Waiter, + is_running: bool, +} + +impl JobRunner { + pub fn from_jobs(jobs: impl OneOrVec) -> CombinedResult { + let jobs = jobs.into_vec(); + let mut waiter = Waiter::new(); + let mut result = CombinedResult::new(); + for job in jobs { + //waiting for try-blocks stabilization + let built_job: UResult<()> = (|| { + let meta = JobCache::get(job.job_id).ok_or(UError::NoJob(job.job_id))?; + let curr_platform = Platform::current(); + if !curr_platform.matches(&meta.platform) { + return Err(UError::InsuitablePlatform( + meta.platform.clone(), + curr_platform.into_string(), + )); + } + let job = AssignedJob::from((&*meta, job)); + waiter.push(run_assigned_job(job)); + Ok(()) + })(); + if let Err(e) = built_job { + result.err(e) + } + } + result.ok(Self { + waiter, + is_running: false, + }); + result + } + + pub fn from_meta(metas: impl OneOrVec) -> CombinedResult { + let jobs = metas + .into_vec() + .into_iter() + .map(|jm| { + let job_id = jm.id; + if !JobCache::contains(job_id) { + JobCache::insert(jm); + } + AssignedJobById { + job_id, + ..Default::default() + } + }) + .collect::>(); + JobRunner::from_jobs(jobs) + } + + /// Spawn jobs + pub async fn spawn(mut self) -> Self { + self.waiter = self.waiter.spawn().await; + self.is_running = true; + self + } + + /// Spawn jobs and wait for result + pub async fn wait(self) -> Vec { + let waiter = if !self.is_running { + self.spawn().await.waiter + } else { + self.waiter + }; + waiter.wait().await + } + + /// Spawn one job and wait for result + pub async fn wait_one(self) -> ExecResult { + self.wait().await.pop().unwrap() + } +} + +pub async fn run_assigned_job(mut job: AssignedJob) -> ExecResult { + match job.exec_type { + JobType::Shell => { + let (argv, _payload) = { + let meta = JobCache::get(job.job_id).unwrap(); + if let Some(ref payload) = meta.payload { + let extracted_payload = match TempFile::write_exec(payload) { + Ok(p) => p, + Err(e) => return Err(UError::Runtime(e.to_string())), + }; + ( + meta.argv.replace("{}", &extracted_payload.get_path()), + Some(extracted_payload), + ) + } else { + (meta.argv.clone(), None) + } + }; + let mut split_cmd = shlex::split(&argv).unwrap().into_iter(); + let cmd = split_cmd.nth(0).unwrap(); + let args = split_cmd.collect::>(); + let cmd_result = Command::new(cmd).args(args).output().await; + let (data, retcode) = match cmd_result { + Ok(output) => ( + ProcOutput::from_output(&output).into_vec(), + output.status.code(), + ), + Err(e) => ( + ProcOutput::new() + .stderr(e.to_string().into_bytes()) + .into_vec(), + None, + ), + }; + job.result = Some(data); + job.retcode = retcode; + } + JobType::Init => { + job.set_result(&Agent::run().await); + job.retcode = Some(0); + } + JobType::Update => todo!(), + JobType::Terminate => exit(0), + }; + Ok(job) +} + +/// Store jobs and get results by name +pub struct NamedJobRunner { + runner: Option, + job_names: Vec<&'static str>, + results: HashMap<&'static str, ExecResult>, +} + +impl NamedJobRunner { + pub fn from_shell( + named_jobs: impl OneOrVec<(&'static str, &'static str)>, + ) -> CombinedResult { + let mut result = CombinedResult::new(); + let jobs: Vec<(&'static str, JobMeta)> = named_jobs + .into_vec() + .into_iter() + .filter_map( + |(alias, cmd)| match JobMeta::builder().with_shell(cmd).build() { + Ok(meta) => Some((alias, meta)), + Err(e) => { + result.err(e); + None + } + }, + ) + .collect(); + result.ok(Self::from_meta(jobs)); + result + } + + pub fn from_meta(named_jobs: impl OneOrVec<(&'static str, JobMeta)>) -> Self { + let mut job_names = vec![]; + let job_metas: Vec = named_jobs + .into_vec() + .into_iter() + .map(|(alias, mut meta)| { + job_names.push(alias); + meta.alias = Some(alias.to_string()); + meta + }) + .collect(); + Self { + runner: Some(JobRunner::from_meta(job_metas).unwrap_one()), + job_names, + results: HashMap::new(), + } + } + + pub async fn wait(mut self) -> Self { + let results = self.runner.take().unwrap().wait().await; + for (name, result) in self.job_names.iter().zip(results.into_iter()) { + self.results.insert(name, result); + } + self + } + + pub fn pop_opt(&mut self, name: &'static str) -> Option { + self.results.remove(name) + } + + pub fn pop(&mut self, name: &'static str) -> ExecResult { + self.pop_opt(name).unwrap() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + models::{misc::JobType, JobMeta}, + runner::{JobRunner, NamedJobRunner}, + unwrap_enum, + }; + use std::time::SystemTime; + + type TestResult = Result>; + + #[tokio::test] + async fn test_is_really_async() { + const SLEEP_SECS: u64 = 1; + let job = JobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); + let sleep_jobs: Vec = (0..50).map(|_| job.clone()).collect(); + let now = SystemTime::now(); + JobRunner::from_meta(sleep_jobs).unwrap_one().wait().await; + assert!(now.elapsed().unwrap().as_secs() < SLEEP_SECS + 2) + } + + #[rstest] + #[case::sh_payload( + "/bin/sh {}", + Some(b"echo test01 > /tmp/asd; cat /tmp/asd".as_slice()), + "test01" + )] + #[case::python_cmd(r#"/usr/bin/python3 -c 'print("test02")'"#, None, "test02")] + #[case::sh_multiline_payload( + "/{}", + Some( + br#"#!/bin/sh + TMPPATH=/tmp/lol + mkdir -p $TMPPATH + echo test03 > $TMPPATH/t + cat $TMPPATH/t + rm -rf $TMPPATH"#.as_slice() + ), + "test03" + )] + #[case::standalone_binary_with_args( + "/{} 'some msg as arg'", + Some(include_bytes!("../tests/fixtures/echoer").as_slice()), + "some msg as arg" + )] + #[tokio::test] + async fn test_shell_job( + #[case] cmd: &str, + #[case] payload: Option<&[u8]>, + #[case] expected_result: &str, + ) -> TestResult { + let mut job = JobMeta::builder().with_shell(cmd); + if let Some(p) = payload { + job = job.with_payload(p); + } + let job = job.build().unwrap(); + let result = JobRunner::from_meta(job) + .unwrap_one() + .wait_one() + .await + .unwrap(); + let result = result.to_str_result(); + assert_eq!(result.trim(), expected_result); + Ok(()) + } + + #[tokio::test] + async fn test_complex_load() -> TestResult { + const SLEEP_SECS: u64 = 1; + let now = SystemTime::now(); + let longest_job = JobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); + let longest_job = JobRunner::from_meta(longest_job).unwrap_one().spawn().await; + let ls = JobRunner::from_meta(JobMeta::from_shell("ls")?) + .unwrap_one() + .wait_one() + .await + .unwrap(); + assert_eq!(ls.retcode.unwrap(), 0); + let folders = ls.to_str_result(); + let subfolders_jobs: Vec = folders + .lines() + .map(|f| JobMeta::from_shell(format!("ls {}", f)).unwrap()) + .collect(); + let ls_subfolders = JobRunner::from_meta(subfolders_jobs) + .unwrap_one() + .wait() + .await; + for result in ls_subfolders { + assert_eq!(result.unwrap().retcode.unwrap(), 0); + } + longest_job.wait().await; + assert_eq!(now.elapsed().unwrap().as_secs(), SLEEP_SECS); + Ok(()) + } + /* + #[tokio::test] + async fn test_exec_multiple_jobs_nowait() -> UResult<()> { + const REPEATS: usize = 10; + let job = JobMeta::from_shell("whoami"); + let sleep_jobs: Vec = (0..=REPEATS).map(|_| job.clone()).collect(); + build_jobs(sleep_jobs).spawn().await; + let mut completed = 0; + while completed < REPEATS { + let c = pop_completed().await.len(); + if c > 0 { + completed += c; + println!("{}", c); + } + } + Ok(()) + } + */ + #[tokio::test] + async fn test_failing_shell_job() -> TestResult { + let job = JobMeta::from_shell("lol_kek_puk")?; + let job_result = JobRunner::from_meta(job) + .unwrap_one() + .wait_one() + .await + .unwrap(); + let output = job_result.to_str_result(); + assert!(output.contains("No such file")); + assert!(job_result.retcode.is_none()); + Ok(()) + } + + #[rstest] + #[case::no_binary("/bin/bash {}", None, "contains executable")] + #[case::no_path_to_binary("/bin/bash", Some(b"whoami".as_slice()), "contains no executable")] + #[tokio::test] + async fn test_job_building_failed( + #[case] cmd: &str, + #[case] payload: Option<&[u8]>, + #[case] err_str: &str, + ) -> TestResult { + let mut job = JobMeta::builder().with_shell(cmd); + if let Some(p) = payload { + job = job.with_payload(p); + } + let err = job.build().unwrap_err(); + let err_msg = unwrap_enum!(err, UError::JobArgsError); + assert!(err_msg.contains(err_str)); + Ok(()) + } + + #[tokio::test] + async fn test_different_job_types() -> TestResult { + let mut jobs = NamedJobRunner::from_meta(vec![ + ("sleeper", JobMeta::from_shell("sleep 3")?), + ( + "gatherer", + JobMeta::builder().with_type(JobType::Init).build()?, + ), + ]) + .wait() + .await; + let gathered = jobs.pop("gatherer"); + assert_eq!(gathered.unwrap().alias, None); + Ok(()) + } +} diff --git a/lib/u_lib/src/utils/combined_result.rs b/lib/u_lib/src/utils/combined_result.rs index 4699ae6..12dd4d6 100644 --- a/lib/u_lib/src/utils/combined_result.rs +++ b/lib/u_lib/src/utils/combined_result.rs @@ -1,12 +1,14 @@ +use std::fmt::Debug; + use crate::utils::OneOrVec; -use crate::UError; +use anyhow::Error; -pub struct CombinedResult { +pub struct CombinedResult { ok: Vec, err: Vec, } -impl CombinedResult { +impl CombinedResult { pub fn new() -> Self { Self { ok: vec![], @@ -18,18 +20,27 @@ impl CombinedResult { self.ok.extend(result.into_vec()); } - pub fn err(&mut self, err: impl OneOrVec) { - self.err.extend(err.into_vec()); + pub fn err>(&mut self, err: impl OneOrVec) { + self.err.extend( + err.into_vec() + .into_iter() + .map(Into::into) + .collect::>(), + ); } pub fn unwrap(self) -> Vec { let err_len = self.err.len(); if err_len > 0 { - panic!("CombinedResult has {} errors", err_len); + panic!("CombinedResult has errors: {:?}", self.err); } self.ok } + pub fn has_err(&self) -> bool { + !self.err.is_empty() + } + pub fn unwrap_one(self) -> T { self.unwrap().pop().unwrap() } diff --git a/lib/u_lib/src/utils/env.rs b/lib/u_lib/src/utils/env.rs new file mode 100644 index 0000000..a3c9345 --- /dev/null +++ b/lib/u_lib/src/utils/env.rs @@ -0,0 +1,29 @@ +use envy::{from_env, Result as EnvResult}; +use serde::{de::DeserializeOwned, Deserialize}; + +#[derive(Deserialize)] +pub struct DefaultEnv { + #[serde(default = "default_host")] + pub u_server: String, +} + +pub fn load_env() -> EnvResult { + dot(); + from_env() +} + +pub fn load_env_default() -> EnvResult { + dot(); + from_env() +} + +fn dot() { + let envs = [".env", ".env.private"]; + for envfile in &envs { + dotenv::from_filename(envfile).ok(); + } +} + +pub fn default_host() -> String { + "ortem.xyz".to_string() +} diff --git a/lib/u_lib/src/utils/fmt/hexlify.rs b/lib/u_lib/src/utils/fmt/hexlify.rs new file mode 100644 index 0000000..69bda69 --- /dev/null +++ b/lib/u_lib/src/utils/fmt/hexlify.rs @@ -0,0 +1,24 @@ +use std::fmt; + +pub struct Hexlify<'b>(pub &'b [u8]); + +impl<'a> fmt::LowerHex for Hexlify<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for byte in self.0.iter() { + write!(f, "{:02x}", byte)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hexlify() { + let data = b"\x5a\x6b\x23\x4f\xa3\x7f\x9e"; + let result = "5a6b234fa37f9e"; + assert_eq!(format!("{:x}", Hexlify(data)), result); + } +} diff --git a/lib/u_lib/src/utils/fmt/mod.rs b/lib/u_lib/src/utils/fmt/mod.rs new file mode 100644 index 0000000..aa14890 --- /dev/null +++ b/lib/u_lib/src/utils/fmt/mod.rs @@ -0,0 +1,5 @@ +mod hexlify; +mod stripped; + +pub use hexlify::*; +pub use stripped::*; diff --git a/lib/u_lib/src/utils/fmt/stripped.rs b/lib/u_lib/src/utils/fmt/stripped.rs new file mode 100644 index 0000000..44af3ca --- /dev/null +++ b/lib/u_lib/src/utils/fmt/stripped.rs @@ -0,0 +1,80 @@ +use std::fmt; +use std::iter::Iterator; +use std::slice::Iter as SliceIter; +use std::str::Chars; + +const MAX_DATA_LEN: usize = 2000; + +pub trait Strippable { + type Item: fmt::Display; + type TypeIter: Iterator; + + fn length(&self) -> usize; + fn iterator(&self) -> Self::TypeIter; +} + +impl<'a> Strippable for &'a str { + type Item = char; + type TypeIter = Chars<'a>; + + fn length(&self) -> usize { + self.len() + } + + fn iterator(&self) -> Self::TypeIter { + self.chars() + } +} + +impl<'a> Strippable for &'a Vec { + type Item = &'a u8; + type TypeIter = SliceIter<'a, u8>; + + fn length(&self) -> usize { + self.len() + } + + fn iterator(&self) -> Self::TypeIter { + self.iter() + } +} + +pub struct Stripped<'i, Inner: Strippable + 'i>(pub &'i Inner); + +impl<'i, Inner: Strippable + 'i> Stripped<'i, Inner> { + fn iter(&self) -> Inner::TypeIter { + self.0.iterator() + } + + fn placeholder(&self) -> &'static str { + if self.0.length() >= MAX_DATA_LEN { + " <...>" + } else { + "" + } + } +} + +impl<'i, Inner: Strippable + 'i> fmt::Display for Stripped<'i, Inner> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let placeholder = self.placeholder(); + for c in self.iter().take(MAX_DATA_LEN - placeholder.len()) { + write!(f, "{}", c)?; + } + write!(f, "{}", placeholder) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::*; + + #[rstest] + #[case("abc", 3)] + #[case("abcde".repeat(50), MAX_DATA_LEN)] + fn test_strip(#[case] input: impl Into, #[case] result_len: usize) { + let s = input.into(); + assert_eq!(Stripped(&s.as_str()).to_string().len(), result_len); + } +} diff --git a/lib/u_lib/src/utils/misc.rs b/lib/u_lib/src/utils/misc.rs index 375a6ad..7fddf4c 100644 --- a/lib/u_lib/src/utils/misc.rs +++ b/lib/u_lib/src/utils/misc.rs @@ -1,9 +1,3 @@ -use nix::{ - sys::signal::{signal, SigHandler, Signal}, - unistd::{chdir, close as fdclose, fork, getppid, setsid, ForkResult}, -}; -use std::process::exit; - pub trait OneOrVec { fn into_vec(self) -> Vec; } @@ -22,7 +16,7 @@ impl OneOrVec for Vec { #[macro_export] macro_rules! unwrap_enum { - ($src:ident, $t:path) => { + ($src:expr, $t:path) => { if let $t(result) = $src { result } else { @@ -30,42 +24,3 @@ macro_rules! unwrap_enum { } }; } - -pub fn daemonize() { - if getppid().as_raw() != 1 { - setsig(Signal::SIGTTOU, SigHandler::SigIgn); - setsig(Signal::SIGTTIN, SigHandler::SigIgn); - setsig(Signal::SIGTSTP, SigHandler::SigIgn); - } - for fd in 0..=2 { - match fdclose(fd) { - _ => (), - } - } - match chdir("/") { - _ => (), - }; - - match fork() { - Ok(ForkResult::Parent { .. }) => { - exit(0); - } - Ok(ForkResult::Child) => match setsid() { - _ => (), - }, - Err(_) => exit(255), - } -} - -pub fn setsig(sig: Signal, hnd: SigHandler) { - unsafe { - signal(sig, hnd).unwrap(); - } -} - -pub fn init_env() { - let envs = [".env", ".env.private"]; - for envfile in &envs { - dotenv::from_filename(envfile).ok(); - } -} diff --git a/lib/u_lib/src/utils/mod.rs b/lib/u_lib/src/utils/mod.rs index 13f94d3..5d9f9b4 100644 --- a/lib/u_lib/src/utils/mod.rs +++ b/lib/u_lib/src/utils/mod.rs @@ -1,11 +1,26 @@ pub mod combined_result; pub mod conv; +pub mod env; +pub mod fmt; pub mod misc; +pub mod platform; +pub mod proc_output; +pub mod storage; +#[cfg(not(target_arch = "wasm32"))] pub mod tempfile; -pub mod vec_display; +#[cfg(unix)] +pub mod unix; pub use combined_result::*; pub use conv::*; +pub use env::{load_env, load_env_default}; +pub use fmt::*; pub use misc::*; +pub use platform::*; +pub use proc_output::*; +pub use storage::*; +#[cfg(not(target_arch = "wasm32"))] pub use tempfile::*; -pub use vec_display::*; + +#[cfg(unix)] +pub use unix::*; diff --git a/lib/u_lib/src/utils/platform.rs b/lib/u_lib/src/utils/platform.rs new file mode 100644 index 0000000..04aa4ae --- /dev/null +++ b/lib/u_lib/src/utils/platform.rs @@ -0,0 +1,46 @@ +use guess_host_triple::guess_host_triple; +use platforms::{Platform as _Platform, PlatformReq}; +use serde::Deserialize; +use std::str::FromStr; + +#[derive(Debug, Deserialize)] +pub struct Platform(String); + +impl Platform { + pub fn new(p: impl Into) -> Self { + Self(p.into()) + } + + pub fn current() -> Platform { + Self(guess_host_triple().unwrap_or("unknown").to_string()) + } + + pub fn current_as_string() -> String { + Self::current().into_string() + } + + pub fn matches(&self, pf: impl AsRef) -> bool { + match PlatformReq::from_str(pf.as_ref()) { + Ok(p) => p.matches(&_Platform::find(&self.0).unwrap()), + Err(_) => false, + } + } + + pub fn check(&self) -> bool { + PlatformReq::from_str(&self.0).is_ok() + } + + pub fn into_string(self) -> String { + self.0 + } + + pub fn any() -> Platform { + Self(String::from("*")) + } +} + +impl Default for Platform { + fn default() -> Self { + Self::any() + } +} diff --git a/lib/u_lib/src/utils/proc_output.rs b/lib/u_lib/src/utils/proc_output.rs new file mode 100644 index 0000000..ded026e --- /dev/null +++ b/lib/u_lib/src/utils/proc_output.rs @@ -0,0 +1,104 @@ +use std::process::Output; + +#[derive(Clone, Debug)] +pub struct ProcOutput { + pub stdout: Vec, + pub stderr: Vec, +} + +impl ProcOutput { + const STDERR_DELIMETER: &[u8] = b"\n[STDERR]\n"; + + pub fn from_output(output: &Output) -> Self { + Self::new() + .stdout(output.stdout.to_vec()) + .stderr(output.stderr.to_vec()) + } + + pub fn new() -> Self { + Self { + stdout: vec![], + stderr: vec![], + } + } + + pub fn stdout(mut self, data: Vec) -> Self { + self.stdout = data; + self + } + + pub fn stderr(mut self, data: Vec) -> Self { + self.stderr = data; + self + } + + pub fn into_vec(self) -> Vec { + let mut result: Vec = vec![]; + if !self.stdout.is_empty() { + result.extend(self.stdout); + } + + if !self.stderr.is_empty() { + result.extend(Self::STDERR_DELIMETER); + result.extend(self.stderr); + } + result + } + + pub fn from_raw_proc_output(raw: &[u8]) -> Option { + let stderr_delim_len = Self::STDERR_DELIMETER.len(); + raw.windows(stderr_delim_len) + .position(|w| w == Self::STDERR_DELIMETER) + .map(|split_pos| { + let (stdout, stderr) = raw.split_at(split_pos); + let result = Self::new().stdout(stdout.to_vec()); + if stderr.len() <= stderr_delim_len { + result.stderr(stderr[stderr_delim_len..].to_vec()) + } else { + result + } + }) + } +} + +#[cfg(test)] +mod tests { + use crate::utils::{bytes_to_string, ProcOutput}; + use std::str; + + const STDERR_DELIMETER: &'static str = + unsafe { str::from_utf8_unchecked(ProcOutput::STDERR_DELIMETER) }; + + #[rstest] + #[case::stdout_stderr( + "lol", + "kek", + &format!("lol{}kek", STDERR_DELIMETER) + )] + #[case::stderr( + "", + "kek", + &format!("{}kek", STDERR_DELIMETER) + )] + fn test_to_combined(#[case] stdout: &str, #[case] stderr: &str, #[case] result: &str) { + let output = ProcOutput::new() + .stdout(stdout.as_bytes().to_vec()) + .stderr(stderr.as_bytes().to_vec()); + assert_eq!(&bytes_to_string(&output.into_vec()), result) + } + + #[rstest] + #[case::stdout_stderr( + &format!("lal{}kik", STDERR_DELIMETER), + )] + #[case::stdout( + &format!("qeq"), + )] + #[case::stderr( + &format!("{}vev", STDERR_DELIMETER), + )] + fn test_from_combined(#[case] src_result: &str) { + let output = ProcOutput::from_raw_proc_output(src_result.as_bytes()).unwrap(); + assert_eq!(bytes_to_string(&output.into_vec()).trim(), src_result); + } +} diff --git a/lib/u_lib/src/utils/storage.rs b/lib/u_lib/src/utils/storage.rs new file mode 100644 index 0000000..d96faba --- /dev/null +++ b/lib/u_lib/src/utils/storage.rs @@ -0,0 +1,39 @@ +use once_cell::sync::Lazy; +use std::cmp::Eq; +use std::collections::HashMap; +use std::hash::Hash; +use std::ops::Deref; +use std::sync::Arc; +use std::sync::{Mutex, MutexGuard}; + +//improve this later, replace job cacher with it +//possibly add different backends (memory, disk) +pub struct SharedStorage(Arc>>); + +impl SharedStorage { + pub fn new() -> Lazy> { + Lazy::new(|| SharedStorage(Arc::new(Mutex::new(HashMap::new())))) + } + + pub fn lock(&self) -> MutexGuard<'_, HashMap> { + self.0.lock().unwrap() + } + + pub fn get<'get, 'slf: 'get>(&'slf self, key: &'get Key) -> Option> { + if !self.lock().contains_key(key) { + return None; + } + let lock = self.lock(); + Some(RefHolder(lock, key)) + } +} + +pub struct RefHolder<'h, Key, Val>(pub MutexGuard<'h, HashMap>, pub &'h Key); + +impl<'h, Key: Eq + Hash, Val> Deref for RefHolder<'h, Key, Val> { + type Target = Val; + + fn deref(&self) -> &Self::Target { + self.0.get(self.1).unwrap() + } +} diff --git a/lib/u_lib/src/utils/tempfile.rs b/lib/u_lib/src/utils/tempfile.rs index e76baa8..d2fb58f 100644 --- a/lib/u_lib/src/utils/tempfile.rs +++ b/lib/u_lib/src/utils/tempfile.rs @@ -1,4 +1,7 @@ -use std::{env::temp_dir, fs, ops::Drop, os::unix::fs::PermissionsExt, path::PathBuf}; +use crate::{UError, UResult}; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::{env::temp_dir, fs, ops::Drop, path::PathBuf}; use uuid::Uuid; pub struct TempFile { @@ -17,22 +20,59 @@ impl TempFile { Self { path } } - pub fn write_all(&self, data: &[u8]) -> Result<(), String> { - fs::write(&self.path, data).map_err(|e| e.to_string()) + pub fn write_all(&self, data: &[u8]) -> UResult<()> { + fs::write(&self.path, data).map_err(|e| UError::FSError(self.get_path(), e.to_string()))?; + Ok(()) } - pub fn write_exec(data: &[u8]) -> Result { + pub fn write_exec(data: &[u8]) -> UResult { let this = Self::new(); let path = this.get_path(); - this.write_all(data).map_err(|e| (path.clone(), e))?; - let perms = fs::Permissions::from_mode(0o555); - fs::set_permissions(&path, perms).map_err(|e| (path, e.to_string()))?; + dbg!(&path); + this.write_all(data)?; + + #[cfg(unix)] + { + let perms = fs::Permissions::from_mode(0o555); + fs::set_permissions(&path, perms).map_err(|e| UError::FSError(path, e.to_string()))?; + } Ok(this) } } impl Drop for TempFile { fn drop(&mut self) { - fs::remove_file(&self.path).ok(); + fs::remove_file(&self.path).unwrap(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::utils::bytes_to_string; + use std::path::Path; + use std::process::Command; + + #[test] + fn test_file_is_not_busy() { + let binary = include_bytes!("../../tests/fixtures/echoer"); + for _ in 0..100 { + let executable = TempFile::write_exec(binary).unwrap(); + let path = executable.get_path(); + let result = Command::new(path).arg("qwe").output().unwrap(); + assert_eq!(bytes_to_string(result.stdout.as_ref()).trim(), "qwe"); + } + } + + #[test] + fn test_file_removed_after_dropping() { + let path; + { + let file = TempFile::new(); + file.write_all(b"asdqwe").unwrap(); + path = file.get_path(); + assert!(Path::new(&path).exists()) + } + assert!(!Path::new(&path).exists()) } } diff --git a/lib/u_lib/src/utils/unix.rs b/lib/u_lib/src/utils/unix.rs new file mode 100644 index 0000000..a4e4133 --- /dev/null +++ b/lib/u_lib/src/utils/unix.rs @@ -0,0 +1,37 @@ +use nix::{ + sys::signal::{signal, SigHandler, Signal}, + unistd::{chdir, close as fdclose, fork, getppid, setsid, ForkResult}, +}; +use std::process::exit; + +pub fn daemonize() { + if getppid().as_raw() != 1 { + setsig(Signal::SIGTTOU, SigHandler::SigIgn); + setsig(Signal::SIGTTIN, SigHandler::SigIgn); + setsig(Signal::SIGTSTP, SigHandler::SigIgn); + } + for fd in 0..=2 { + match fdclose(fd) { + _ => (), + } + } + match chdir("/") { + _ => (), + }; + + match fork() { + Ok(ForkResult::Parent { .. }) => { + exit(0); + } + Ok(ForkResult::Child) => match setsid() { + _ => (), + }, + Err(_) => exit(255), + } +} + +pub fn setsig(sig: Signal, hnd: SigHandler) { + unsafe { + signal(sig, hnd).unwrap(); + } +} diff --git a/migrations/2020-10-24-111622_create_all/up.sql b/migrations/2020-10-24-111622_create_all/up.sql index ef956ac..c99082f 100644 --- a/migrations/2020-10-24-111622_create_all/up.sql +++ b/migrations/2020-10-24-111622_create_all/up.sql @@ -1,69 +1,62 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; -CREATE TYPE JobType AS ENUM ('shell', 'manage', 'python'); +CREATE TYPE JobType AS ENUM ('shell', 'init', 'python'); CREATE TYPE JobState AS ENUM ('queued', 'running', 'finished'); CREATE TYPE AgentState AS ENUM ('new', 'active', 'banned'); CREATE TABLE IF NOT EXISTS agents ( - alias TEXT UNIQUE - , hostname TEXT NOT NULL - , id UUID NOT NULL DEFAULT uuid_generate_v4() - , is_root BOOLEAN NOT NULL DEFAULT false - , is_root_allowed BOOLEAN NOT NULL DEFAULT false - , last_active TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - -- target triplet - , platform TEXT NOT NULL - , regtime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - , state AgentState NOT NULL DEFAULT 'new' - -- is needed to processing requests - , token TEXT - , username TEXT NOT NULL - , PRIMARY KEY(id) -); + alias TEXT, + hostname TEXT NOT NULL, + host_info TEXT NOT NULL, + id UUID NOT NULL DEFAULT uuid_generate_v4(), + ip_gray TEXT, + ip_white TEXT, + is_root BOOLEAN NOT NULL DEFAULT false, + is_root_allowed BOOLEAN NOT NULL DEFAULT false, + last_active TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + platform TEXT NOT NULL, + regtime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + state AgentState NOT NULL DEFAULT 'new', + token TEXT, + username TEXT NOT NULL, -CREATE TABLE IF NOT EXISTS ip_addrs ( - agent_id UUID NOT NULL - , check_ts TIMESTAMP NOT NULL - , gateway TEXT - , id UUID NOT NULL DEFAULT uuid_generate_v4() - , iface TEXT NOT NULL - , ip_addr TEXT NOT NULL - , is_gray BOOLEAN NOT NULL DEFAULT true - , netmask TEXT NOT NULL - , PRIMARY KEY(id) - , FOREIGN KEY(agent_id) REFERENCES agents(id) + PRIMARY KEY(id) ); CREATE TABLE IF NOT EXISTS jobs ( - alias TEXT UNIQUE - , argv TEXT NOT NULL - , id UUID NOT NULL DEFAULT uuid_generate_v4() - -- Shell, Binary (with program download), - -- Python (with program and python download if not exist), Management - , exec_type JobType NOT NULL DEFAULT 'shell' - , platform TEXT NOT NULL - , payload BYTEA - , PRIMARY KEY(id) + alias TEXT, + argv TEXT NOT NULL, + id UUID NOT NULL DEFAULT uuid_generate_v4(), + exec_type JobType NOT NULL DEFAULT 'shell', + platform TEXT NOT NULL, + payload BYTEA, + payload_path TEXT, + schedule TEXT, + + PRIMARY KEY(id) ); CREATE TABLE IF NOT EXISTS results ( - agent_id UUID NOT NULL - , alias TEXT UNIQUE - , created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - , id UUID NOT NULL DEFAULT uuid_generate_v4() - , job_id UUID NOT NULL - , result BYTEA - , state JobState NOT NULL DEFAULT 'queued' - , retcode INTEGER - , updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - , FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE - , FOREIGN KEY(job_id) REFERENCES jobs(id) ON DELETE CASCADE - , PRIMARY KEY(id) + agent_id UUID NOT NULL, + alias TEXT, + created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + id UUID NOT NULL DEFAULT uuid_generate_v4(), + job_id UUID NOT NULL, + result BYTEA, + state JobState NOT NULL DEFAULT 'queued', + exec_type JobType NOT NULL DEFAULT 'shell', + retcode INTEGER, + updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + + FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE, + FOREIGN KEY(job_id) REFERENCES jobs(id) ON DELETE CASCADE, + PRIMARY KEY(id) ); CREATE TABLE IF NOT EXISTS certificates ( - agent_id UUID NOT NULL - , id UUID NOT NULL DEFAULT uuid_generate_v4() - , is_revoked BOOLEAN NOT NULL DEFAULT FALSE - , PRIMARY KEY(id) - , FOREIGN KEY(agent_id) REFERENCES agents(id) + agent_id UUID NOT NULL, + id UUID NOT NULL DEFAULT uuid_generate_v4(), + is_revoked BOOLEAN NOT NULL DEFAULT FALSE, + + PRIMARY KEY(id), + FOREIGN KEY(agent_id) REFERENCES agents(id) ); \ No newline at end of file diff --git a/.env.private.sample b/sample.env.private similarity index 100% rename from .env.private.sample rename to sample.env.private diff --git a/scripts/build_musl_libs.sh b/scripts/build_musl_libs.sh new file mode 100755 index 0000000..2ffe66b --- /dev/null +++ b/scripts/build_musl_libs.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -ex +source $(dirname $0)/rootdir.sh #set ROOTDIR +ARGS=$@ +STATIC_LIBS=./static +DOCKER_EXCHG=/musl-share +IMAGE=unki/musllibs +if [[ ! $(find ./static ! -empty -type d) ]]; then + mkdir -p $STATIC_LIBS + cd $ROOTDIR/images && docker build -t $IMAGE . -f musl-libs.Dockerfile + docker run \ + -v $ROOTDIR/$STATIC_LIBS:$DOCKER_EXCHG \ + -w /volume \ + -it \ + $IMAGE \ + bash -c "[[ \$(ls -A $DOCKER_EXCHG) ]] || cp /musl/. $DOCKER_EXCHG -r" +fi diff --git a/scripts/cargo_musl.sh b/scripts/cargo_musl.sh deleted file mode 100755 index 0a8b3aa..0000000 --- a/scripts/cargo_musl.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -ex -source $(dirname $0)/rootdir.sh #set ROOTDIR -ARGS=$@ -docker run \ - --env-file $ROOTDIR/.env \ - -v $ROOTDIR:/volume \ - -v cargo-cache:/root/.cargo/registry \ - -w /volume \ - -it \ - unki/musllibs \ - bash -c "umask 0000; cargo $ARGS" diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..bb019c0 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -xe +source $(dirname $0)/rootdir.sh #set ROOTDIR + +SERVER="ortem" +REMOTE_DIR=/srv/usrv +REMOTE_PATH=$SERVER:$REMOTE_DIR +RSYNC="rsync -arzh --progress" + +ssh $SERVER mkdir -p $REMOTE_DIR/{release,deploy} +$RSYNC $ROOTDIR/target/x86_64-unknown-linux-musl/release/u_server $REMOTE_PATH/release/u_server +$RSYNC $ROOTDIR/certs/server.{crt,key} $REMOTE_PATH/certs +$RSYNC $ROOTDIR/certs/ca.crt $REMOTE_PATH/certs +$RSYNC $ROOTDIR/migrations/ $REMOTE_PATH/migrations +$RSYNC $ROOTDIR/.env* $REMOTE_PATH/ +$RSYNC $ROOTDIR/integration/docker-compose.yml $REMOTE_PATH/deploy/ +$RSYNC $ROOTDIR/images/integration-tests/u_db* $REMOTE_PATH/deploy/ +$RSYNC $ROOTDIR/images/integration-tests/u_server.Dockerfile $REMOTE_PATH/deploy/ +$RSYNC $ROOTDIR/scripts/start_server.sh $REMOTE_PATH/start_server.sh +ssh $SERVER "cd $REMOTE_DIR/deploy && ./start_server.sh" \ No newline at end of file diff --git a/certs/gen_certs.sh b/scripts/gen_certs.sh similarity index 64% rename from certs/gen_certs.sh rename to scripts/gen_certs.sh index a0e9d52..af36f2c 100755 --- a/certs/gen_certs.sh +++ b/scripts/gen_certs.sh @@ -1,17 +1,26 @@ set -ex -DIR=. -V3_CFG=v3.ext +source $(dirname $0)/rootdir.sh #set ROOTDIR +DIR=$ROOTDIR/certs +V3_CFG=$DIR/v3.ext -cat > $DIR/$V3_CFG << EOF +mkdir -p $DIR +cat > $V3_CFG << EOF authorityKeyIdentifier=keyid,issuer basicConstraints=CA:FALSE keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment, keyAgreement, keyCertSign, cRLSign -EOF +subjectAltName = @alt_names +[alt_names] +DNS.1 = ortem.xyz +DNS.2 = u_server +DNS.3 = localhost +EOF openssl req -x509 -newkey rsa:4096 -keyout $DIR/ca.key -out $DIR/ca.crt -nodes -days 365 -subj "/CN=root" openssl req -newkey rsa:4096 -keyout $DIR/alice.key -out $DIR/alice.csr -nodes -days 365 -subj "/CN=alice" -openssl req -newkey rsa:4096 -keyout $DIR/server.key -out $DIR/server.csr -nodes -days 365 -subj "/CN=u_server" -openssl x509 -req -in $DIR/alice.csr -CA $DIR/ca.crt -CAkey $DIR/ca.key -out $DIR/alice.crt -set_serial 01 -days 365 -extfile $DIR/$V3_CFG -openssl x509 -req -in $DIR/server.csr -CA $DIR/ca.crt -CAkey $DIR/ca.key -out $DIR/server.crt -set_serial 01 -days 365 -extfile $DIR/$V3_CFG -openssl pkcs12 -export -out $DIR/alice.p12 -inkey $DIR/alice.key -in $DIR/alice.crt -passin pass: -passout pass: \ No newline at end of file +openssl req -newkey rsa:4096 -keyout $DIR/server.key -out $DIR/server.csr -nodes -days 365 -subj "/CN=ortem.xyz" +openssl x509 -req -in $DIR/alice.csr -CA $DIR/ca.crt -CAkey $DIR/ca.key -out $DIR/alice.crt -set_serial 01 -days 365 -extfile $V3_CFG +openssl x509 -req -in $DIR/server.csr -CA $DIR/ca.crt -CAkey $DIR/ca.key -out $DIR/server.crt -set_serial 01 -days 365 -extfile $V3_CFG +openssl pkcs12 -export -out $DIR/alice.p12 -inkey $DIR/alice.key -in $DIR/alice.crt -passin pass: -passout pass: + +rm $V3_CFG \ No newline at end of file diff --git a/scripts/get_docker_ip.sh b/scripts/get_docker_ip.sh new file mode 100755 index 0000000..9406e5e --- /dev/null +++ b/scripts/get_docker_ip.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +PAT=".+IPAddress\W+\"([0-9\.]+).+" +docker ps | grep unki/$1 | cut -d' ' -f1 | xargs docker inspect | grep -P $PAT | sed -r "s/$PAT/\1/" diff --git a/scripts/start_server.sh b/scripts/start_server.sh new file mode 100755 index 0000000..07a6d6a --- /dev/null +++ b/scripts/start_server.sh @@ -0,0 +1,5 @@ +#!/bin/bash +docker build -t unki/u_db -f u_db.Dockerfile . +docker build -t unki/u_server -f u_server.Dockerfile . +docker-compose down +docker-compose up -d u_server \ No newline at end of file diff --git a/spec.txt b/spec.txt new file mode 100644 index 0000000..53135f6 --- /dev/null +++ b/spec.txt @@ -0,0 +1,9 @@ +todos: +Upload/download files +More tests +Agent update (use more JobType's) +Erase log macros in release mode +Bump wine version to test agent on windows +Store downloaded payload on disk instead of ram +Improve web interface +Migrator binary