Merge pull request '17-pretty-web-interface' (#1) from 17-pretty-web-interface into master

Reviewed-on: #1

.cargo/config.toml

@@ -0,0 +1,6 @@
+[build]
+rustflags = [
+    "-L", "/home/ortem/src/rust/unki/static/lib",
+    "--remap-path-prefix=/home/ortem/src/rust/unki=src",
+    "--remap-path-prefix=/home/ortem/.cargo=cargo"
+]

.env

@@ -2,3 +2,4 @@ DB_HOST=u_db
 DB_NAME=u_db
 DB_USER=postgres
 RUST_BACKTRACE=1
+U_SERVER=u_server

.gitignore

@@ -1,9 +1,15 @@
 target/
-**/*.rs.bk
 .idea/
 data/
+certs/
+static/
+.vscode/
+release/
+**/node_modules/
+
+**/*.rs.bk
 **/*.pyc
-certs/*
 *.log
 echoer
 .env.private
+*.lock

Cargo.toml

@@ -5,12 +5,24 @@ members = [
     "bin/u_run",
     "bin/u_server",
     "lib/u_lib",
-    "lib/u_api_proc_macro",
     "integration"
 ]
 
+[workspace.dependencies]
+anyhow = "=1.0.63"
+reqwest = { version = "0.11", features = ["json"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+thiserror = "=1.0.31"
+tokio = { version = "1.11", features = ["macros"] }
+tracing = "0.1.35"
+tracing-appender = "0.2.0"
+tracing-subscriber = { version = "0.3.0", features = ["env-filter"]}
+uuid = "0.6.5"
+
 [profile.release]
 panic = "abort"
+strip = "symbols"
 
 [profile.dev]
 debug = true  # Добавляет флаг `-g` для компилятора;

Makefile

@@ -1,26 +0,0 @@
-.PHONY: _pre_build debug release run clean unit-tests integration-tests test
-
-CARGO=./scripts/cargo_musl.sh
-
-clean:
-	${CARGO} clean
-
-_pre_build:
-	docker build -t unki/musllibs ./muslrust
-
-debug: _pre_build
-	${CARGO} build
-
-release: _pre_build
-	${CARGO} build --release
-
-run: build
-	${CARGO} run
-
-unit-tests:
-	${CARGO} test --lib
-
-integration-tests:
-	cd ./integration && ./integration_tests.sh
-
-test: unit-tests integration-tests

Makefile.toml

@@ -0,0 +1,99 @@
+# i need to preserve --release in args, not to pass cargo make -p release
+# due to cargo failing to parse "" argument
+env_scripts = ['''
+#!@duckscript
+args = array ${1} ${2} ${3} ${4} ${5} ${6} ${7}
+set_env PROFILE_OVERRIDE debug
+
+for arg in ${args}
+    e = eq ${arg} "--release"
+    if ${e}
+        set_env PROFILE_OVERRIDE release
+    end
+end
+
+profile = get_env PROFILE_OVERRIDE
+echo PROFILE_OVERRIDE=${profile}
+''']
+
+[config]
+default_to_workspace = false
+
+[env]
+TARGET = "x86_64-unknown-linux-musl"
+CARGO = "cargo"
+ROOTDIR = "${CARGO_MAKE_WORKING_DIRECTORY}"
+STATIC_PREFIX = "${ROOTDIR}/static"
+PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true"
+PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = "${STATIC_PREFIX}/bin/pg_config"
+OPENSSL_STATIC = "true"
+OPENSSL_DIR = "${STATIC_PREFIX}"
+
+
+[tasks.build_static_libs]
+script = "./scripts/build_musl_libs.sh"
+
+[tasks.build_frontend]
+script = '''
+cd ./bin/u_panel/src/server/fe
+ng build
+'''
+
+[tasks.clean]
+command = "${CARGO}"
+args = ["clean"]
+
+[tasks.cargo_build]
+dependencies = ["build_static_libs", "build_frontend"]
+command = "${CARGO}"
+args = ["build", "--target", "${TARGET}", "${@}"]
+
+[tasks.cargo_update]
+command = "${CARGO}"
+args = ["update"]
+
+[tasks.release_tasks]
+condition = { env = { PROFILE_OVERRIDE = "release"} }
+script = '''
+BINS=$(ls ./target/${TARGET}/${PROFILE_OVERRIDE}/u_* -1 | grep -v ".d")
+echo "Stripping..."
+strip $BINS
+echo "Packing..."
+upx -9 $BINS
+'''
+
+[tasks.build]
+dependencies = ["cargo_update", "cargo_build", "release_tasks"]
+clear = true
+
+[tasks.run]
+disabled = true
+
+[tasks.unit]
+command = "${CARGO}"
+args = ["test", "--target", "${TARGET}", "--lib", "--", "${@}"]
+
+[tasks.integration]
+dependencies = ["cargo_update"]
+script = '''
+[[ ! -d "./target/${TARGET}/${PROFILE_OVERRIDE}" ]] && echo 'No target folder. Build project first' && exit 1
+cd ./integration
+bash integration_tests.sh ${@}
+'''
+
+[tasks.gen_schema]
+script = '''
+docker run --rm \
+    --env-file=$PWD/.env \
+    --env-file=$PWD/.env.private \
+    -v $PWD:/unki \
+    -w /unki \
+    unki/u_db \
+    /unki/images/integration-tests/u_db_entrypoint.sh || true
+'''
+
+[tasks.test]
+dependencies = ["unit", "integration"]
+
+[tasks.deploy]
+script = './scripts/deploy.sh'

bin/u_agent/Cargo.toml

@@ -2,19 +2,15 @@
 name = "u_agent"
 version = "0.1.0"
 authors = ["plazmoid <kronos44@mail.ru>"]
-edition = "2018"
+edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process", "time"] }
-sysinfo = "0.10.5"
 log = "^0.4"
-env_logger = "0.8.3"
+reqwest = { workspace = true }
-uuid = "0.6.5"
+sysinfo = "0.10.5"
-reqwest = { version = "0.11", features = ["json"] }
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "time"] }
-openssl = "*"
+uuid = { workspace = true }
-u_lib = { version = "*", path = "../../lib/u_lib" }
+u_lib = { path = "../../lib/u_lib" }
 
-[build-dependencies]
-openssl = "*"

bin/u_agent/build.rs

@@ -3,6 +3,6 @@ use std::path::PathBuf;
 fn main() {
     let server_cert = PathBuf::from("../../certs/ca.crt");
     if !server_cert.exists() {
-        panic!("CA certificate doesn't exist. Create it first with certs/gen_certs.sh");
+        panic!("CA certificate doesn't exist. Create it first with scripts/gen_certs.sh");
     }
 }

bin/u_agent/src/lib.rs

@@ -1,88 +1,120 @@
 // TODO:
 // поддержка питона
 // резолв адреса управляющего сервера через DoT
-// кроссплатформенность (реализовать интерфейс для винды и никсов)
-// проверка обнов
-// самоуничтожение
 
 #[macro_use]
 extern crate log;
-extern crate env_logger;
 
-use std::env;
+//use daemonize::Daemonize;
+use std::sync::Arc;
 use tokio::time::{sleep, Duration};
 use u_lib::{
-    api::ClientHandler,
+    api::ClientHandler, cache::JobCache, config::get_self_uid, errors::ErrChan,
-    builder::JobBuilder,
+    executor::pop_completed, logging::init_logger, messaging::Reportable, models::AssignedJobById,
-    cache::JobCache,
+    runner::JobRunner, utils::load_env_default,
-    executor::pop_completed,
-    models::{AssignedJob, ExecResult},
-    UID,
-    //daemonize
 };
 
-#[macro_export]
+const ITERATION_LATENCY: u64 = 5;
-macro_rules! retry_until_ok {
-    ( $body:expr ) => {
-        loop {
-            match $body {
-                Ok(r) => break r,
-                Err(e) => error!("{:?}", e),
-            };
-            sleep(Duration::from_secs(5)).await;
-        }
-    };
-}
 
-pub async fn process_request(job_requests: Vec<AssignedJob>, client: &ClientHandler) {
+pub async fn process_request(jobs: Vec<AssignedJobById>, client: &ClientHandler) {
-    if job_requests.len() > 0 {
+    if !jobs.is_empty() {
-        for jr in &job_requests {
+        for jr in &jobs {
-            if !JobCache::contains(&jr.job_id) {
+            if !JobCache::contains(jr.job_id) {
                 info!("Fetching job: {}", &jr.job_id);
-                let fetched_job = retry_until_ok!(client.get_jobs(Some(jr.job_id)).await)
+                let fetched_job = loop {
-                    .pop()
+                    match client.get_jobs(Some(jr.job_id)).await {
-                    .unwrap();
+                        Ok(mut result) => break result.pop().unwrap(),
+                        Err(err) => {
+                            debug!("{:?} \nretrying...", err);
+                            sleep(Duration::from_secs(ITERATION_LATENCY)).await;
+                        }
+                    }
+                };
                 JobCache::insert(fetched_job);
             }
         }
         info!(
             "Scheduling jobs: {}",
-            job_requests
+            jobs.iter()
-                .iter()
                 .map(|j| j.job_id.to_string())
                 .collect::<Vec<String>>()
                 .join(", ")
         );
-        let mut builder = JobBuilder::from_request(job_requests);
+        let mut runner = JobRunner::from_jobs(jobs);
-        let errors = builder.pop_errors();
+        let errors = runner.pop_errors();
-        if errors.len() > 0 {
+        if !errors.is_empty() {
-            error!(
+            for e in errors {
-                "Some errors encountered: \n{}",
+                ErrChan::send(e, "ebld").await;
-                errors
+            }
-                    .iter()
+        }
-                    .map(|j| j.to_string())
+        runner.unwrap_one().spawn().await;
-                    .collect::<Vec<String>>()
+    }
-                    .join("\n")
+}
-            );
+
+async fn error_reporting(client: Arc<ClientHandler>) -> ! {
+    loop {
+        match ErrChan::recv().await {
+            Some(err) => {
+                'retry: for _ in 0..3 {
+                    match client.report(Reportable::Error(err.clone())).await {
+                        Ok(_) => break 'retry,
+                        Err(e) => {
+                            debug!("Reporting error: {:?}", e);
+                            sleep(Duration::from_secs(10)).await;
+                        }
+                    }
+                }
+            }
+            None => sleep(Duration::from_secs(3)).await,
         }
-        builder.unwrap_one().spawn().await;
     }
 }
 
-pub async fn run_forever() {
+async fn agent_loop(client: Arc<ClientHandler>) -> ! {
-    //daemonize();
-    env_logger::init();
-    let arg_ip = env::args().nth(1);
-    let instance = ClientHandler::new(arg_ip.as_deref());
-    info!("Connecting to the server");
     loop {
-        let job_requests: Vec<AssignedJob> =
+        match client.get_personal_jobs(get_self_uid()).await {
-            retry_until_ok!(instance.get_personal_jobs(Some(*UID)).await).into_builtin_vec();
+            Ok(jobs) => {
-        process_request(job_requests, &instance).await;
+                process_request(jobs, &client).await;
-        let result: Vec<ExecResult> = pop_completed().await.into_iter().collect();
+            }
-        if result.len() > 0 {
+            Err(err) => ErrChan::send(err, "processing").await,
-            retry_until_ok!(instance.report(&result).await);
+        }
+        let result: Vec<Reportable> = pop_completed()
+            .await
+            .into_iter()
+            .map(|result| match result {
+                Ok(r) => Reportable::Assigned(r),
+                Err(e) => Reportable::Error(e),
+            })
+            .collect();
+        if !result.is_empty() {
+            if let Err(err) = client.report(result).await {
+                ErrChan::send(err, "report").await;
+            }
         }
-        sleep(Duration::from_secs(5)).await;
+        sleep(Duration::from_secs(ITERATION_LATENCY)).await;
+    }
+}
+
+pub async fn run_forever() -> ! {
+    let env = load_env_default().unwrap();
+    let client = Arc::new(ClientHandler::new(&env.u_server, None));
+    tokio::spawn(error_reporting(client.clone()));
+
+    if cfg!(debug_assertions) {
+        init_logger(Some(format!(
+            "u_agent-{}",
+            get_self_uid()
+                .hyphenated()
+                .to_string()
+                .split("-")
+                .next()
+                .unwrap()
+        )));
+        // } else {
+        //     if let Err(e) = Daemonize::new().start() {
+        //         ErrChan::send(UError::Runtime(e.to_string()), "deeeemon").await
+        //     }
     }
+    info!("Starting agent {}", get_self_uid());
+    agent_loop(client).await
 }

bin/u_agent/src/main.rs

@@ -1,4 +1,3 @@
-use tokio;
 use u_agent::run_forever;
 
 #[tokio::main]

bin/u_panel/Cargo.toml

@@ -2,18 +2,27 @@
 name = "u_panel"
 version = "0.1.0"
 authors = ["plazmoid <kronos44@mail.ru>"]
-edition = "2018"
+edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process"] }
+actix-cors = "0.6.1"
+actix-web = "4.1"
+anyhow = { workspace = true }
+futures-util = "0.3.21"
+mime_guess = "2.0.4"
+once_cell = "1.8.0"
+rust-embed = { version = "6.3.0", features = ["debug-embed", "compression"] }
+serde = { workspace = true }
+serde_json = { workspace = true }
+strum = { version = "0.22.0", features = ["derive"] }
+tokio = { workspace = true, features = ["rt", "rt-multi-thread"] }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
+tracing-appender = { workspace = true }
+shlex = "1.1.0"
 structopt = "0.3.21"
-log = "^0.4"
+thiserror = "1.0.31"
-env_logger = "0.7.1"
+uuid = { workspace = true }
-uuid = "0.6.5"
+u_lib = { version = "*", path = "../../lib/u_lib", features = ["panel"] }
-reqwest = { version = "0.11", features = ["json"] }
-openssl = "*"
-u_lib = { version = "*", path = "../../lib/u_lib" }
-serde_json = "1.0.4"
-serde = { version = "1.0.114", features = ["derive"] }

bin/u_panel/src/argparse.rs

@@ -0,0 +1,129 @@
+use serde_json::{from_str, to_value, Value};
+use structopt::StructOpt;
+use u_lib::{
+    api::ClientHandler,
+    datatypes::PanelResult,
+    messaging::AsMsg,
+    models::{Agent, AssignedJob, JobMeta},
+    UError, UResult,
+};
+use uuid::Uuid;
+
+#[derive(StructOpt, Debug)]
+pub struct Args {
+    #[structopt(subcommand)]
+    cmd: Cmd,
+}
+
+#[derive(StructOpt, Debug)]
+enum Cmd {
+    Agents(RUD),
+    Jobs(JobCRUD),
+    Map(JobMapCRUD),
+    Ping,
+    Serve,
+}
+
+#[derive(StructOpt, Debug)]
+enum JobCRUD {
+    Create {
+        job: String,
+    },
+    #[structopt(flatten)]
+    RUD(RUD),
+}
+
+#[derive(StructOpt, Debug)]
+enum JobCmd {
+    #[structopt(external_subcommand)]
+    Cmd(Vec<String>),
+}
+
+#[derive(StructOpt, Debug)]
+enum JobMapCRUD {
+    Create {
+        #[structopt(parse(try_from_str = parse_uuid))]
+        agent_uid: Uuid,
+
+        job_idents: Vec<String>,
+    },
+    #[structopt(flatten)]
+    RUD(RUD),
+}
+
+#[derive(StructOpt, Debug)]
+enum RUD {
+    Read {
+        #[structopt(parse(try_from_str = parse_uuid))]
+        uid: Option<Uuid>,
+    },
+    Update {
+        item: String,
+    },
+    Delete {
+        #[structopt(parse(try_from_str = parse_uuid))]
+        uid: Uuid,
+    },
+}
+
+fn parse_uuid(src: &str) -> Result<Uuid, String> {
+    Uuid::parse_str(src).map_err(|e| e.to_string())
+}
+
+pub fn into_value<M: AsMsg>(data: M) -> Value {
+    to_value(data).unwrap()
+}
+
+pub async fn process_cmd(client: ClientHandler, args: Args) -> PanelResult<Value> {
+    let catcher: UResult<Value> = (|| async {
+        Ok(match args.cmd {
+            Cmd::Agents(action) => match action {
+                RUD::Read { uid } => into_value(client.get_agents(uid).await?),
+                RUD::Update { item } => {
+                    let agent = from_str::<Agent>(&item)?;
+                    into_value(client.update_agent(agent).await?)
+                }
+                RUD::Delete { uid } => into_value(client.del(uid).await?),
+            },
+            Cmd::Jobs(action) => match action {
+                JobCRUD::Create { job } => {
+                    let raw_job = from_str::<JobMeta>(&job)?;
+                    let job = raw_job.validated()?;
+                    into_value(client.upload_jobs(job).await?)
+                }
+                JobCRUD::RUD(RUD::Read { uid }) => into_value(client.get_jobs(uid).await?),
+                JobCRUD::RUD(RUD::Update { item }) => {
+                    let raw_job = from_str::<JobMeta>(&item)?;
+                    let job = raw_job.validated()?;
+                    into_value(client.update_job(job).await?)
+                }
+                JobCRUD::RUD(RUD::Delete { uid }) => into_value(client.del(uid).await?),
+            },
+            Cmd::Map(action) => match action {
+                JobMapCRUD::Create {
+                    agent_uid,
+                    job_idents,
+                } => into_value(client.set_jobs(agent_uid, job_idents).await?),
+                JobMapCRUD::RUD(RUD::Read { uid }) => into_value(client.get_agent_jobs(uid).await?),
+                JobMapCRUD::RUD(RUD::Update { item }) => {
+                    let assigned = from_str::<AssignedJob>(&item)?;
+                    into_value(client.update_result(assigned).await?)
+                }
+                JobMapCRUD::RUD(RUD::Delete { uid }) => into_value(client.del(uid).await?),
+            },
+            Cmd::Ping => into_value(client.ping().await?),
+            Cmd::Serve => {
+                crate::server::serve(client)
+                    .await
+                    .map_err(|e| UError::PanelError(format!("{e:?}")))?;
+                Value::Null
+            }
+        })
+    })()
+    .await;
+
+    match catcher {
+        Ok(r) => PanelResult::Ok(r),
+        Err(e) => PanelResult::Err(e),
+    }
+}

bin/u_panel/src/main.rs

@@ -1,143 +1,33 @@
-use std::env;
+mod argparse;
-use std::fmt;
+mod server;
-use structopt::StructOpt;
-use u_lib::{
-    api::ClientHandler, datatypes::DataResult, messaging::AsMsg, models::JobMeta, utils::init_env,
-    UResult,
-};
-use uuid::Uuid;
-
-#[derive(StructOpt, Debug)]
-struct Args {
-    #[structopt(subcommand)]
-    cmd: Cmd,
-    #[structopt(long)]
-    json: bool,
-}
-
-#[derive(StructOpt, Debug)]
-enum Cmd {
-    Agents(LD),
-    Jobs(JobALD),
-    Jobmap(JobMapALD),
-}
 
-#[derive(StructOpt, Debug)]
+#[macro_use]
-enum JobALD {
+extern crate tracing;
-    Add {
-        #[structopt(long, parse(try_from_str = parse_uuid))]
-        agent: Option<Uuid>,
 
-        #[structopt(long)]
+use anyhow::Result as AnyResult;
-        alias: String,
+use argparse::{process_cmd, Args};
-
+use serde::Deserialize;
-        #[structopt(subcommand)]
+use structopt::StructOpt;
-        cmd: JobCmd,
+use u_lib::api::ClientHandler;
-    },
+use u_lib::logging::init_logger;
-    #[structopt(flatten)]
+use u_lib::utils::{env::default_host, load_env};
-    LD(LD),
-}
-
-#[derive(StructOpt, Debug)]
-enum JobCmd {
-    #[structopt(external_subcommand)]
-    Cmd(Vec<String>),
-}
-
-#[derive(StructOpt, Debug)]
-enum JobMapALD {
-    Add {
-        #[structopt(parse(try_from_str = parse_uuid))]
-        agent_uid: Uuid,
-
-        job_idents: Vec<String>,
-    },
-    List {
-        #[structopt(parse(try_from_str = parse_uuid))]
-        uid: Option<Uuid>,
-    },
-    Delete {
-        #[structopt(parse(try_from_str = parse_uuid))]
-        uid: Uuid,
-    },
-}
-
-#[derive(StructOpt, Debug)]
-enum LD {
-    List {
-        #[structopt(parse(try_from_str = parse_uuid))]
-        uid: Option<Uuid>,
-    },
-    Delete {
-        #[structopt(parse(try_from_str = parse_uuid))]
-        uid: Uuid,
-    },
-}
 
-fn parse_uuid(src: &str) -> Result<Uuid, String> {
+#[derive(Deserialize)]
-    Uuid::parse_str(src).map_err(|e| e.to_string())
+struct AccessEnv {
+    admin_auth_token: String,
+    #[serde(default = "default_host")]
+    u_server: String,
 }
 
-async fn process_cmd(args: Args) {
+#[actix_web::main]
-    struct Printer {
+async fn main() -> AnyResult<()> {
-        json: bool,
+    let env = load_env::<AccessEnv>()?;
-    }
+    let client = ClientHandler::new(&env.u_server, Some(env.admin_auth_token));
+    let args = Args::from_args();
 
-    impl Printer {
+    init_logger(None::<&str>);
-        pub fn print<Msg: AsMsg + fmt::Display>(&self, data: UResult<Msg>) {
-            if self.json {
-                let data = match data {
-                    Ok(r) => DataResult::Ok(r),
-                    Err(e) => DataResult::Err(e),
-                };
-                println!("{}", serde_json::to_string_pretty(&data).unwrap());
-            } else {
-                match data {
-                    Ok(r) => println!("{}", r),
-                    Err(e) => eprintln!("Error: {}", e),
-                }
-            }
-        }
-    }
-
-    let token = env::var("ADMIN_AUTH_TOKEN").expect("Authentication token is not set");
-    let cli_handler = ClientHandler::new(None).password(token);
-    let printer = Printer { json: args.json };
-    match args.cmd {
-        Cmd::Agents(action) => match action {
-            LD::List { uid } => printer.print(cli_handler.get_agents(uid).await),
-            LD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await),
-        },
-        Cmd::Jobs(action) => match action {
-            JobALD::Add {
-                cmd: JobCmd::Cmd(cmd),
-                alias,
-                agent: _agent,
-            } => {
-                let job = JobMeta::builder()
-                    .with_shell(cmd.join(" "))
-                    .with_alias(alias)
-                    .build()
-                    .unwrap();
-                printer.print(cli_handler.upload_jobs(&[job]).await);
-            }
-            JobALD::LD(LD::List { uid }) => printer.print(cli_handler.get_jobs(uid).await),
-            JobALD::LD(LD::Delete { uid }) => printer.print(cli_handler.del(Some(uid)).await),
-        },
-        Cmd::Jobmap(action) => match action {
-            JobMapALD::Add {
-                agent_uid,
-                job_idents,
-            } => printer.print(cli_handler.set_jobs(Some(agent_uid), &job_idents).await),
-            JobMapALD::List { uid } => printer.print(cli_handler.get_agent_jobs(uid).await),
-            JobMapALD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await),
-        },
-    }
-}
 
-#[tokio::main]
+    let result = process_cmd(client, args).await.to_string();
-async fn main() {
+    println!("{result}");
-    init_env();
+    Ok(())
-    let args: Args = Args::from_args();
-    process_cmd(args).await;
 }

bin/u_panel/src/server/error.rs

@@ -0,0 +1,17 @@
+use actix_web::http::StatusCode;
+use actix_web::ResponseError;
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("Arg parse error: {0}")]
+    ArgparseError(#[from] structopt::clap::Error),
+
+    #[error("Just an error: {0}")]
+    JustError(String),
+}
+
+impl ResponseError for Error {
+    fn status_code(&self) -> actix_web::http::StatusCode {
+        StatusCode::BAD_REQUEST
+    }
+}

bin/u_panel/src/server/fe/.gitignore

@@ -0,0 +1,48 @@
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+
+# compiled output
+/dist
+/tmp
+/out-tsc
+# Only exists if Bazel was run
+/bazel-out
+
+# dependencies
+/node_modules
+
+# profiling files
+chrome-profiler-events*.json
+
+# IDEs and editors
+/.idea
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# IDE - VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+.history/*
+
+# misc
+/.angular/cache
+/.sass-cache
+/connect.lock
+/coverage
+/libpeerconnection.log
+npm-debug.log
+yarn-error.log
+testem.log
+/typings
+
+# System Files
+.DS_Store
+Thumbs.db
+
+package-lock.json

bin/u_panel/src/server/fe/README.md

@@ -0,0 +1,27 @@
+# Fe
+
+This project was generated with [Angular CLI](https://github.com/angular/angular-cli) version 13.1.2.
+
+## Development server
+
+Run `ng serve` for a dev server. Navigate to `http://localhost:4200/`. The app will automatically reload if you change any of the source files.
+
+## Code scaffolding
+
+Run `ng generate component component-name` to generate a new component. You can also use `ng generate directive|pipe|service|class|guard|interface|enum|module`.
+
+## Build
+
+Run `ng build` to build the project. The build artifacts will be stored in the `dist/` directory.
+
+## Running unit tests
+
+Run `ng test` to execute the unit tests via [Karma](https://karma-runner.github.io).
+
+## Running end-to-end tests
+
+Run `ng e2e` to execute the end-to-end tests via a platform of your choice. To use this command, you need to first add a package that implements end-to-end testing capabilities.
+
+## Further help
+
+To get more help on the Angular CLI use `ng help` or go check out the [Angular CLI Overview and Command Reference](https://angular.io/cli) page.

bin/u_panel/src/server/fe/angular.json

@@ -0,0 +1,114 @@
+{
+  "$schema": "./node_modules/@angular/cli/lib/config/schema.json",
+  "version": 1,
+  "newProjectRoot": "projects",
+  "projects": {
+    "fe": {
+      "projectType": "application",
+      "schematics": {
+        "@schematics/angular:component": {
+          "style": "less"
+        },
+        "@schematics/angular:application": {
+          "strict": true
+        }
+      },
+      "root": "",
+      "sourceRoot": "src",
+      "prefix": "app",
+      "architect": {
+        "build": {
+          "builder": "@angular-devkit/build-angular:browser",
+          "options": {
+            "outputPath": "dist/fe",
+            "index": "src/index.html",
+            "main": "src/main.ts",
+            "polyfills": "src/polyfills.ts",
+            "tsConfig": "tsconfig.app.json",
+            "inlineStyleLanguage": "less",
+            "assets": [
+              "src/favicon.ico",
+              "src/assets"
+            ],
+            "styles": [
+              "./node_modules/@angular/material/prebuilt-themes/deeppurple-amber.css",
+              "src/styles.less"
+            ],
+            "scripts": []
+          },
+          "configurations": {
+            "production": {
+              "baseHref": "/core/",
+              "budgets": [
+                {
+                  "type": "initial",
+                  "maximumWarning": "500kb",
+                  "maximumError": "1mb"
+                },
+                {
+                  "type": "anyComponentStyle",
+                  "maximumWarning": "2kb",
+                  "maximumError": "4kb"
+                }
+              ],
+              "fileReplacements": [
+                {
+                  "replace": "src/environments/environment.ts",
+                  "with": "src/environments/environment.prod.ts"
+                }
+              ],
+              "outputHashing": "all"
+            },
+            "development": {
+              "buildOptimizer": false,
+              "optimization": false,
+              "vendorChunk": true,
+              "extractLicenses": false,
+              "sourceMap": true,
+              "namedChunks": true
+            }
+          },
+          "defaultConfiguration": "production"
+        },
+        "serve": {
+          "builder": "@angular-devkit/build-angular:dev-server",
+          "configurations": {
+            "production": {
+              "browserTarget": "fe:build:production"
+            },
+            "development": {
+              "browserTarget": "fe:build:development"
+            }
+          },
+          "defaultConfiguration": "development"
+        },
+        "extract-i18n": {
+          "builder": "@angular-devkit/build-angular:extract-i18n",
+          "options": {
+            "browserTarget": "fe:build"
+          }
+        },
+        "test": {
+          "builder": "@angular-devkit/build-angular:karma",
+          "options": {
+            "main": "src/test.ts",
+            "polyfills": "src/polyfills.ts",
+            "tsConfig": "tsconfig.spec.json",
+            "karmaConfig": "karma.conf.js",
+            "inlineStyleLanguage": "less",
+            "assets": [
+              "src/favicon.ico",
+              "src/assets"
+            ],
+            "styles": [
+              "./node_modules/@angular/material/prebuilt-themes/deeppurple-amber.css",
+              "src/styles.less"
+            ],
+            "scripts": []
+          }
+        }
+      }
+    }
+  },
+  "defaultProject": "fe"
+}

bin/u_panel/src/server/fe/karma.conf.js

@@ -0,0 +1,44 @@
+// Karma configuration file, see link for more information
+// https://karma-runner.github.io/1.0/config/configuration-file.html
+
+module.exports = function (config) {
+  config.set({
+    basePath: '',
+    frameworks: ['jasmine', '@angular-devkit/build-angular'],
+    plugins: [
+      require('karma-jasmine'),
+      require('karma-chrome-launcher'),
+      require('karma-jasmine-html-reporter'),
+      require('karma-coverage'),
+      require('@angular-devkit/build-angular/plugins/karma')
+    ],
+    client: {
+      jasmine: {
+        // you can add configuration options for Jasmine here
+        // the possible options are listed at https://jasmine.github.io/api/edge/Configuration.html
+        // for example, you can disable the random execution with `random: false`
+        // or set a specific seed with `seed: 4321`
+      },
+      clearContext: false // leave Jasmine Spec Runner output visible in browser
+    },
+    jasmineHtmlReporter: {
+      suppressAll: true // removes the duplicated traces
+    },
+    coverageReporter: {
+      dir: require('path').join(__dirname, './coverage/fe'),
+      subdir: '.',
+      reporters: [
+        { type: 'html' },
+        { type: 'text-summary' }
+      ]
+    },
+    reporters: ['progress', 'kjhtml'],
+    port: 9876,
+    colors: true,
+    logLevel: config.LOG_INFO,
+    autoWatch: true,
+    browsers: ['Chrome'],
+    singleRun: false,
+    restartOnFileChange: true
+  });
+};

bin/u_panel/src/server/fe/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "fe",
+  "version": "0.0.0",
+  "scripts": {
+    "ng": "ng",
+    "start": "ng serve",
+    "build": "ng build",
+    "watch": "ng build --watch --configuration development",
+    "test": "ng test"
+  },
+  "private": true,
+  "dependencies": {
+    "@angular/animations": "~13.1.0",
+    "@angular/cdk": "^13.3.9",
+    "@angular/common": "~13.1.0",
+    "@angular/compiler": "~13.1.0",
+    "@angular/core": "~13.1.0",
+    "@angular/forms": "~13.1.0",
+    "@angular/material": "^13.3.9",
+    "@angular/platform-browser": "~13.1.0",
+    "@angular/platform-browser-dynamic": "~13.1.0",
+    "@angular/router": "~13.1.0",
+    "@types/uuid": "^8.3.4",
+    "rxjs": "~7.4.0",
+    "tslib": "^2.3.0",
+    "uuid": "^8.3.2",
+    "zone.js": "~0.11.4"
+  },
+  "devDependencies": {
+    "@angular-devkit/build-angular": "^13.3.9",
+    "@angular/cli": "~13.1.2",
+    "@angular/compiler-cli": "~13.1.0",
+    "@types/jasmine": "~3.10.0",
+    "@types/node": "^12.11.1",
+    "jasmine-core": "~3.10.0",
+    "karma": "~6.3.0",
+    "karma-chrome-launcher": "~3.1.0",
+    "karma-coverage": "~2.1.0",
+    "karma-jasmine": "~4.0.0",
+    "karma-jasmine-html-reporter": "~1.7.0",
+    "typescript": "~4.5.2"
+  }
+}

bin/u_panel/src/server/fe/src/app/app-routing.module.ts

@@ -0,0 +1,19 @@
+import { NgModule } from '@angular/core';
+import { RouterModule, Routes } from '@angular/router';
+import { AgentComponent } from './core/tables/agent.component';
+import { JobComponent } from './core/tables/job.component';
+import { ResultComponent } from './core/tables/result.component';
+import { AgentInfoDialogComponent } from './core/tables/dialogs/agent_info.component';
+
+const routes: Routes = [
+  { path: '', redirectTo: 'agents', pathMatch: 'full' },
+  { path: 'agents', component: AgentComponent },
+  { path: 'jobs', component: JobComponent },
+  { path: 'results', component: ResultComponent },
+];
+
+@NgModule({
+  imports: [RouterModule.forRoot(routes)],
+  exports: [RouterModule]
+})
+export class AppRoutingModule { }

bin/u_panel/src/server/fe/src/app/app.component.html

@@ -0,0 +1,5 @@
+<nav mat-tab-nav-bar animationDuration="0ms" mat-align-tabs="center">
+  <a mat-tab-link *ngFor="let tab of tabs" routerLink={{tab.link}} routerLinkActive #rla="routerLinkActive"
+    [active]="rla.isActive" [routerLinkActiveOptions]="{ exact: true }">{{tab.name}}</a>
+</nav>
+<router-outlet></router-outlet>

bin/u_panel/src/server/fe/src/app/app.component.spec.ts

@@ -0,0 +1,35 @@
+import { TestBed } from '@angular/core/testing';
+import { RouterTestingModule } from '@angular/router/testing';
+import { AppComponent } from './app.component';
+
+describe('AppComponent', () => {
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [
+        RouterTestingModule
+      ],
+      declarations: [
+        AppComponent
+      ],
+    }).compileComponents();
+  });
+
+  it('should create the app', () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    const app = fixture.componentInstance;
+    expect(app).toBeTruthy();
+  });
+
+  it(`should have as title 'fe'`, () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    const app = fixture.componentInstance;
+    expect(app.title).toEqual('fe');
+  });
+
+  it('should render title', () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    fixture.detectChanges();
+    const compiled = fixture.nativeElement as HTMLElement;
+    expect(compiled.querySelector('.content span')?.textContent).toContain('fe app is running!');
+  });
+});

bin/u_panel/src/server/fe/src/app/app.component.ts

@@ -0,0 +1,14 @@
+import { Component, ViewChild, AfterViewInit } from '@angular/core';
+
+@Component({
+  selector: 'app-root',
+  templateUrl: './app.component.html',
+  styleUrls: ['./app.component.less']
+})
+export class AppComponent {
+  tabs = [
+    { name: 'Agents', link: '/agents' },
+    { name: 'Jobs', link: '/jobs' },
+    { name: 'Results', link: '/results' }
+  ];
+}

bin/u_panel/src/server/fe/src/app/app.module.ts

@@ -0,0 +1,60 @@
+import { NgModule } from '@angular/core';
+import { BrowserModule } from '@angular/platform-browser';
+import { AppRoutingModule } from './app-routing.module';
+import { AppComponent } from './app.component';
+import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
+import { MatTabsModule } from '@angular/material/tabs';
+import { MatTableModule } from '@angular/material/table';
+import { MatFormFieldModule } from '@angular/material/form-field';
+import { MatButtonModule } from '@angular/material/button'
+import { MatInputModule } from '@angular/material/input';
+import { MatProgressSpinnerModule } from '@angular/material/progress-spinner';
+import { HttpClientModule } from '@angular/common/http';
+import { MatDialogModule } from '@angular/material/dialog';
+import { MatIconModule } from '@angular/material/icon';
+import { FormsModule } from '@angular/forms';
+import { AgentComponent, JobComponent, ResultComponent } from './core/tables';
+import {
+  AgentInfoDialogComponent,
+  AssignJobDialogComponent,
+  JobInfoDialogComponent,
+  ResultInfoDialogComponent
+} from './core/tables/dialogs';
+import { APP_BASE_HREF } from '@angular/common';
+import { MatTooltipModule } from '@angular/material/tooltip';
+import { MatSnackBarModule } from '@angular/material/snack-bar';
+import { MatListModule } from '@angular/material/list';
+
+@NgModule({
+  declarations: [
+    AppComponent,
+    AgentComponent,
+    JobComponent,
+    ResultComponent,
+    AgentInfoDialogComponent,
+    JobInfoDialogComponent,
+    ResultInfoDialogComponent,
+    AssignJobDialogComponent
+  ],
+  imports: [
+    BrowserModule,
+    HttpClientModule,
+    AppRoutingModule,
+    MatTabsModule,
+    MatTableModule,
+    MatButtonModule,
+    MatFormFieldModule,
+    MatInputModule,
+    MatDialogModule,
+    MatProgressSpinnerModule,
+    MatIconModule,
+    MatTooltipModule,
+    MatSnackBarModule,
+    MatListModule,
+    FormsModule,
+    BrowserAnimationsModule
+  ],
+  providers: [{ provide: APP_BASE_HREF, useValue: '/' }],
+  bootstrap: [AppComponent]
+})
+export class AppModule { }

bin/u_panel/src/server/fe/src/app/core/index.ts

@@ -0,0 +1 @@
+export * from './services'; 

bin/u_panel/src/server/fe/src/app/core/models/agent.model.ts

@@ -0,0 +1,16 @@
+import { UTCDate, ApiModel } from ".";
+
+export interface AgentModel extends ApiModel {
+    alias: string | null,
+    hostname: string,
+    host_info: string,
+    id: string,
+    is_root: boolean,
+    is_root_allowed: boolean,
+    last_active: UTCDate,
+    platform: string,
+    regtime: UTCDate,
+    state: "new" | "active" | "banned",
+    token: string | null,
+    username: string,
+}

bin/u_panel/src/server/fe/src/app/core/models/index.ts

@@ -0,0 +1,14 @@
+export * from './agent.model';
+export * from './result.model';
+export * from './job.model';
+
+export interface UTCDate {
+    secs_since_epoch: number,
+    nanos_since_epoch: number
+}
+
+export abstract class ApiModel { }
+
+export interface Empty extends ApiModel { }
+
+export type Area = "agents" | "jobs" | "map";

bin/u_panel/src/server/fe/src/app/core/models/job.model.ts

@@ -0,0 +1,12 @@
+import { ApiModel } from ".";
+
+export interface JobModel extends ApiModel {
+    alias: string,
+    argv: string,
+    id: string,
+    exec_type: string,
+    platform: string,
+    payload: number[] | null,
+    payload_path: string | null,
+    schedule: string | null,
+}

bin/u_panel/src/server/fe/src/app/core/models/result.model.ts

@@ -0,0 +1,13 @@
+import { UTCDate, ApiModel } from ".";
+
+export interface ResultModel extends ApiModel {
+    agent_id: string,
+    alias: string,
+    created: UTCDate,
+    id: string,
+    job_id: string,
+    result: number[],
+    state: "Queued" | "Running" | "Finished",
+    retcode: number | null,
+    updated: UTCDate,
+}

bin/u_panel/src/server/fe/src/app/core/services/api.service.ts

@@ -0,0 +1,53 @@
+import { environment } from 'src/environments/environment';
+import { HttpClient } from '@angular/common/http';
+import { firstValueFrom } from 'rxjs';
+import { ApiModel, Empty, Area } from '../models';
+
+interface ServerResponse<T extends ApiModel> {
+  status: "ok" | "err",
+  data: T | string
+}
+
+export class ApiTableService<T extends ApiModel> {
+  area: Area;
+
+  constructor(private http: HttpClient, area: Area) {
+    this.area = area;
+  }
+
+  requestUrl = `${environment.server}/cmd/`;
+
+  async req<R extends ApiModel>(cmd: string): Promise<ServerResponse<R>> {
+    return await firstValueFrom(this.http.post<ServerResponse<R>>(this.requestUrl, cmd))
+  }
+
+  async getOne(id: string, area: string = this.area): Promise<ServerResponse<T>> {
+    const resp = await this.req<T[]>(`${area} read ${id}`)
+    if (resp.data.length === 0) {
+      return {
+        status: 'err',
+        data: `${id} not found in ${area}`
+      }
+    }
+    return {
+      status: resp.status,
+      data: resp.data[0]
+    }
+  }
+
+  async getMany(): Promise<ServerResponse<T[]>> {
+    return await this.req(`${this.area} read`)
+  }
+
+  async update(item: T): Promise<ServerResponse<Empty>> {
+    return await this.req(`${this.area} update '${JSON.stringify(item)}'`)
+  }
+
+  async delete(id: string): Promise<ServerResponse<Empty>> {
+    return await this.req(`${this.area} delete ${id}`)
+  }
+
+  async create(item: string): Promise<ServerResponse<string[]>> {
+    return await this.req(`${this.area} create ${item}`)
+  }
+}

bin/u_panel/src/server/fe/src/app/core/services/index.ts

@@ -0,0 +1 @@
+export * from './api.service'

bin/u_panel/src/server/fe/src/app/core/tables/agent.component.html

@@ -0,0 +1,78 @@
+<div class="mat-elevation-z8">
+
+    <div class="table-container">
+        <div class="loading-shade" *ngIf="isLoadingResults">
+            <mat-spinner *ngIf="isLoadingResults"></mat-spinner>
+        </div>
+        <mat-form-field appearance="standard">
+            <mat-label>Filter</mat-label>
+            <input matInput (keyup)="apply_filter($event)" #input>
+        </mat-form-field>
+        <button id="refresh_btn" mat-raised-button color="primary" (click)="loadTableData()">Refresh</button>
+
+        <table mat-table fixedLayout="true" [dataSource]="table_data" class="data-table" matSort matSortActive="id"
+            matSortDisableClear matSortDirection="desc">
+
+            <ng-container matColumnDef="id">
+                <th mat-header-cell *matHeaderCellDef>ID</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.id}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="alias">
+                <th mat-header-cell *matHeaderCellDef>Alias</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.alias}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="username">
+                <th mat-header-cell *matHeaderCellDef>User</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.username}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="hostname">
+                <th mat-header-cell *matHeaderCellDef>Hostname</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.hostname}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="last_active">
+                <th mat-header-cell *matHeaderCellDef>Last active</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.last_active.secs_since_epoch * 1000 | date:'long'}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="actions">
+                <th mat-header-cell *matHeaderCellDef></th>
+                <td mat-cell *matCellDef="let row">
+                    <button mat-icon-button (click)="assignJobs(row.id)">
+                        <mat-icon>add_task</mat-icon>
+                    </button>
+                    |
+                    <button mat-icon-button routerLink='.' [queryParams]="{id: row.id}">
+                        <mat-icon>more_horiz</mat-icon>
+                    </button>
+                    |
+                    <button mat-icon-button (click)="deleteItem(row.id)">
+                        <mat-icon>delete</mat-icon>
+                    </button>
+                </td>
+            </ng-container>
+
+            <tr mat-header-row *matHeaderRowDef="displayedColumns"></tr>
+            <tr mat-row class="data-table-row" *matRowDef="let row; columns: displayedColumns;"></tr>
+            <tr class="mat-row" *matNoDataRow>
+                <td class="mat-cell">No data</td>
+            </tr>
+        </table>
+    </div>
+
+    <!-- <mat-paginator [length]="resultsLength" [pageSize]="30" aria-label="Select page of GitHub search results">
+    </mat-paginator> -->
+</div>

bin/u_panel/src/server/fe/src/app/core/tables/agent.component.ts

@@ -0,0 +1,52 @@
+import { Component, OnDestroy, OnInit } from '@angular/core';
+import { TablesComponent } from './table.component';
+import { AgentModel } from '../models';
+import { AgentInfoDialogComponent } from './dialogs/agent_info.component';
+import { HttpErrorResponse } from '@angular/common/http';
+import { AssignJobDialogComponent } from './dialogs';
+
+@Component({
+  selector: 'agent-table',
+  templateUrl: './agent.component.html',
+  styleUrls: ['./table.component.less']
+})
+export class AgentComponent extends TablesComponent<AgentModel> implements OnInit {
+
+  //dialogSubscr!: Subscription;
+  area = 'agents' as const;
+
+  displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions']
+
+  show_item_dialog(id: string) {
+    this.data_source!.getOne(id).then(resp => {
+      if (resp.status === 'ok') {
+        const dialog = this.infoDialog.open(AgentInfoDialogComponent, {
+          data: resp.data as AgentModel,
+          width: '1000px',
+        });
+
+        const saveSub = dialog.componentInstance.onSave.subscribe(result => {
+          this.data_source!.update(result).then(_ => {
+            this.openSnackBar('Saved', false)
+            this.loadTableData()
+          })
+            .catch((err: HttpErrorResponse) => this.openSnackBar(err.error))
+        })
+
+        dialog.afterClosed().subscribe(result => {
+          saveSub.unsubscribe()
+          this.router.navigate(['.'], { relativeTo: this.route })
+        })
+      } else {
+        this.openSnackBar(resp.data)
+      }
+    }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error))
+  }
+
+  assignJobs(id: string) {
+    const dialog = this.infoDialog.open(AssignJobDialogComponent, {
+      data: id,
+      width: '1000px',
+    });
+  }
+}

bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent-info-dialog.html

@@ -0,0 +1,64 @@
+<h2 mat-dialog-title *ngIf="is_preview">Agent info</h2>
+<h2 mat-dialog-title *ngIf="!is_preview">Editing agent info</h2>
+<mat-dialog-content>
+    <p>
+        <mat-form-field class="info-dlg-field" cdkFocusInitial>
+            <mat-label>ID</mat-label>
+            <input matInput disabled value="{{data.id}}">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Alias</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.alias">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Username</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.username">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Hostname</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.hostname">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Host info</mat-label>
+            <textarea matInput cdkTextareaAutosize [readonly]="is_preview" [(ngModel)]="data.host_info">
+            </textarea>
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Platform</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.platform">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Is root</mat-label>
+            <input matInput disabled value="{{data.is_root}}">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Registration time</mat-label>
+            <input matInput disabled value="{{data.regtime.secs_since_epoch * 1000 | date:'long'}}">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Last active time</mat-label>
+            <input matInput disabled value="{{data.last_active.secs_since_epoch * 1000 | date:'long'}}">
+        </mat-form-field>
+    </p>
+</mat-dialog-content>
+<mat-dialog-actions align="end">
+    <button mat-raised-button *ngIf="is_preview" (click)="is_preview = false">Edit</button>
+    <button mat-raised-button *ngIf="!is_preview" (click)="updateAgent()">Save</button>
+    <button mat-button mat-dialog-close>Cancel</button>
+</mat-dialog-actions>

bin/u_panel/src/server/fe/src/app/core/tables/dialogs/agent_info.component.ts

@@ -0,0 +1,21 @@
+import { Component, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { AgentModel } from '../../models/agent.model';
+import { EventEmitter } from '@angular/core';
+
+@Component({
+    selector: 'agent-info-dialog',
+    templateUrl: 'agent-info-dialog.html',
+    styleUrls: ['info-dialog.component.less']
+})
+export class AgentInfoDialogComponent {
+    is_preview = true;
+    onSave = new EventEmitter();
+
+    constructor(@Inject(MAT_DIALOG_DATA) public data: AgentModel) { }
+
+    updateAgent() {
+        console.log(this.data);
+        this.onSave.emit(this.data);
+    }
+}

bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign-job-dialog.html

@@ -0,0 +1,12 @@
+<h2 mat-dialog-title>Assign job</h2>
+<mat-dialog-content>
+    <mat-selection-list #jobsList [(ngModel)]="selected_rows">
+        <mat-list-option *ngFor="let row of rows" [value]="row">
+            {{row}}
+        </mat-list-option>
+    </mat-selection-list>
+</mat-dialog-content>
+<mat-dialog-actions align="end">
+    <button mat-raised-button mat-dialog-close (click)="assignSelectedJobs()">Assign</button>
+    <button mat-button mat-dialog-close>Cancel</button>
+</mat-dialog-actions>

bin/u_panel/src/server/fe/src/app/core/tables/dialogs/assign_job.component.ts

@@ -0,0 +1,33 @@
+import { Component, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { HttpClient } from '@angular/common/http';
+import { ApiTableService } from '../../services';
+import { JobModel } from '../../models';
+import { MatListOption } from '@angular/material/list';
+
+@Component({
+    selector: 'assign-job-dialog',
+    templateUrl: 'assign-job-dialog.html',
+    styleUrls: []
+})
+export class AssignJobDialogComponent {
+    rows: string[] = [];
+    selected_rows: string[] = [];
+
+    constructor(@Inject(MAT_DIALOG_DATA) public agent_id: string, private http: HttpClient) {
+        new ApiTableService(http, "jobs").getMany().then(result => {
+            if (result.status == "ok") {
+                const jobs = result.data as JobModel[]
+                this.rows = jobs.map(j => `${j.id} ${j.alias}`)
+            } else {
+                alert(result.data as string)
+            }
+        }).catch(err => alert(err))
+    }
+
+    assignSelectedJobs() {
+        const job_ids = this.selected_rows.map(row => row.split(' ', 1)[0]).join(' ');
+        const request = `${this.agent_id} ${job_ids}`
+        new ApiTableService(this.http, "map").create(request).catch(err => alert(err))
+    }
+}

bin/u_panel/src/server/fe/src/app/core/tables/dialogs/index.ts

@@ -0,0 +1,4 @@
+export * from './agent_info.component';
+export * from './result_info.component';
+export * from './job_info.component';
+export * from './assign_job.component';

bin/u_panel/src/server/fe/src/app/core/tables/dialogs/info-dialog.component.less

@@ -0,0 +1,14 @@
+.info-dlg-field {
+    width: 100%;
+}
+
+div.info-dialog-forms-box {
+    width: 100%;
+    margin-right: 10px;
+}
+
+div.info-dialog-forms-box-smol {
+    width: 30%;
+    float: left;
+    margin-right: 10px;
+}

bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job-info-dialog.html

@@ -0,0 +1,50 @@
+<h2 mat-dialog-title *ngIf="is_preview">Job info</h2>
+<h2 mat-dialog-title *ngIf="!is_preview">Editing job info</h2>
+<mat-dialog-content>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field" cdkFocusInitial>
+            <mat-label>ID</mat-label>
+            <input matInput disabled value="{{data.id}}">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Alias</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.alias">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Args</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.argv">
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Type</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.exec_type">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Platform</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.platform">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Schedule</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.schedule">
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Payload path</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.payload_path">
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box">
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Payload</mat-label>
+            <textarea matInput cdkTextareaAutosize [readonly]="is_preview" [(ngModel)]="decodedPayload">
+                </textarea>
+        </mat-form-field>
+    </div>
+</mat-dialog-content>
+<mat-dialog-actions align="end">
+    <button mat-raised-button *ngIf="is_preview" (click)="is_preview = false">Edit</button>
+    <button mat-raised-button *ngIf="!is_preview" (click)="updateJob()">Save</button>
+    <button mat-button mat-dialog-close>Cancel</button>
+</mat-dialog-actions>

bin/u_panel/src/server/fe/src/app/core/tables/dialogs/job_info.component.ts

@@ -0,0 +1,30 @@
+import { Component, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { JobModel } from '../../models/job.model';
+import { EventEmitter } from '@angular/core';
+
+@Component({
+    selector: 'job-info-dialog',
+    templateUrl: 'job-info-dialog.html',
+    styleUrls: ['info-dialog.component.less']
+})
+export class JobInfoDialogComponent {
+    is_preview = true;
+    decodedPayload: string;
+    onSave = new EventEmitter();
+
+    constructor(@Inject(MAT_DIALOG_DATA) public data: JobModel) {
+        if (data.payload !== null) {
+            this.decodedPayload = new TextDecoder().decode(new Uint8Array(data.payload))
+        } else {
+            this.decodedPayload = ""
+        }
+    }
+
+    updateJob() {
+        if (this.decodedPayload.length > 0) {
+            this.data.payload = Array.from(new TextEncoder().encode(this.decodedPayload))
+        }
+        this.onSave.emit(this.data);
+    }
+}

bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result-info-dialog.html

@@ -0,0 +1,53 @@
+<h2 mat-dialog-title>Result</h2>
+<mat-dialog-content>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field" cdkFocusInitial>
+            <mat-label>ID</mat-label>
+            <input matInput readonly value="{{data.id}}">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Job ID</mat-label>
+            <input matInput readonly value="{{data.job_id}}">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Agent ID</mat-label>
+            <input matInput readonly value="{{data.agent_id}}">
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Alias</mat-label>
+            <input matInput readonly value="{{data.alias}}">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>State</mat-label>
+            <input matInput readonly value="{{data.state}}">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Return code</mat-label>
+            <input matInput readonly value="{{data.retcode}}">
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Created</mat-label>
+            <input matInput readonly value="{{data.created.secs_since_epoch * 1000 | date:'long'}}">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Updated</mat-label>
+            <input matInput readonly value="{{data.updated.secs_since_epoch * 1000 | date:'long'}}">
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box">
+        <p>
+            <mat-form-field class="info-dlg-field">
+                <mat-label>Result</mat-label>
+                <textarea matInput cdkTextareaAutosize readonly value="{{decodedResult}}">
+                </textarea>
+            </mat-form-field>
+        </p>
+    </div>
+</mat-dialog-content>
+<mat-dialog-actions align="end">
+    <button mat-button mat-dialog-close>Close</button>
+</mat-dialog-actions>

bin/u_panel/src/server/fe/src/app/core/tables/dialogs/result_info.component.ts

@@ -0,0 +1,20 @@
+import { Component, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { ResultModel } from '../../models/result.model';
+
+@Component({
+    selector: 'result-info-dialog',
+    templateUrl: 'result-info-dialog.html',
+    styleUrls: ['info-dialog.component.less']
+})
+export class ResultInfoDialogComponent {
+    decodedResult: string;
+
+    constructor(@Inject(MAT_DIALOG_DATA) public data: ResultModel) {
+        if (data.result !== null) {
+            this.decodedResult = new TextDecoder().decode(new Uint8Array(data.result))
+        } else {
+            this.decodedResult = ""
+        }
+    }
+}

bin/u_panel/src/server/fe/src/app/core/tables/index.ts

@@ -0,0 +1,3 @@
+export * from './agent.component';
+export * from './job.component';
+export * from './result.component';

bin/u_panel/src/server/fe/src/app/core/tables/job.component.html

@@ -0,0 +1,83 @@
+<div class="mat-elevation-z8">
+
+    <div class="table-container">
+        <div class="loading-shade" *ngIf="isLoadingResults">
+            <mat-spinner *ngIf="isLoadingResults"></mat-spinner>
+        </div>
+        <mat-form-field appearance="standard">
+            <mat-label>Filter</mat-label>
+            <input matInput (keyup)="apply_filter($event)" #input>
+        </mat-form-field>
+        <button id="refresh_btn" mat-raised-button color="basic" (click)="loadTableData()">Refresh</button>
+        <button id="new_btn" mat-raised-button color="primary" routerLink='.' [queryParams]="{new: true}">Add
+            job</button>
+
+        <table mat-table fixedLayout="true" [dataSource]="table_data" class="data-table" matSort matSortActive="id"
+            matSortDisableClear matSortDirection="desc">
+
+            <ng-container matColumnDef="id">
+                <th mat-header-cell *matHeaderCellDef>ID</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.id}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="alias">
+                <th mat-header-cell *matHeaderCellDef>Alias</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.alias}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="argv">
+                <th mat-header-cell *matHeaderCellDef>Cmd-line args</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.argv}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="platform">
+                <th mat-header-cell *matHeaderCellDef>Platform</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.platform}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="schedule">
+                <th mat-header-cell *matHeaderCellDef>Schedule</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.schedule}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="exec_type">
+                <th mat-header-cell *matHeaderCellDef>Type</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.exec_type}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="actions">
+                <th mat-header-cell *matHeaderCellDef></th>
+                <td mat-cell *matCellDef="let row">
+                    <button mat-icon-button routerLink='.' [queryParams]="{id: row.id}">
+                        <mat-icon>more_horiz</mat-icon>
+                    </button>
+                    |
+                    <button mat-icon-button (click)="deleteItem(row.id)">
+                        <mat-icon>delete</mat-icon>
+                    </button>
+                </td>
+            </ng-container>
+
+            <tr mat-header-row *matHeaderRowDef="displayedColumns"></tr>
+            <tr mat-row class="data-table-row" *matRowDef="let row; columns: displayedColumns;"></tr>
+            <tr class="mat-row" *matNoDataRow>
+                <td class="mat-cell">No data</td>
+            </tr>
+        </table>
+    </div>
+
+    <!-- <mat-paginator [length]="resultsLength" [pageSize]="30" aria-label="Select page of GitHub search results">
+    </mat-paginator> -->
+</div>

bin/u_panel/src/server/fe/src/app/core/tables/job.component.ts

@@ -0,0 +1,59 @@
+import { Component, OnInit } from '@angular/core';
+import { TablesComponent } from './table.component';
+import { JobModel } from '../models';
+import { JobInfoDialogComponent } from './dialogs';
+import { HttpErrorResponse } from '@angular/common/http';
+
+@Component({
+  selector: 'job-table',
+  templateUrl: './job.component.html',
+  styleUrls: ['./table.component.less']
+})
+export class JobComponent extends TablesComponent<JobModel> {
+  area = 'jobs' as const;
+  displayedColumns = ['id', 'alias', 'platform', 'schedule', 'exec_type', 'actions']
+
+  show_item_dialog(id: string | null) {
+    const show_dlg = (id: string, edit: boolean) => {
+      this.data_source!.getOne(id).then(resp => {
+        if (resp.status === 'ok') {
+          var dialog = this.infoDialog.open(JobInfoDialogComponent, {
+            data: resp.data as JobModel,
+            width: '1000px',
+          });
+          if (edit) {
+            dialog.componentInstance.is_preview = false
+          }
+
+          const saveSub = dialog.componentInstance.onSave.subscribe(result => {
+            this.data_source!.update(result)
+              .then(_ => {
+                this.openSnackBar("Saved", false)
+                this.loadTableData()
+              })
+              .catch((err: HttpErrorResponse) => this.openSnackBar(err.error))
+          })
+
+          dialog.afterClosed().subscribe(result => {
+            saveSub.unsubscribe()
+            this.router.navigate(['.'], { relativeTo: this.route })
+          })
+        } else {
+          this.openSnackBar(resp.data)
+        }
+      }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error))
+    }
+
+    if (id) {
+      show_dlg(id, false)
+    } else {
+      this.data_source!.create('"{}"').then(resp => {
+        if (resp.status === 'ok') {
+          show_dlg(resp.data[0], true)
+        } else {
+          this.openSnackBar(resp.data)
+        }
+      }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error))
+    }
+  }
+}

bin/u_panel/src/server/fe/src/app/core/tables/result.component.html

@@ -0,0 +1,81 @@
+<div class="mat-elevation-z8">
+
+    <div class="table-container">
+        <div class="loading-shade" *ngIf="isLoadingResults">
+            <mat-spinner *ngIf="isLoadingResults"></mat-spinner>
+        </div>
+        <mat-form-field appearance="standard">
+            <mat-label>Filter</mat-label>
+            <input matInput (keyup)="apply_filter($event)" #input>
+        </mat-form-field>
+        <button id="refresh_btn" mat-raised-button color="primary" (click)="loadTableData()">Refresh</button>
+
+        <table mat-table fixedLayout="true" [dataSource]="table_data" class="data-table" matSort matSortActive="id"
+            matSortDisableClear matSortDirection="desc">
+
+            <ng-container matColumnDef="id">
+                <th mat-header-cell *matHeaderCellDef>ID</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.id}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="alias">
+                <th mat-header-cell *matHeaderCellDef>Alias</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.alias}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="agent_id">
+                <th mat-header-cell *matHeaderCellDef>Agent</th>
+                <td mat-cell *matCellDef="let row">
+                    <a routerLink='/agents' [queryParams]="{id: row.agent_id}">{{row.agent_id}}</a>
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="job_id">
+                <th mat-header-cell *matHeaderCellDef>Job</th>
+                <td mat-cell *matCellDef="let row">
+                    <a routerLink='/jobs' [queryParams]="{id: row.job_id}">{{row.job_id}}</a>
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="state">
+                <th mat-header-cell *matHeaderCellDef>State</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.state}} {{(row.state === "Finished") ? '(' + row.retcode + ')' : ''}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="last_updated">
+                <th mat-header-cell *matHeaderCellDef>ID</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.updated.secs_since_epoch * 1000| date:'long'}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="actions">
+                <th mat-header-cell *matHeaderCellDef></th>
+                <td mat-cell *matCellDef="let row">
+                    <button mat-icon-button routerLink='.' [queryParams]="{id: row.id}">
+                        <mat-icon>more_horiz</mat-icon>
+                    </button>
+                    |
+                    <button mat-icon-button (click)="deleteItem(row.id)">
+                        <mat-icon>delete</mat-icon>
+                    </button>
+                </td>
+            </ng-container>
+
+            <tr mat-header-row *matHeaderRowDef="displayedColumns"></tr>
+            <tr mat-row class="data-table-row" *matRowDef="let row; columns: displayedColumns;"></tr>
+            <tr class="mat-row" *matNoDataRow>
+                <td class="mat-cell">No data</td>
+            </tr>
+        </table>
+    </div>
+
+    <!-- <mat-paginator [length]="resultsLength" [pageSize]="30" aria-label="Select page of GitHub search results">
+    </mat-paginator> -->
+</div>

bin/u_panel/src/server/fe/src/app/core/tables/result.component.ts

@@ -0,0 +1,41 @@
+import { Component, OnInit } from '@angular/core';
+import { TablesComponent } from './table.component';
+import { ResultModel } from '../models';
+import { ResultInfoDialogComponent } from './dialogs';
+import { HttpErrorResponse } from '@angular/common/http';
+
+@Component({
+  selector: 'results-table',
+  templateUrl: './result.component.html',
+  styleUrls: ['./table.component.less']
+})
+export class ResultComponent extends TablesComponent<ResultModel> {
+  area = 'map' as const;
+
+  displayedColumns = [
+    'id',
+    'alias',
+    'agent_id',
+    'job_id',
+    'state',
+    'last_updated',
+    'actions'
+  ];
+
+  show_item_dialog(id: string) {
+    this.data_source!.getOne(id).then(resp => {
+      if (resp.status === 'ok') {
+        const dialog = this.infoDialog.open(ResultInfoDialogComponent, {
+          data: resp.data as ResultModel,
+          width: '1000px',
+        });
+
+        dialog.afterClosed().subscribe(result => {
+          this.router.navigate(['.'], { relativeTo: this.route })
+        })
+      } else {
+        this.openSnackBar(resp.data)
+      }
+    }).catch((err: HttpErrorResponse) => this.openSnackBar(err.message))
+  }
+}

bin/u_panel/src/server/fe/src/app/core/tables/table.component.less

@@ -0,0 +1,32 @@
+.data-table {
+    width: 100%;
+}
+
+.table-container {
+    margin: 50px;
+}
+
+.loading-shade {
+    position: absolute;
+    top: 0;
+    left: 0;
+    bottom: 56px;
+    right: 0;
+    //background: rgba(0, 0, 0, 0.15);
+    z-index: 1;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+}
+
+#refresh_btn {
+    margin-left: 10px;
+}
+
+.data-table-row {
+    height: 30px;
+}
+
+.data-table-row:hover {
+    background: whitesmoke;
+}

bin/u_panel/src/server/fe/src/app/core/tables/table.component.ts

@@ -0,0 +1,84 @@
+import { OnInit, Directive } from '@angular/core';
+import { HttpClient } from '@angular/common/http';
+import { ApiTableService } from '../';
+import { MatTableDataSource } from '@angular/material/table';
+import { MatDialog } from '@angular/material/dialog';
+import { ApiModel, Area } from '../models';
+import { ActivatedRoute, Router } from '@angular/router';
+import { interval } from 'rxjs';
+import { MatSnackBar, MatSnackBarConfig } from '@angular/material/snack-bar';
+
+@Directive()
+export abstract class TablesComponent<T extends ApiModel> implements OnInit {
+  abstract area: Area;
+  data_source!: ApiTableService<T>;
+  table_data!: MatTableDataSource<T>;
+
+  isLoadingResults = true;
+
+  constructor(
+    public httpClient: HttpClient,
+    public infoDialog: MatDialog,
+    public route: ActivatedRoute,
+    public router: Router,
+    public snackBar: MatSnackBar
+  ) {
+    this.table_data = new MatTableDataSource;
+  }
+
+  ngOnInit() {
+    this.data_source = new ApiTableService(this.httpClient, this.area);
+    this.loadTableData();
+    this.route.queryParams.subscribe(params => {
+      const id = params['id']
+      const new_agent = params['new']
+      if (id) {
+        this.show_item_dialog(id);
+      }
+      if (new_agent) {
+        this.show_item_dialog(null);
+      }
+    })
+    //interval(10000).subscribe(_ => this.loadTableData());
+  }
+
+  async loadTableData() {
+    this.isLoadingResults = true;
+    //possibly needs try/catch
+    const data = await this.data_source!.getMany();
+    this.isLoadingResults = false;
+
+    if (typeof data.data !== 'string') {
+      this.table_data.data = data.data
+    } else {
+      alert(`Error: ${data}`)
+    };
+  }
+
+  apply_filter(event: Event) {
+    const filterValue = (event.target as HTMLInputElement).value;
+    this.table_data.filter = filterValue.trim().toLowerCase();
+  }
+
+  deleteItem(id: string) {
+    if (confirm(`Delete ${id}?`)) {
+      this.data_source!.delete(id).catch(this.openSnackBar)
+    }
+  }
+
+  openSnackBar(message: any, error: boolean = true) {
+    const msg = JSON.stringify(message)
+    const _config = (duration: number): MatSnackBarConfig => {
+      return {
+        horizontalPosition: 'right',
+        verticalPosition: 'bottom',
+        duration
+      }
+    }
+    const cfg = error ? _config(0) : _config(2000)
+    this.snackBar.open(msg, 'Ok', cfg);
+  }
+
+  abstract displayedColumns: string[];
+  abstract show_item_dialog(id: string | null): void;
+}

bin/u_panel/src/server/fe/src/app/core/utils.ts

@@ -0,0 +1,3 @@
+export function epochToStr(epoch: number): string {
+    return new Date(epoch * 1000).toLocaleString('en-GB')
+}

bin/u_panel/src/server/fe/src/environments/environment.prod.ts

@@ -0,0 +1,4 @@
+export const environment = {
+  production: true,
+  server: "",
+};

bin/u_panel/src/server/fe/src/environments/environment.ts

@@ -0,0 +1,17 @@
+// This file can be replaced during build by using the `fileReplacements` array.
+// `ng build` replaces `environment.ts` with `environment.prod.ts`.
+// The list of file replacements can be found in `angular.json`.
+
+export const environment = {
+  production: false,
+  server: "http://127.0.0.1:8080",
+};
+
+/*
+ * For easier debugging in development mode, you can import the following file
+ * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`.
+ *
+ * This import should be commented out in production mode because it will have a negative impact
+ * on performance if an error is thrown.
+ */
+// import 'zone.js/plugins/zone-error';  // Included with Angular CLI.

bin/u_panel/src/server/fe/src/index.html

@@ -0,0 +1,18 @@
+<!doctype html>
+<html lang="en">
+
+<head>
+  <meta charset="utf-8">
+  <title>Fe</title>
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <link rel="icon" type="image/x-icon" href="favicon.ico">
+  <link rel="preconnect" href="https://fonts.gstatic.com">
+  <link href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500&display=swap" rel="stylesheet">
+  <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
+</head>
+
+<body class="mat-typography">
+  <app-root></app-root>
+</body>
+
+</html>

bin/u_panel/src/server/fe/src/main.ts

@@ -0,0 +1,12 @@
+import { enableProdMode } from '@angular/core';
+import { platformBrowserDynamic } from '@angular/platform-browser-dynamic';
+
+import { AppModule } from './app/app.module';
+import { environment } from './environments/environment';
+
+if (environment.production) {
+  enableProdMode();
+}
+
+platformBrowserDynamic().bootstrapModule(AppModule)
+  .catch(err => console.error(err));

bin/u_panel/src/server/fe/src/polyfills.ts

@@ -0,0 +1,53 @@
+/**
+ * This file includes polyfills needed by Angular and is loaded before the app.
+ * You can add your own extra polyfills to this file.
+ *
+ * This file is divided into 2 sections:
+ *   1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers.
+ *   2. Application imports. Files imported after ZoneJS that should be loaded before your main
+ *      file.
+ *
+ * The current setup is for so-called "evergreen" browsers; the last versions of browsers that
+ * automatically update themselves. This includes recent versions of Safari, Chrome (including
+ * Opera), Edge on the desktop, and iOS and Chrome on mobile.
+ *
+ * Learn more in https://angular.io/guide/browser-support
+ */
+
+/***************************************************************************************************
+ * BROWSER POLYFILLS
+ */
+
+/**
+ * By default, zone.js will patch all possible macroTask and DomEvents
+ * user can disable parts of macroTask/DomEvents patch by setting following flags
+ * because those flags need to be set before `zone.js` being loaded, and webpack
+ * will put import in the top of bundle, so user need to create a separate file
+ * in this directory (for example: zone-flags.ts), and put the following flags
+ * into that file, and then add the following code before importing zone.js.
+ * import './zone-flags';
+ *
+ * The flags allowed in zone-flags.ts are listed here.
+ *
+ * The following flags will work for all browsers.
+ *
+ * (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame
+ * (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick
+ * (window as any).__zone_symbol__UNPATCHED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames
+ *
+ *  in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js
+ *  with the following flag, it will bypass `zone.js` patch for IE/Edge
+ *
+ *  (window as any).__Zone_enable_cross_context_check = true;
+ *
+ */
+
+/***************************************************************************************************
+ * Zone JS is required by default for Angular itself.
+ */
+import 'zone.js';  // Included with Angular CLI.
+
+
+/***************************************************************************************************
+ * APPLICATION IMPORTS
+ */

bin/u_panel/src/server/fe/src/styles.less

@@ -0,0 +1,4 @@
+/* You can add global styles to this file, and also import other style files */
+
+html, body { height: 100%; }
+body { margin: 0; font-family: Roboto, "Helvetica Neue", sans-serif; }

bin/u_panel/src/server/fe/src/test.ts

@@ -0,0 +1,26 @@
+// This file is required by karma.conf.js and loads recursively all the .spec and framework files
+
+import 'zone.js/testing';
+import { getTestBed } from '@angular/core/testing';
+import {
+  BrowserDynamicTestingModule,
+  platformBrowserDynamicTesting
+} from '@angular/platform-browser-dynamic/testing';
+
+declare const require: {
+  context(path: string, deep?: boolean, filter?: RegExp): {
+    <T>(id: string): T;
+    keys(): string[];
+  };
+};
+
+// First, initialize the Angular testing environment.
+getTestBed().initTestEnvironment(
+  BrowserDynamicTestingModule,
+  platformBrowserDynamicTesting(),
+);
+
+// Then we find all the tests.
+const context = require.context('./', true, /\.spec\.ts$/);
+// And load the modules.
+context.keys().map(context);

bin/u_panel/src/server/fe/tsconfig.app.json

@@ -0,0 +1,15 @@
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./out-tsc/app",
+    "types": []
+  },
+  "files": [
+    "src/main.ts",
+    "src/polyfills.ts"
+  ],
+  "include": [
+    "src/**/*.d.ts"
+  ]
+}

bin/u_panel/src/server/fe/tsconfig.json

@@ -0,0 +1,32 @@
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+  "compileOnSave": false,
+  "compilerOptions": {
+    "baseUrl": "./",
+    "outDir": "./dist/out-tsc",
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "noImplicitOverride": true,
+    "noPropertyAccessFromIndexSignature": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "sourceMap": true,
+    "declaration": false,
+    "downlevelIteration": true,
+    "experimentalDecorators": true,
+    "moduleResolution": "node",
+    "importHelpers": true,
+    "target": "es2017",
+    "module": "es2020",
+    "lib": [
+      "es2020",
+      "dom"
+    ]
+  },
+  "angularCompilerOptions": {
+    "enableI18nLegacyMessageIdFormat": false,
+    "strictInjectionParameters": true,
+    "strictInputAccessModifiers": true,
+    "strictTemplates": true
+  }
+}

bin/u_panel/src/server/fe/tsconfig.spec.json

@@ -0,0 +1,18 @@
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./out-tsc/spec",
+    "types": [
+      "jasmine"
+    ]
+  },
+  "files": [
+    "src/test.ts",
+    "src/polyfills.ts"
+  ],
+  "include": [
+    "src/**/*.spec.ts",
+    "src/**/*.d.ts"
+  ]
+}

bin/u_panel/src/server/mod.rs

@@ -0,0 +1,98 @@
+mod error;
+
+use crate::{process_cmd, Args};
+use actix_cors::Cors;
+use actix_web::{get, middleware::Logger, post, web, App, HttpResponse, HttpServer, Responder};
+use error::Error;
+use futures_util::StreamExt;
+use rust_embed::RustEmbed;
+use std::borrow::Cow;
+use structopt::StructOpt;
+use u_lib::{api::ClientHandler, unwrap_enum};
+
+#[derive(RustEmbed)]
+#[folder = "./src/server/fe/dist/fe/"]
+struct Files;
+
+impl Files {
+    pub fn get_static(path: impl AsRef<str>) -> Option<&'static [u8]> {
+        let file = Self::get(path.as_ref())?.data;
+        Some(unwrap_enum!(file, Cow::Borrowed))
+    }
+}
+
+async fn spa_main() -> impl Responder {
+    let index = Files::get_static("index.html").unwrap();
+    HttpResponse::Ok().body(index)
+}
+
+#[get("/core/{path}")]
+async fn resources_adapter(path: web::Path<(String,)>) -> impl Responder {
+    let path = path.into_inner().0;
+    let mimetype = mime_guess::from_path(&path).first_or_octet_stream();
+
+    match Files::get_static(path) {
+        Some(data) => HttpResponse::Ok()
+            .content_type(mimetype.to_string())
+            .body(data),
+        None => HttpResponse::NotFound().finish(),
+    }
+}
+
+#[post("/cmd/")]
+async fn send_cmd(
+    mut body: web::Payload,
+    client: web::Data<ClientHandler>,
+) -> Result<impl Responder, Error> {
+    let mut bytes = web::BytesMut::new();
+
+    while let Some(item) = body.next().await {
+        bytes.extend_from_slice(
+            &item.map_err(|e| Error::JustError(format!("payload loading failure: {e}")))?,
+        );
+    }
+
+    let cmd = String::from_utf8(bytes.to_vec())
+        .map_err(|_| Error::JustError("cmd contains non-utf8 data".to_string()))?;
+    let mut cmd = shlex::split(&cmd).ok_or(Error::JustError("argparse failed".to_string()))?;
+
+    info!("cmd: {:?}", cmd);
+    cmd.insert(0, String::from("u_panel"));
+
+    let parsed_cmd = Args::from_iter_safe(cmd)?;
+    let result = process_cmd(client.as_ref().clone(), parsed_cmd).await;
+    let result_string = result.to_string();
+
+    let response = if result.is_ok() {
+        HttpResponse::Ok().body(result_string)
+    } else if result.is_err() {
+        HttpResponse::BadRequest().body(result_string)
+    } else {
+        unreachable!()
+    };
+
+    Ok(response)
+}
+
+pub async fn serve(client: ClientHandler) -> anyhow::Result<()> {
+    info!("Connecting to u_server...");
+    client.ping().await?;
+
+    let addr = "127.0.0.1:8080";
+    info!("Connected, instanciating u_panel at http://{}", addr);
+
+    HttpServer::new(move || {
+        App::new()
+            .wrap(Logger::default())
+            .wrap(Cors::permissive())
+            .app_data(web::Data::new(client.clone()))
+            .service(send_cmd)
+            .service(resources_adapter)
+            .service(web::resource("/").to(spa_main))
+            .service(web::resource("/{_}").to(spa_main))
+    })
+    .bind(addr)?
+    .run()
+    .await?;
+    Ok(())
+}

bin/u_run/Cargo.toml

@@ -7,5 +7,5 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-nix = "0.17"
 libc = "^0.2"
+nix = "0.17"

bin/u_server/Cargo.toml

@@ -1,39 +1,27 @@
 [package]
 authors = ["plazmoid <kronos44@mail.ru>"]
-edition = "2018"
+edition = "2021"
 name = "u_server"
 version = "0.1.0"
 
 [dependencies]
-log = "0.4.11"
+anyhow = { workspace = true }
-simplelog = "0.10"
+diesel = { version = "1.4.5", features = ["postgres", "uuid"] }
-thiserror = "*"
-warp = { version = "0.3.1", features = ["tls"] }
-uuid = { version = "0.6.5", features = ["serde", "v4"] }
-once_cell = "1.7.2"
 hyper = "0.14"
-mockall = "0.9.1"
+once_cell = "1.7.2"
-mockall_double = "0.2"
 openssl = "*"
+serde = { workspace = true }
+serde_json = { workspace = true }
+thiserror = { workspace = true }
+tracing = { workspace = true }
+tokio = { workspace = true, features = ["macros"] }
+uuid = { workspace = true, features = ["serde", "v4"] }
+u_lib = { path = "../../lib/u_lib", features = ["server"] }
+warp = { version = "0.3.1", features = ["tls"] }
 
-[dependencies.diesel]
-features = ["postgres", "uuid"]
-version = "1.4.5"
-
-[dependencies.serde]
-features = ["derive"]
-version = "1.0.114"
-
-[dependencies.tokio]
-features = ["macros"]
-version = "1.9"
-
-[dependencies.u_lib]
-path = "../../lib/u_lib"
-version = "*"
 
 [dev-dependencies]
-test-case = "1.1.0"
+rstest = "0.12"
 
 [lib]
 name = "u_server_lib"

bin/u_server/src/db.rs

@@ -1,109 +1,130 @@
+use crate::error::Error as ServerError;
 use diesel::{pg::PgConnection, prelude::*, result::Error as DslError};
 use once_cell::sync::OnceCell;
-use std::{
+use serde::Deserialize;
-    env,
+use std::sync::{Mutex, MutexGuard};
-    sync::{Arc, Mutex, MutexGuard},
-};
 use u_lib::{
     models::{schema, Agent, AssignedJob, JobMeta, JobState},
-    ULocalError, ULocalResult,
+    utils::load_env,
 };
 use uuid::Uuid;
 
+type Result<T> = std::result::Result<T, ServerError>;
+
 pub struct UDB {
-    pub conn: PgConnection,
+    conn: PgConnection,
 }
 
-static DB: OnceCell<Arc<Mutex<UDB>>> = OnceCell::new();
+static DB: OnceCell<Mutex<UDB>> = OnceCell::new();
+
+#[derive(Deserialize)]
+struct DBEnv {
+    db_host: String,
+    db_name: String,
+    db_user: String,
+    db_password: String,
+}
 
-#[cfg_attr(test, automock)]
 impl UDB {
-    pub fn lock_db() -> MutexGuard<'static, UDB> {
+    pub fn lock_db() -> MutexGuard<'static, Self> {
         DB.get_or_init(|| {
-            let _getenv = |v| env::var(v).unwrap();
+            let env = load_env::<DBEnv>().unwrap();
-            let db_host = _getenv("DB_HOST");
-            let db_name = _getenv("DB_NAME");
-            let db_user = _getenv("DB_USER");
-            let db_password = _getenv("DB_PASSWORD");
             let db_url = format!(
                 "postgres://{}:{}@{}/{}",
-                db_user, db_password, db_host, db_name
+                env.db_user, env.db_password, env.db_host, env.db_name
             );
-            let conn = PgConnection::establish(&db_url).unwrap();
+            let instance = UDB {
-            let instance = UDB { conn };
+                conn: PgConnection::establish(&db_url).unwrap(),
-            Arc::new(Mutex::new(instance))
+            };
+            Mutex::new(instance)
         })
         .lock()
         .unwrap()
     }
 
-    pub fn insert_jobs(&self, job_metas: &[JobMeta]) -> ULocalResult<()> {
+    pub fn insert_jobs(&self, job_metas: &[JobMeta]) -> Result<Vec<Uuid>> {
         use schema::jobs;
+
         diesel::insert_into(jobs::table)
             .values(job_metas)
-            .execute(&self.conn)?;
+            .get_results(&self.conn)
-        Ok(())
+            .map(|rows| rows.iter().map(|job: &JobMeta| job.id).collect())
+            .map_err(with_err_ctx("Can't insert jobs"))
     }
 
-    pub fn get_jobs(&self, uid: Option<Uuid>) -> ULocalResult<Vec<JobMeta>> {
+    pub fn get_jobs(&self, ouid: Option<Uuid>) -> Result<Vec<JobMeta>> {
         use schema::jobs;
-        let result = if uid.is_some() {
+
-            jobs::table
+        match ouid {
-                .filter(jobs::id.eq(uid.unwrap()))
+            Some(uid) => jobs::table
-                .get_results::<JobMeta>(&self.conn)?
+                .filter(jobs::id.eq(uid))
-        } else {
+                .get_results::<JobMeta>(&self.conn),
-            jobs::table.load::<JobMeta>(&self.conn)?
+            None => jobs::table.load::<JobMeta>(&self.conn),
-        };
+        }
-        Ok(result)
+        .map_err(with_err_ctx("Can't get exact jobs"))
     }
 
-    pub fn find_job_by_alias(&self, alias: &str) -> ULocalResult<JobMeta> {
+    pub fn find_job_by_alias(&self, alias: &str) -> Result<Option<JobMeta>> {
         use schema::jobs;
+
         let result = jobs::table
             .filter(jobs::alias.eq(alias))
-            .first::<JobMeta>(&self.conn)?;
+            .first::<JobMeta>(&self.conn)
+            .optional()
+            .map_err(with_err_ctx(format!("Can't find job by alias {alias}")))?;
         Ok(result)
     }
 
-    pub fn insert_agent(&self, agent: &Agent) -> ULocalResult<()> {
+    pub fn insert_agent(&self, agent: &Agent) -> Result<()> {
         use schema::agents;
+
         diesel::insert_into(agents::table)
             .values(agent)
             .on_conflict(agents::id)
             .do_update()
             .set(agent)
-            .execute(&self.conn)?;
+            .execute(&self.conn)
+            .map_err(with_err_ctx(format!("Can't insert agent {agent:x?}")))?;
         Ok(())
     }
 
-    pub fn get_agents(&self, uid: Option<Uuid>) -> ULocalResult<Vec<Agent>> {
+    pub fn insert_result(&self, result: &AssignedJob) -> Result<()> {
+        use schema::results;
+
+        diesel::insert_into(results::table)
+            .values(result)
+            .execute(&self.conn)
+            .map_err(with_err_ctx(format!("Can't insert result {result:x?}")))?;
+        Ok(())
+    }
+
+    pub fn get_agents(&self, ouid: Option<Uuid>) -> Result<Vec<Agent>> {
         use schema::agents;
-        let result = if uid.is_some() {
+
-            agents::table
+        match ouid {
-                .filter(agents::id.eq(uid.unwrap()))
+            Some(uid) => agents::table
-                .load::<Agent>(&self.conn)?
+                .filter(agents::id.eq(uid))
-        } else {
+                .load::<Agent>(&self.conn),
-            agents::table.load::<Agent>(&self.conn)?
+            None => agents::table.load::<Agent>(&self.conn),
-        };
+        }
-        Ok(result)
+        .map_err(with_err_ctx(format!("Can't get agent(s) {ouid:?}")))
     }
 
-    pub fn update_job_status(&self, uid: Uuid, status: JobState) -> ULocalResult<()> {
+    pub fn update_job_status(&self, uid: Uuid, status: JobState) -> Result<()> {
         use schema::results;
+
         diesel::update(results::table)
             .filter(results::id.eq(uid))
             .set(results::state.eq(status))
-            .execute(&self.conn)?;
+            .execute(&self.conn)
+            .map_err(with_err_ctx(format!("Can't update status of job {uid}")))?;
         Ok(())
     }
 
     //TODO: filters possibly could work in a wrong way, check
-    pub fn get_exact_jobs(
+    pub fn get_exact_jobs(&self, uid: Option<Uuid>, personal: bool) -> Result<Vec<AssignedJob>> {
-        &self,
-        uid: Option<Uuid>,
-        personal: bool,
-    ) -> ULocalResult<Vec<AssignedJob>> {
         use schema::results;
+
         let mut q = results::table.into_boxed();
         /*if uid.is_some() {
             q = q.filter(results::agent_id.eq(uid.unwrap()))
@@ -120,115 +141,101 @@ impl UDB {
                 .or_filter(results::job_id.eq(uid.unwrap()))
                 .or_filter(results::id.eq(uid.unwrap()))
         }
-        let result = q.load::<AssignedJob>(&self.conn)?;
+        let result = q
+            .load::<AssignedJob>(&self.conn)
+            .map_err(with_err_ctx("Can't get exact jobs"))?;
         Ok(result)
     }
 
-    pub fn set_jobs_for_agent(
+    pub fn set_jobs_for_agent(&self, agent_uid: &Uuid, job_uids: &[Uuid]) -> Result<Vec<Uuid>> {
-        &self,
+        use schema::results;
-        agent_uid: &Uuid,
+
-        job_uids: &[Uuid],
-    ) -> ULocalResult<Vec<Uuid>> {
-        use schema::{agents::dsl::agents, jobs::dsl::jobs, results};
-        if let Err(DslError::NotFound) = agents.find(agent_uid).first::<Agent>(&self.conn) {
-            return Err(ULocalError::NotFound(agent_uid.to_string()));
-        }
-        let not_found_jobs = job_uids
-            .iter()
-            .filter_map(|job_uid| {
-                if let Err(DslError::NotFound) = jobs.find(job_uid).first::<JobMeta>(&self.conn) {
-                    Some(job_uid.to_string())
-                } else {
-                    None
-                }
-            })
-            .collect::<Vec<String>>();
-        if not_found_jobs.len() > 0 {
-            return Err(ULocalError::NotFound(not_found_jobs.join(", ")));
-        }
         let job_requests = job_uids
             .iter()
-            .map(|job_uid| {
+            .map(|job_uid| AssignedJob {
-                info!("set_jobs_for_agent: set {} for {}", job_uid, agent_uid);
+                job_id: *job_uid,
-                AssignedJob {
+                agent_id: *agent_uid,
-                    job_id: *job_uid,
+                ..Default::default()
-                    agent_id: *agent_uid,
-                    ..Default::default()
-                }
             })
             .collect::<Vec<AssignedJob>>();
+
         diesel::insert_into(results::table)
             .values(&job_requests)
-            .execute(&self.conn)?;
+            .execute(&self.conn)
-        let assigned_uids = job_requests.iter().map(|aj| aj.id).collect();
+            .map_err(with_err_ctx(format!(
-        Ok(assigned_uids)
+                "Can't setup jobs {job_uids:?} for agent {agent_uid:?}"
+            )))?;
+
+        Ok(job_requests.iter().map(|aj| aj.id).collect())
     }
 
-    pub fn del_jobs(&self, uids: &Vec<Uuid>) -> ULocalResult<usize> {
+    pub fn del_jobs(&self, uids: &[Uuid]) -> Result<usize> {
         use schema::jobs;
+
         let mut affected = 0;
         for &uid in uids {
             let deleted = diesel::delete(jobs::table)
                 .filter(jobs::id.eq(uid))
-                .execute(&self.conn)?;
+                .execute(&self.conn)
+                .map_err(with_err_ctx("Can't delete jobs"))?;
             affected += deleted;
         }
         Ok(affected)
     }
 
-    pub fn del_results(&self, uids: &Vec<Uuid>) -> ULocalResult<usize> {
+    pub fn del_results(&self, uids: &[Uuid]) -> Result<usize> {
         use schema::results;
+
         let mut affected = 0;
         for &uid in uids {
             let deleted = diesel::delete(results::table)
                 .filter(results::id.eq(uid))
-                .execute(&self.conn)?;
+                .execute(&self.conn)
+                .map_err(with_err_ctx("Can't delete results"))?;
             affected += deleted;
         }
         Ok(affected)
     }
 
-    pub fn del_agents(&self, uids: &Vec<Uuid>) -> ULocalResult<usize> {
+    pub fn del_agents(&self, uids: &[Uuid]) -> Result<usize> {
         use schema::agents;
+
         let mut affected = 0;
         for &uid in uids {
             let deleted = diesel::delete(agents::table)
                 .filter(agents::id.eq(uid))
-                .execute(&self.conn)?;
+                .execute(&self.conn)
+                .map_err(with_err_ctx("Can't delete agents"))?;
             affected += deleted;
         }
         Ok(affected)
     }
-}
+
-/*
+    pub fn update_agent(&self, agent: &Agent) -> Result<()> {
-#[cfg(test)]
+        agent
-mod tests {
+            .save_changes::<Agent>(&self.conn)
-    use super::*;
+            .map_err(with_err_ctx(format!("Can't update agent {agent:x?}")))?;
-
+        Ok(())
-    fn setup_db() -> Storage {
+    }
-        return UDB::new().unwrap();
+
-    }
+    pub fn update_job(&self, job: &JobMeta) -> Result<()> {
-
+        job.save_changes::<JobMeta>(&self.conn)
-    #[tokio::test]
+            .map_err(with_err_ctx(format!("Can't update job {job:x?}")))?;
-    async fn test_add_agent() {
+        Ok(())
-        let db = setup_db();
-        let agent = IAgent {
-            alias: None,
-            id: "000-000".to_string(),
-            hostname: "test".to_string(),
-            is_root: false,
-            is_root_allowed: false,
-            platform: "linux".to_string(),
-            status: None,
-            token: None,
-            username: "test".to_string()
-        };
-        db.lock().unwrap().new_agent(agent).unwrap();
-        let result = db.lock().unwrap().get_agents().unwrap();
-        assert_eq!(
-            result[0].username,
-            "test".to_string()
-        )
     }
+
+    pub fn update_result(&self, result: &AssignedJob) -> Result<()> {
+        debug!(
+            "updating result: id = {}, job_id = {}, agent_id = {}",
+            result.id, result.job_id, result.agent_id
+        );
+        result
+            .save_changes::<AssignedJob>(&self.conn)
+            .map_err(with_err_ctx(format!("Can't update result {result:x?}")))?;
+        Ok(())
+    }
+}
+
+fn with_err_ctx(msg: impl AsRef<str>) -> impl Fn(DslError) -> ServerError {
+    move |err| ServerError::DBErrorCtx(format!("{}, reason: {err}", msg.as_ref()))
 }
-*/

bin/u_server/src/error.rs

@@ -0,0 +1,59 @@
+use diesel::result::Error as DslError;
+use thiserror::Error;
+use warp::{
+    http::StatusCode,
+    reject::Reject,
+    reply::{with_status, Response},
+    Reply,
+};
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("Error processing {0}")]
+    ProcessingError(String),
+
+    #[error(transparent)]
+    DBError(#[from] DslError),
+
+    #[error("DB error: {0}")]
+    DBErrorCtx(String),
+
+    #[error("General error: {0}")]
+    Other(String),
+}
+
+impl Reject for Error {}
+
+pub struct RejResponse {
+    message: String,
+    status: StatusCode,
+}
+
+impl RejResponse {
+    pub fn not_found(msg: impl Into<String>) -> Self {
+        Self {
+            message: msg.into(),
+            status: StatusCode::NOT_FOUND,
+        }
+    }
+
+    pub fn bad_request(msg: impl Into<String>) -> Self {
+        Self {
+            message: msg.into(),
+            status: StatusCode::BAD_REQUEST,
+        }
+    }
+
+    pub fn internal() -> Self {
+        Self {
+            message: "INTERNAL_SERVER_ERROR".to_string(),
+            status: StatusCode::INTERNAL_SERVER_ERROR,
+        }
+    }
+}
+
+impl Reply for RejResponse {
+    fn into_response(self) -> Response {
+        with_status(self.message, self.status).into_response()
+    }
+}

bin/u_server/src/filters.rs

@@ -1,87 +0,0 @@
-use crate::handlers::Endpoints;
-use serde::de::DeserializeOwned;
-use std::env;
-use u_lib::{
-    messaging::{AsMsg, BaseMessage},
-    models::*,
-};
-use uuid::Uuid;
-use warp::{body, Filter, Rejection, Reply};
-
-fn get_content<M>() -> impl Filter<Extract = (BaseMessage<'static, M>,), Error = Rejection> + Clone
-where
-    M: AsMsg + Sync + Send + DeserializeOwned + 'static,
-{
-    body::content_length_limit(1024 * 64).and(body::json::<BaseMessage<M>>())
-}
-
-pub fn make_filters() -> impl Filter<Extract = (impl Reply,), Error = Rejection> + Clone {
-    let infallible_none = |_| async { Ok::<(Option<Uuid>,), std::convert::Infallible>((None,)) };
-
-    let get_agents = warp::get()
-        .and(warp::path("get_agents"))
-        .and(
-            warp::path::param::<Uuid>()
-                .map(Some)
-                .or_else(infallible_none),
-        )
-        .and_then(Endpoints::get_agents);
-
-    let upload_jobs = warp::post()
-        .and(warp::path("upload_jobs"))
-        .and(get_content::<Vec<JobMeta>>())
-        .and_then(Endpoints::upload_jobs);
-
-    let get_jobs = warp::get()
-        .and(warp::path("get_jobs"))
-        .and(
-            warp::path::param::<Uuid>()
-                .map(Some)
-                .or_else(infallible_none),
-        )
-        .and_then(Endpoints::get_jobs);
-
-    let get_agent_jobs = warp::get()
-        .and(warp::path("get_agent_jobs"))
-        .and(
-            warp::path::param::<Uuid>()
-                .map(Some)
-                .or_else(infallible_none),
-        )
-        .and_then(|uid| Endpoints::get_agent_jobs(uid));
-
-    let get_personal_jobs = warp::get()
-        .and(warp::path("get_personal_jobs"))
-        .and(warp::path::param::<Uuid>().map(Some))
-        .and_then(|uid| Endpoints::get_personal_jobs(uid));
-
-    let del = warp::get()
-        .and(warp::path("del"))
-        .and(warp::path::param::<Uuid>())
-        .and_then(Endpoints::del);
-
-    let set_jobs = warp::post()
-        .and(warp::path("set_jobs"))
-        .and(warp::path::param::<Uuid>())
-        .and(get_content::<Vec<String>>())
-        .and_then(Endpoints::set_jobs);
-
-    let report = warp::post()
-        .and(warp::path("report"))
-        .and(get_content::<Vec<ExecResult>>().and_then(Endpoints::report));
-
-    let auth_token = format!("Bearer {}", env::var("ADMIN_AUTH_TOKEN").unwrap()).into_boxed_str();
-    let auth_header = warp::header::exact("authorization", Box::leak(auth_token));
-
-    let auth_zone = (get_agents
-        .or(get_jobs)
-        .or(upload_jobs)
-        .or(del)
-        .or(set_jobs)
-        .or(get_agent_jobs))
-    .and(auth_header);
-
-    let agent_zone = get_jobs.clone().or(get_personal_jobs).or(report);
-
-    auth_zone.or(agent_zone)
-}

bin/u_server/src/handlers.rs

@@ -1,171 +1,162 @@
 use crate::db::UDB;
-use diesel::SaveChangesDsl;
+use crate::error::Error;
-use hyper::Body;
-use serde::Serialize;
 use u_lib::{
-    messaging::{AsMsg, BaseMessage},
+    messaging::{AsMsg, BaseMessage, Reportable},
-    models::{Agent, AgentState, AssignedJob, ExecResult, JobMeta, JobState},
+    models::*,
-    ULocalError,
+    utils::OneOrVec,
 };
 use uuid::Uuid;
-use warp::{
+use warp::Rejection;
-    http::{Response, StatusCode},
-    Rejection, Reply,
-};
-
-pub fn build_response<S: Into<Body>>(code: StatusCode, body: S) -> Response<Body> {
-    Response::builder().status(code).body(body.into()).unwrap()
-}
-
-pub fn build_ok<S: Into<Body>>(body: S) -> Response<Body> {
-    build_response(StatusCode::OK, body)
-}
-
-pub fn build_err<S: ToString>(body: S) -> Response<Body> {
-    build_response(StatusCode::BAD_REQUEST, body.to_string())
-}
 
-pub fn build_message<M: AsMsg + Serialize>(m: M) -> Response<Body> {
+type EndpResult<T> = Result<T, Rejection>;
-    warp::reply::json(&m.as_message()).into_response()
-}
 
 pub struct Endpoints;
 
-#[cfg_attr(test, automock)]
 impl Endpoints {
-    pub async fn add_agent(msg: Agent) -> Result<Response<Body>, Rejection> {
+    pub async fn add_agent(msg: Agent) -> EndpResult<()> {
-        info!("hnd: add_agent");
+        UDB::lock_db().insert_agent(&msg).map_err(From::from)
-        UDB::lock_db()
-            .insert_agent(&msg)
-            .map(|_| build_ok(""))
-            .or_else(|e| Ok(build_err(e)))
     }
 
-    pub async fn get_agents(uid: Option<Uuid>) -> Result<Response<Body>, Rejection> {
+    pub async fn get_agents(uid: Option<Uuid>) -> EndpResult<Vec<Agent>> {
-        info!("hnd: get_agents");
+        UDB::lock_db().get_agents(uid).map_err(From::from)
-        UDB::lock_db()
-            .get_agents(uid)
-            .map(|m| build_message(m))
-            .or_else(|e| Ok(build_err(e)))
     }
 
-    pub async fn get_jobs(uid: Option<Uuid>) -> Result<Response<Body>, Rejection> {
+    pub async fn get_jobs(uid: Option<Uuid>) -> EndpResult<Vec<JobMeta>> {
-        info!("hnd: get_jobs");
+        UDB::lock_db().get_jobs(uid).map_err(From::from)
-        UDB::lock_db()
-            .get_jobs(uid)
-            .map(|m| build_message(m))
-            .or_else(|e| Ok(build_err(e)))
     }
 
-    pub async fn get_agent_jobs(uid: Option<Uuid>) -> Result<Response<Body>, Rejection> {
+    pub async fn get_agent_jobs(uid: Option<Uuid>) -> EndpResult<Vec<AssignedJob>> {
-        info!("hnd: get_agent_jobs");
         UDB::lock_db()
             .get_exact_jobs(uid, false)
-            .map(|m| build_message(m))
+            .map_err(From::from)
-            .or_else(|e| Ok(build_err(e)))
     }
 
-    pub async fn get_personal_jobs(uid: Option<Uuid>) -> Result<Response<Body>, Rejection> {
+    pub async fn get_personal_jobs(uid: Uuid) -> EndpResult<Vec<AssignedJob>> {
-        info!("hnd: get_personal_jobs");
+        let db = UDB::lock_db();
-        let agents = UDB::lock_db().get_agents(uid).unwrap();
+        let mut agents = db.get_agents(Some(uid))?;
-        if agents.len() == 0 {
+        if agents.is_empty() {
-            let db = UDB::lock_db();
+            let new_agent = Agent::with_id(uid);
-            db.insert_agent(&Agent::with_id(uid.unwrap())).unwrap();
+            db.insert_agent(&new_agent)?;
-            let job = db.find_job_by_alias("agent_hello").unwrap();
+            let job = db
-            if let Err(e) = db.set_jobs_for_agent(&uid.unwrap(), &[job.id]) {
+                .find_job_by_alias("agent_hello")?
-                return Ok(build_err(e));
+                .expect("agent_hello job not found");
-            }
+            db.set_jobs_for_agent(&uid, &[job.id])?;
+        } else {
+            let mut agent = agents.pop().unwrap();
+            agent.touch();
+            db.update_agent(&agent)?;
         }
-        let result = UDB::lock_db().get_exact_jobs(uid, true);
+        let result = db.get_exact_jobs(Some(uid), true)?;
-        match result {
+
-            Ok(r) => {
+        for j in result.iter() {
-                let db = UDB::lock_db();
+            db.update_job_status(j.id, JobState::Running)?;
-                for j in r.iter() {
-                    db.update_job_status(j.id, JobState::Running).unwrap();
-                }
-                Ok(build_message(r))
-            }
-            Err(e) => Ok(build_err(e)),
         }
+        Ok(result)
     }
 
-    pub async fn upload_jobs(
+    pub async fn upload_jobs(msg: BaseMessage<'static, Vec<JobMeta>>) -> EndpResult<Vec<Uuid>> {
-        msg: BaseMessage<'static, Vec<JobMeta>>,
-    ) -> Result<Response<Body>, Rejection> {
-        info!("hnd: upload_jobs");
         UDB::lock_db()
             .insert_jobs(&msg.into_inner())
-            .map(|_| build_ok(""))
+            .map_err(From::from)
-            .or_else(|e| Ok(build_err(e)))
     }
 
-    pub async fn del(uid: Uuid) -> Result<Response<Body>, Rejection> {
+    pub async fn del(uid: Uuid) -> EndpResult<usize> {
-        info!("hnd: del");
         let db = UDB::lock_db();
         let del_fns = &[UDB::del_agents, UDB::del_jobs, UDB::del_results];
         for del_fn in del_fns {
-            let affected = del_fn(&db, &vec![uid]).unwrap();
+            let affected = del_fn(&db, &[uid])?;
             if affected > 0 {
-                return Ok(build_message(affected as i32));
+                return Ok(affected);
             }
         }
-        Ok(build_message(0))
+        Ok(0)
     }
 
     pub async fn set_jobs(
         agent_uid: Uuid,
         msg: BaseMessage<'static, Vec<String>>,
-    ) -> Result<Response<Body>, Rejection> {
+    ) -> EndpResult<Vec<Uuid>> {
-        info!("hnd: set_jobs_by_alias, agent: {}", agent_uid);
+        msg.into_inner()
-        let jobs: Result<Vec<Uuid>, ULocalError> = msg
-            .into_inner()
             .into_iter()
             .map(|ident| {
-                info!("hnd: set_jobs_by_alias, job: {}", ident);
+                Uuid::parse_str(&ident).or_else(|_| {
-                Uuid::parse_str(&ident)
+                    let job_from_db = UDB::lock_db().find_job_by_alias(&ident);
-                    .or_else(|_| UDB::lock_db().find_job_by_alias(&ident).map(|j| j.id))
+                    match job_from_db {
+                        Ok(job) => match job {
+                            Some(j) => Ok(j.id),
+                            None => Err(Error::ProcessingError(format!("unknown ident {ident}"))),
+                        },
+                        Err(e) => Err(e),
+                    }
+                })
             })
-            .collect();
+            .collect::<Result<Vec<Uuid>, Error>>()
-        match jobs {
+            .and_then(|j| UDB::lock_db().set_jobs_for_agent(&agent_uid, &j))
-            Ok(j) => UDB::lock_db()
+            .map_err(From::from)
-                .set_jobs_for_agent(&agent_uid, &j)
-                .map(|assigned_uids| build_message(assigned_uids))
-                .or_else(|e| Ok(build_err(e))),
-            Err(e) => Ok(build_err(e)),
-        }
     }
 
-    pub async fn report(
+    pub async fn report<Data: OneOrVec<Reportable> + AsMsg + 'static>(
-        msg: BaseMessage<'static, Vec<ExecResult>>,
+        msg: BaseMessage<'static, Data>,
-    ) -> Result<Response<Body>, Rejection> {
+    ) -> EndpResult<()> {
-        info!("hnd: report");
         let id = msg.id;
-        let mut failed = vec![];
+        for entry in msg.into_inner().into_vec() {
-        for entry in msg.into_inner() {
             match entry {
-                ExecResult::Assigned(res) => {
+                Reportable::Assigned(mut result) => {
-                    if id != res.agent_id {
+                    let result_agent_id = &result.agent_id;
+                    if id != *result_agent_id {
+                        warn!("Ids are not equal! actual id: {id}, id from job: {result_agent_id}");
                         continue;
                     }
-                    let db = UDB::lock_db();
+                    result.state = JobState::Finished;
-                    if let Err(e) = res
+                    result.touch();
-                        .save_changes::<AssignedJob>(&db.conn)
+                    match result.exec_type {
-                        .map_err(ULocalError::from)
+                        JobType::Init => match &result.result {
-                    {
+                            Some(rbytes) => {
-                        failed.push(e.to_string())
+                                let mut agent: Agent = match serde_json::from_slice(&rbytes) {
+                                    Ok(a) => a,
+                                    Err(e) => {
+                                        warn!("Error deserializing agent from {id}: {e}");
+                                        continue;
+                                    }
+                                };
+                                agent.state = AgentState::Active;
+                                Self::add_agent(agent).await?;
+                            }
+                            None => warn!("Empty agent data"),
+                        },
+                        JobType::Shell => (),
+                        JobType::Terminate => todo!(),
+                        JobType::Update => todo!(),
                     }
+                    UDB::lock_db().update_result(&result)?;
                 }
-                ExecResult::Agent(mut a) => {
+                Reportable::Error(e) => {
-                    a.state = AgentState::Active;
+                    warn!("{} reported an error: {}", id, e);
-                    Self::add_agent(a).await?;
                 }
-                ExecResult::Dummy => (),
+                Reportable::Dummy => (),
             }
         }
-        if failed.len() > 0 {
+        Ok(())
-            let err_msg = ULocalError::ProcessingError(failed.join(", "));
+    }
-            return Ok(build_err(err_msg));
+
-        }
+    pub async fn update_agent(agent: BaseMessage<'static, Agent>) -> EndpResult<()> {
-        Ok(build_ok(""))
+        UDB::lock_db().update_agent(&agent.into_inner())?;
+        Ok(())
+    }
+
+    pub async fn update_job(job: BaseMessage<'static, JobMeta>) -> EndpResult<()> {
+        UDB::lock_db().update_job(&job.into_inner())?;
+        Ok(())
+    }
+
+    pub async fn update_assigned_job(
+        assigned: BaseMessage<'static, AssignedJob>,
+    ) -> EndpResult<()> {
+        UDB::lock_db().update_result(&assigned.into_inner())?;
+        Ok(())
+    }
+
+    pub async fn download(_file_uid: String) -> EndpResult<Vec<u8>> {
+        todo!()
     }
 }

bin/u_server/src/main.rs

@@ -1,6 +1,11 @@
 use u_server_lib::serve;
 
+#[macro_use]
+extern crate tracing;
+
 #[tokio::main]
 async fn main() {
-    serve().await;
+    if let Err(e) = serve().await {
+        error!("U_SERVER error: {}", e);
+    }
 }

bin/u_server/src/u_server.rs

@@ -1,117 +1,267 @@
 #[macro_use]
-extern crate log;
+extern crate tracing;
 
+#[cfg(test)]
 #[macro_use]
-extern crate mockall;
+extern crate rstest;
-#[macro_use]
-extern crate mockall_double;
 
-// because of linking errors
+// due to linking errors
 extern crate openssl;
-#[macro_use]
+// don't touch anything
 extern crate diesel;
-//
+// in this block
+
 mod db;
-mod filters;
+mod error;
 mod handlers;
 
-use db::UDB;
+use error::{Error as ServerError, RejResponse};
-use filters::make_filters;
+use serde::{de::DeserializeOwned, Deserialize};
-use u_lib::{config::MASTER_PORT, models::*, utils::init_env};
+use std::{convert::Infallible, path::PathBuf};
-use warp::Filter;
+use u_lib::{
+    config::MASTER_PORT,
+    logging::init_logger,
+    messaging::{AsMsg, BaseMessage, Reportable},
+    models::*,
+    utils::load_env,
+};
+use uuid::Uuid;
+use warp::{
+    body,
+    log::{custom, Info},
+    reply::{json, reply, Json, Response},
+    Filter, Rejection, Reply,
+};
 
-const LOGFILE: &str = "u_server.log";
+use crate::db::UDB;
+use crate::handlers::Endpoints;
 
-fn prefill_jobs() {
+#[derive(Deserialize)]
-    let agent_hello = JobMeta::builder()
+struct ServEnv {
-        .with_type(misc::JobType::Manage)
+    admin_auth_token: String,
-        .with_alias("agent_hello")
-        .build()
-        .unwrap();
-    UDB::lock_db().insert_jobs(&[agent_hello]).ok();
 }
 
-fn init_logger() {
+fn get_content<M>() -> impl Filter<Extract = (BaseMessage<'static, M>,), Error = Rejection> + Clone
-    use simplelog::*;
+where
-    use std::fs::OpenOptions;
+    M: AsMsg + Sync + Send + DeserializeOwned + 'static,
-    let log_cfg = ConfigBuilder::new()
+{
-        .set_time_format_str("%x %X")
+    body::content_length_limit(1024 * 64).and(body::json::<BaseMessage<M>>())
-        .set_time_to_local(true)
+}
-        .build();
+
-    let logfile = OpenOptions::new()
+fn into_message<M: AsMsg>(msg: M) -> Json {
-        .append(true)
+    json(&msg.as_message())
-        .create(true)
+}
-        .open(LOGFILE)
+
-        .unwrap();
+pub fn init_endpoints(
-    let level = LevelFilter::Info;
+    auth_token: &str,
-    let loggers = vec![
+) -> impl Filter<Extract = (impl Reply,), Error = Rejection> + Clone {
-        WriteLogger::new(level, log_cfg.clone(), logfile) as Box<dyn SharedLogger>,
+    let path = |p: &'static str| warp::post().and(warp::path(p));
-        TermLogger::new(level, log_cfg, TerminalMode::Stderr, ColorChoice::Auto),
+    let infallible_none = |_| async { Ok::<_, Infallible>((None::<Uuid>,)) };
-    ];
+
-    CombinedLogger::init(loggers).unwrap();
+    let get_agents = path("get_agents")
+        .and(
+            warp::path::param::<Uuid>()
+                .map(Some)
+                .or_else(infallible_none),
+        )
+        .and_then(Endpoints::get_agents)
+        .map(into_message);
+
+    let upload_jobs = path("upload_jobs")
+        .and(get_content::<Vec<JobMeta>>())
+        .and_then(Endpoints::upload_jobs)
+        .map(into_message);
+
+    let get_jobs = path("get_jobs")
+        .and(
+            warp::path::param::<Uuid>()
+                .map(Some)
+                .or_else(infallible_none),
+        )
+        .and_then(Endpoints::get_jobs)
+        .map(into_message);
+
+    let get_agent_jobs = path("get_agent_jobs")
+        .and(
+            warp::path::param::<Uuid>()
+                .map(Some)
+                .or_else(infallible_none),
+        )
+        .and_then(Endpoints::get_agent_jobs)
+        .map(into_message);
+
+    let get_personal_jobs = path("get_personal_jobs")
+        .and(warp::path::param::<Uuid>())
+        .and_then(Endpoints::get_personal_jobs)
+        .map(into_message);
+
+    let del = path("del")
+        .and(warp::path::param::<Uuid>())
+        .and_then(Endpoints::del)
+        .map(ok);
+
+    let set_jobs = path("set_jobs")
+        .and(warp::path::param::<Uuid>())
+        .and(get_content::<Vec<String>>())
+        .and_then(Endpoints::set_jobs)
+        .map(into_message);
+
+    let report = path("report")
+        .and(get_content::<Vec<Reportable>>())
+        .and_then(Endpoints::report)
+        .map(ok);
+
+    let update_agent = path("update_agent")
+        .and(get_content::<Agent>())
+        .and_then(Endpoints::update_agent)
+        .map(ok);
+
+    let update_job = path("update_job")
+        .and(get_content::<JobMeta>())
+        .and_then(Endpoints::update_job)
+        .map(ok);
+
+    let update_assigned_job = path("update_result")
+        .and(get_content::<AssignedJob>())
+        .and_then(Endpoints::update_assigned_job)
+        .map(ok);
+
+    let download = path("download")
+        .and(warp::path::param::<String>())
+        .and_then(Endpoints::download)
+        .map(ok);
+
+    let ping = path("ping").map(reply);
+
+    let auth_token = format!("Bearer {auth_token}",).into_boxed_str();
+    let auth_header = warp::header::exact("authorization", Box::leak(auth_token));
+
+    let auth_zone = (get_agents
+        .or(get_jobs)
+        .or(upload_jobs)
+        .or(del)
+        .or(set_jobs)
+        .or(get_agent_jobs)
+        .or(update_agent.or(update_job).or(update_assigned_job))
+        .or(download)
+        .or(ping))
+    .and(auth_header);
+
+    let agent_zone = get_jobs.or(get_personal_jobs).or(report).or(download);
+
+    auth_zone.or(agent_zone)
 }
 
-fn init_all() {
+pub fn preload_jobs() -> Result<(), ServerError> {
-    init_logger();
+    let job_alias = "agent_hello";
-    init_env();
+    let if_job_exists = UDB::lock_db().find_job_by_alias(job_alias)?;
-    prefill_jobs();
+    if if_job_exists.is_none() {
+        let agent_hello = JobMeta::builder()
+            .with_type(JobType::Init)
+            .with_alias(job_alias)
+            .build()
+            .unwrap();
+        UDB::lock_db().insert_jobs(&[agent_hello])?;
+    }
+    Ok(())
 }
 
-pub async fn serve() {
+pub async fn serve() -> Result<(), ServerError> {
-    init_all();
+    init_logger(Some("u_server"));
-    let routes = make_filters();
+    preload_jobs()?;
-    warp::serve(routes.with(warp::log("warp")))
+
+    let certs_dir = PathBuf::from("certs");
+    let env = load_env::<ServEnv>().map_err(|e| ServerError::Other(e.to_string()))?;
+    let routes = init_endpoints(&env.admin_auth_token)
+        .recover(handle_rejection)
+        .with(custom(logger));
+
+    warp::serve(routes)
         .tls()
-        .cert_path("./certs/server.crt")
+        .cert_path(certs_dir.join("server.crt"))
-        .key_path("./certs/server.key")
+        .key_path(certs_dir.join("server.key"))
-        .client_auth_required_path("./certs/ca.crt")
+        .client_auth_required_path(certs_dir.join("ca.crt"))
         .run(([0, 0, 0, 0], MASTER_PORT))
         .await;
+    Ok(())
 }
 
+async fn handle_rejection(rej: Rejection) -> Result<Response, Infallible> {
+    let resp = if let Some(err) = rej.find::<ServerError>() {
+        error!("{:x?}", err);
+        RejResponse::bad_request(err.to_string())
+    } else if rej.is_not_found() {
+        RejResponse::not_found("not found placeholder")
+    } else {
+        error!("{:?}", rej);
+        RejResponse::internal()
+    };
+    Ok(resp.into_response())
+}
+
+fn logger(info: Info<'_>) {
+    info!(target: "warp",
+        "{raddr} {agent_uid} \"{path}\" {status}",
+        raddr = info.remote_addr().unwrap_or(([0, 0, 0, 0], 0).into()),
+        path = info.path(),
+        agent_uid = info.user_agent()
+            .map(|uid: &str| uid.splitn(3, '-')
+                .take(2)
+                .collect::<String>()
+            )
+            .unwrap_or_else(|| "NO_AGENT".to_string()),
+        status = info.status()
+    );
+}
+
+fn ok<T>(_: T) -> impl Reply {
+    reply()
+}
+
+/*
 #[cfg(test)]
 mod tests {
     use super::*;
-    #[double]
     use crate::handlers::Endpoints;
     use handlers::build_ok;
-    use mockall::predicate::*;
+    use u_lib::messaging::{AsMsg, BaseMessage, Reportable};
-    use test_case::test_case;
-    use u_lib::messaging::{AsMsg, BaseMessage};
     use uuid::Uuid;
-    use warp::test::request;
+    use warp::test;
 
-    #[test_case(Some(Uuid::new_v4()))]
+    #[rstest]
-    #[test_case(None => panics)]
+    #[case(Some(Uuid::new_v4()))]
+    #[should_panic]
+    #[case(None)]
     #[tokio::test]
-    async fn test_get_agent_jobs_unauthorized(uid: Option<Uuid>) {
+    async fn test_get_agent_jobs_unauthorized(#[case] uid: Option<Uuid>) {
-        let mock = Endpoints::get_agent_jobs_context();
+        let mock = Endpoints::faux();
-        mock.expect().with(eq(uid)).returning(|_| Ok(build_ok("")));
+        when!(mock.get_agent_jobs).then_return(Ok(build_ok("")));
-        request()
+        //mock.expect().with(eq(uid)).returning(|_| Ok(build_ok("")));
+        test::request()
             .path(&format!(
                 "/get_agent_jobs/{}",
                 uid.map(|u| u.simple().to_string()).unwrap_or(String::new())
             ))
             .method("GET")
-            .filter(&make_filters())
+            .filter(&init_filters(""))
             .await
             .unwrap();
-        mock.checkpoint();
     }
 
     #[tokio::test]
     async fn test_report_unauth_successful() {
-        let mock = Endpoints::report_context();
+        let mock = Endpoints::report();
         mock.expect()
-            .withf(|msg: &BaseMessage<'_, Vec<ExecResult>>| msg.inner_ref()[0] == ExecResult::Dummy)
+            .withf(|msg: &BaseMessage<'_, Vec<Reportable>>| msg.inner_ref()[0] == Reportable::Dummy)
             .returning(|_| Ok(build_ok("")));
-        request()
+        test::request()
             .path("/report/")
             .method("POST")
-            .json(&vec![ExecResult::Dummy].as_message())
+            .json(&vec![Reportable::Dummy].as_message())
-            .filter(&make_filters())
+            .filter(&init_filters(""))
             .await
             .unwrap();
         mock.checkpoint();
     }
 }
+*/

images/integration-tests/tests_runner.Dockerfile

@@ -0,0 +1,5 @@
+FROM rust:1.64
+
+RUN rustup target add x86_64-unknown-linux-musl
+RUN mkdir -p /tests && chmod 777 /tests
+CMD ["sleep", "3600"]

images/integration-tests/u_db.Dockerfile

@@ -0,0 +1,17 @@
+FROM postgres:14.5
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update && apt upgrade -y
+RUN apt install -y curl build-essential libpq-dev iproute2
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal 
+ENV PATH /root/.cargo/bin:$PATH
+RUN rustup target add x86_64-unknown-linux-musl
+RUN cargo install diesel_cli --no-default-features --features postgres
+
+RUN mkdir -p /unki
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US.UTF-8
+RUN apt install -y locales locales-all
+COPY u_db_entrypoint.sh /unki/

images/integration-tests/u_db_entrypoint.sh

@@ -0,0 +1,7 @@
+set -m 
+
+export DATABASE_URL=postgres://${DB_USER}:${DB_PASSWORD}@127.0.0.1/${DB_NAME}
+touch /unki/Cargo.toml
+/usr/local/bin/docker-entrypoint.sh postgres & 
+sleep 10 && diesel setup && diesel migration run
+[[ $1 == "svc" ]] && fg %1

images/integration-tests/u_server.Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine:latest
+
+RUN apk add iproute2 bash

images/musl-libs.Dockerfile

@@ -0,0 +1,96 @@
+FROM ubuntu:xenial
+LABEL maintainer="Eirik Albrigtsen <sszynrae@gmail.com>"
+
+# Required packages:
+# - musl-dev, musl-tools - the musl toolchain
+# - curl, g++, make, pkgconf, cmake - for fetching and building third party libs
+# - ca-certificates - openssl + curl + peer verification of downloads
+# - xutils-dev - for openssl makedepend
+# - libssl-dev and libpq-dev - for dynamic linking during diesel_codegen build process
+# - git - cargo builds in user projects
+# - linux-headers-amd64 - needed for building openssl 1.1 (stretch only)
+# - file - needed by rustup.sh install
+# - automake autoconf libtool - support crates building C deps as part cargo build
+# recently removed:
+# cmake (not used), nano, zlib1g-dev
+RUN apt-get update && apt-get install -y \
+  musl-dev \
+  musl-tools \
+  git \
+  file \
+  openssh-client \
+  make \
+  g++ \
+  curl \
+  pkgconf \
+  ca-certificates \
+  xutils-dev \
+  libssl-dev \
+  libpq-dev \
+  automake \
+  autoconf \
+  libtool \
+  python3 \
+  --no-install-recommends && \
+  rm -rf /var/lib/apt/lists/*
+
+# Convenience list of versions and variables for compilation later on
+# This helps continuing manually if anything breaks.
+ENV SSL_VER="1.0.2u" \
+    CURL_VER="7.77.0" \
+    ZLIB_VER="1.2.13" \
+    PQ_VER="11.12" \
+    SQLITE_VER="3350500" \
+    CC=musl-gcc \
+    PREFIX=/musl \
+    PATH=/usr/local/bin:/root/.cargo/bin:$PATH \
+    PKG_CONFIG_PATH=/usr/local/lib/pkgconfig \
+    LD_LIBRARY_PATH=$PREFIX
+
+# Set up a prefix for musl build libraries, make the linker's job of finding them easier
+# Primarily for the benefit of postgres.
+# Lastly, link some linux-headers for openssl 1.1 (not used herein)
+RUN mkdir $PREFIX && \
+    echo "$PREFIX/lib" >> /etc/ld-musl-x86_64.path && \
+    ln -s /usr/include/x86_64-linux-gnu/asm /usr/include/x86_64-linux-musl/asm && \
+    ln -s /usr/include/asm-generic /usr/include/x86_64-linux-musl/asm-generic && \
+    ln -s /usr/include/linux /usr/include/x86_64-linux-musl/linux
+
+# Build zlib (used in openssl and pq)
+RUN curl -sSL https://zlib.net/zlib-$ZLIB_VER.tar.gz | tar xz && \
+    cd zlib-$ZLIB_VER && \
+    CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure --static --prefix=$PREFIX && \
+    make -j$(nproc) && make install && \
+    cd .. && rm -rf zlib-$ZLIB_VER
+
+# Build openssl (used in curl and pq)
+# Would like to use zlib here, but can't seem to get it to work properly
+# TODO: fix so that it works
+RUN curl -sSL https://www.openssl.org/source/old/1.0.2/openssl-$SSL_VER.tar.gz | tar xz && \
+    cd openssl-$SSL_VER && \
+    ./Configure no-zlib no-shared -fPIC --prefix=$PREFIX --openssldir=$PREFIX/ssl linux-x86_64 && \
+    env C_INCLUDE_PATH=$PREFIX/include make depend 2> /dev/null && \
+    make -j$(nproc) && make install && \
+    cd .. && rm -rf openssl-$SSL_VER
+
+# Build curl (needs with-zlib and all this stuff to allow https)
+# curl_LDFLAGS needed on stretch to avoid fPIC errors - though not sure from what
+RUN curl -sSL https://curl.se/download/curl-$CURL_VER.tar.gz | tar xz && \
+    cd curl-$CURL_VER && \
+    CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \
+      --enable-shared=no --with-zlib --enable-static=ssl --enable-optimize --prefix=$PREFIX \
+      --with-ca-path=/etc/ssl/certs/ --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt --without-ca-fallback \
+      --with-openssl && \
+    make -j$(nproc) curl_LDFLAGS="-all-static" && make install && \
+    cd .. && rm -rf curl-$CURL_VER
+
+# Build libpq
+RUN curl -sSL https://ftp.postgresql.org/pub/source/v$PQ_VER/postgresql-$PQ_VER.tar.gz | tar xz && \
+    cd postgresql-$PQ_VER && \
+    CC="musl-gcc -fPIE -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \
+    --without-readline \
+    --with-openssl \
+    --prefix=$PREFIX --host=x86_64-unknown-linux-musl && \
+    cd src/interfaces/libpq make -s -j$(nproc) all-static-lib && make -s install install-lib-static && \
+    cd ../../bin/pg_config && make -j $(nproc) && make install && \
+    cd .. && rm -rf postgresql-$PQ_VER

integration/Cargo.toml

@@ -2,26 +2,23 @@
 name = "integration"
 version = "0.1.0"
 authors = ["plazmoid <kronos44@mail.ru>"]
-edition = "2018"
+edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process", "time"] }
+once_cell = "1.10.0"
-log = "^0.4"
+reqwest = { workspace = true }
-env_logger = "0.8.3"
+rstest = "0.12"
-uuid = { version = "0.6.5", features = ["serde", "v4"] }
+serde = { workspace = true }
-reqwest = { version = "0.11", features = ["json"] }
+serde_json = { workspace = true }
-serde_json = "1.0"
-serde = { version = "1.0.114", features = ["derive"] }
-futures = "0.3.5"
 shlex = "1.0.0"
-
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "time"] }
-[dependencies.u_lib]
+tracing = { workspace = true }
-path = "../lib/u_lib"
+uuid = { workspace = true, features = ["serde", "v4"] }
-version = "*"
+u_lib = { path = "../lib/u_lib", features = ["panel"] }
 
 
 [[test]]
 name = "integration"
-path = "tests/tests.rs"
+path = "tests/lib.rs"

integration/docker-compose.yml

@@ -1,4 +1,8 @@
-version: "2.1"
+version: "3.4"
+
+x-global:
+  user: &user
+    "${DOCKER_UID:-1000}:${DOCKER_GID:-1000}"
 
 networks:
   u_net:
@@ -6,28 +10,28 @@ networks:
 services:
 
   u_server:
+    user: *user
     image: unki/u_server
     networks:
       - u_net
     volumes:
-      - ../target/x86_64-unknown-linux-musl/release/u_server:/u_server
+      - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_server:/unki/u_server
-      - ../:/unki/
+      - ../certs:/unki/certs
+      - ../logs:/unki/logs:rw
     working_dir: /unki
-    command: bash -c "
+    command: /unki/u_server
-        export DATABASE_URL=postgres://$${DB_USER}:$${DB_PASSWORD}@$${DB_HOST}/$${DB_NAME} &&
-        diesel setup && diesel migration run && /u_server"
     depends_on:
       u_db:
         condition: service_healthy
-    expose:
+    ports:
-      - '63714'
+      - 63714:63714
     env_file:
       - ../.env
       - ../.env.private
     environment:
-      RUST_LOG: trace
+      RUST_LOG: warp=info,u_server_lib=debug
     healthcheck:
-        test: /bin/ss -tlpn | grep 63714
+        test: ss -tlpn | grep 63714
         interval: 5s
         timeout: 2s
         retries: 2
@@ -36,64 +40,57 @@ services:
     image: unki/u_db
     networks:
       - u_net
-    expose:
+    ports:
-      - '5432'
+      - 54321:5432
     env_file:
       - ../.env
       - ../.env.private
+    working_dir: /unki
+    volumes:
+      - ../migrations:/unki/migrations
+    command: /unki/u_db_entrypoint.sh svc
     healthcheck:
-        test: /bin/ss -tlpn | grep 5432
+        # test if db's port is open and db is created
+        test: ss -tlpn | grep 5432 && psql -lqt -U $${DB_USER} | grep -qw $${DB_NAME}
         interval: 5s
-        timeout: 2s
+        timeout: 5s
-        retries: 2
+        retries: 3
-
-  u_agent_1:
-    image: unki/u_agent
-    networks:
-      - u_net
-    volumes:
-      - ../target/x86_64-unknown-linux-musl/release/u_agent:/u_agent
-    command: /u_agent u_server
-    env_file:
-      - ../.env
-    environment:
-      RUST_LOG: u_agent=debug
-    depends_on:
-      u_server:
-        condition: service_healthy
 
-  u_agent_2:
+  u_agent:
+    user: *user
     image: unki/u_agent
     networks:
       - u_net
     volumes:
-      - ../target/x86_64-unknown-linux-musl/release/u_agent:/u_agent
+      - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_agent:/unki/u_agent
-    command: /u_agent u_server
+      - ../logs:/unki/logs:rw
+    working_dir: /unki
+    command: /unki/u_agent u_server
     env_file:
       - ../.env
     environment:
-      RUST_LOG: u_agent=debug
+      RUST_LOG: u_agent=debug,u_lib=debug
     depends_on:
       u_server:
         condition: service_healthy
 
   tests_runner:
+    user: *user
     image: unki/tests_runner
     networks:
       - u_net
     volumes:
-      - ~/.cargo/registry:/root/.cargo/registry
+      - ${HOME}/.cargo/registry/:/usr/local/cargo/registry/
-      - ./:/tests/
+      - ../__Cargo_integration.toml:/tests/Cargo.toml
-      - ../certs:/certs
+      - ./:/tests/integration/
-      - ../target/x86_64-unknown-linux-musl/release/u_panel:/u_panel
+      - ../certs:/tests/certs
-      - ../lib/u_lib:/lib/u_lib
+      - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_panel:/u_panel
-      - ../lib/u_api_proc_macro:/lib/u_api_proc_macro
+      - ../lib/u_lib:/tests/lib/u_lib
+      - ../logs:/tests/integration/logs:rw
     working_dir:
-      /tests/
+      /tests/integration/
     depends_on:
-      u_agent_1:
+      u_agent:
-        condition: service_started
-      u_agent_2:
         condition: service_started
       u_server:
         condition: service_healthy

integration/docker.py

@@ -1,26 +1,28 @@
 import subprocess
+import shlex
+
 from utils import *
 
-BASE_IMAGE_DIR = 'images'
+BASE_IMAGE_DIR = '../images/integration-tests'
 
-DOCKERFILES = {
+DOCKERFILES = [
-    'u_agent': {
+    {
+        'name': 'u_agent',
         'ctx': BASE_IMAGE_DIR,
-        'dockerfile_prefix': 'u_agent'
     },
-    'u_server': {
+    {
+        'name': 'u_server',
         'ctx': BASE_IMAGE_DIR,
-        'dockerfile_prefix': 'u_server'
     },
-    'u_db': {
+    {
+        'name': 'u_db',
         'ctx': BASE_IMAGE_DIR,
-        'dockerfile_prefix': 'u_db'
     },
-    'tests_runner': {
+    {
+        'name': 'tests_runner',
         'ctx': BASE_IMAGE_DIR,
-        'dockerfile_prefix': 'tests_runner'
     },
-}
+]
 
 
 def docker(args):
@@ -62,20 +64,96 @@ def check_state(containers):
 
 
 def rebuild_images_if_needed(force_rebuild=False):
-    for img_name, data in DOCKERFILES.items():
+    for img in DOCKERFILES:
-        ctx = data['ctx']
+        ctx = img['ctx']
-        df_prefix = data.get('dockerfile_prefix')
+        name = img.get('name')
         df_suffix = 'Dockerfile'
-        img_name = f'unki/{img_name}'
+        img_name = f'unki/{name}'
         log(f'Building docker image {img_name}')
         cmd = [
             'build',
-            '-t',
+            '-t', img_name,
-            img_name,
+            '-f', f'{BASE_IMAGE_DIR}/{name}.{df_suffix}',
             ctx,
         ]
-        if df_prefix:
-            cmd += ['-f', f'{ctx}/{df_prefix}.{df_suffix}']
         if force_rebuild:
             cmd += ['--no-cache']
         docker(cmd)
+
+
+class Compose:
+    ALL_IMAGES = [
+        'u_agent',
+        'u_server',
+        'u_db',
+        'tests_runner',
+    ]
+
+    def __init__(self):
+        self.container_tpl = 'integration-%s-%d'
+        self.cmd_container = self.container_tpl % ('tests_runner', 1)
+        self.ALL_CONTAINERS = [self.container_tpl %
+                               (c, 1) for c in self.ALL_IMAGES]
+        self.scaled_svc = {}
+        self.scale("u_agent", 2)
+
+    def scale(self, svc, count):
+        for c in range(1, count):
+            new_container = self.container_tpl % (svc, c + 1)
+            self.ALL_CONTAINERS.append(new_container)
+        self.scaled_svc[svc] = count
+
+    def _call(self, *args):
+        cmd = [
+            'docker-compose',
+            '--ansi=never',
+        ] + list(args)
+        log(f'Running docker-compose command: {cmd}')
+        subprocess.check_call(cmd)
+
+    def up(self):
+        log(f'Instanciating cluster: {self.ALL_CONTAINERS}')
+        scaled = [f"{k}={v}" for k, v in self.scaled_svc.items()]
+        if len(scaled) > 0:
+            scaled.insert(0, '--scale')
+        self._call('up', '-d', *scaled)
+
+    def down(self):
+        log('Shutting down cluster')
+        self._call('down')
+
+    def stop(self):
+        log('Stopping cluster')
+        self._call('stop')
+
+    def run(self, cmd):
+        container = self.cmd_container
+        if isinstance(cmd, str):
+            cmd = shlex.split(cmd)
+        result = docker([
+            'exec',
+            '-ti',
+            container
+        ] + cmd)
+        return result
+
+    def is_alive(self):
+        log('Check if all containers are alive')
+
+        errors = check_state(self.ALL_CONTAINERS)
+
+        if errors:
+            print_errors(errors)
+            raise TestsError('Error during `is_alive` check')
+        else:
+            log('All containers are alive')
+
+    def print_containers_logs(self):
+        for container in self.ALL_CONTAINERS:
+            try:
+                docker([
+                    'logs',
+                    container
+                ])
+            except Exception:
+                pass

integration/docker_compose.py

@@ -1,66 +0,0 @@
-import subprocess
-import shlex
-from utils import *
-from docker import docker, check_state, print_errors
-
-
-class Compose:
-    ALL_CONTAINERS = [
-        'u_agent_1',
-        'u_agent_2',
-        'u_server',
-        'u_db',
-        'tests_runner',
-    ]
-
-    def __init__(self):
-        self.container_tpl = 'integration_%s_1'
-        self.cmd_container = self.container_tpl % 'tests_runner'
-        self.ALL_CONTAINERS = [self.container_tpl % c for c in self.ALL_CONTAINERS]
-
-    def _call(self, *args):
-        subprocess.check_call([
-            'docker-compose',
-            '--no-ansi',
-        ] + list(args)
-        )
-
-    def up(self):
-        log('Instanciating cluster')
-        self._call('up', '-d')
-        log('Ok')
-
-    def down(self):
-        log('Shutting down cluster')
-        self._call('down')
-        log('Ok')
-
-    def stop(self):
-        log('Stopping cluster')
-        self._call('stop')
-        log('Ok')
-
-    def run(self, cmd):
-        container = self.cmd_container
-        if isinstance(cmd, str):
-            cmd = shlex.split(cmd)
-        log(f'Running command "{cmd}" in container {container}')
-        result = docker([
-            'exec',
-            '-ti',
-            container
-        ] + cmd)
-        log('Ok')
-        return result
-
-    def is_alive(self):
-        log('Check if all containers are alive')
-
-        errors = check_state(self.ALL_CONTAINERS)
-        log('Check done')
-
-        if errors:
-            print_errors(errors)
-            raise TestsError('Error during `is_alive` check')
-        else:
-            log('All containers are alive')

integration/images/tests_runner.Dockerfile

@@ -1,4 +0,0 @@
-FROM rust:1.53
-
-RUN rustup target add x86_64-unknown-linux-musl
-CMD ["sleep", "3600"]

integration/images/u_db.Dockerfile

@@ -1,3 +0,0 @@
-FROM postgres:13.3
-
-RUN apt update && apt -y upgrade && apt install -y iproute2

integration/images/u_server.Dockerfile

@@ -1,3 +0,0 @@
-FROM rust:1.53
-
-RUN cargo install diesel_cli --no-default-features --features postgres

integration/integration_tests.py

@@ -1,35 +1,73 @@
 import signal
 import sys
+import toml
+
+from docker import rebuild_images_if_needed, Compose
+from pathlib import Path
 from utils import *
-from docker import rebuild_images_if_needed
-from docker_compose import Compose
 
+CARGO_INTEGRATION_TOML = Path('../__Cargo_integration.toml')
+CLUSTER = Compose()
+
+
+def fail(msg):
+    err(msg)
+    sys.exit(1)
+
+
+def usage_exit():
+    usage = f"""Usage:
+    python {__file__.split('/')[-1]} [--rebuild] [--preserve] [--no-run] [--down]"""
 
-cluster = Compose()
+    fail(usage)
 
 
-def abort_handler(s, _):
+def create_integration_workspace():
-    warn(f'Received signal: {s}')
+    if CARGO_INTEGRATION_TOML.exists():
-    warn(f'Gracefully stopping...')
+        CARGO_INTEGRATION_TOML.unlink()
-    cluster.down()
+    workspace = toml.load('../Cargo.toml')
+    workspace['workspace']['members'] = ['integration']
+    with open(CARGO_INTEGRATION_TOML, 'w') as fo:
+        toml.dump(workspace, fo)
 
 
 def run_tests():
-    force_rebuild = '--rebuild' in sys.argv
+    allowed_args = set(["--rebuild", "--preserve", "--no-run", "--release", "--down"])
-    preserve_containers = '--preserve' in sys.argv
+    args = sys.argv[1:]
+    if not set(args).issubset(allowed_args):
+        usage_exit()
+    force_rebuild = '--rebuild' in args
+    preserve_containers = '--preserve' in args
+    only_setup_cluster = '--no-run' in args
+    down_cluster = "--down" in args
+
+    def _cleanup():
+        if not preserve_containers and not only_setup_cluster:
+            CLUSTER.down()
+        CARGO_INTEGRATION_TOML.unlink(missing_ok=True)
+
+    def abort_handler(s, _):
+        warn(f'Received signal: {s}, gracefully stopping...')
+        _cleanup()
+
+    if down_cluster:
+        _cleanup()
+        return
+        
     for s in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP):
         signal.signal(s, abort_handler)
     rebuild_images_if_needed(force_rebuild)
+    create_integration_workspace()
     try:
-        cluster.up()
+        CLUSTER.up()
-        cluster.is_alive()
+        CLUSTER.is_alive()
-        cluster.run('cargo test --test integration')
+        if not only_setup_cluster:
+            CLUSTER.run('cargo test --test integration')
     except Exception as e:
-        err(e)
+        CLUSTER.print_containers_logs()
-        sys.exit(1)
+        fail(e)
     finally:
-        if not preserve_containers:
+        _cleanup()
-            cluster.down()
 
 
 if __name__ == '__main__':

integration/integration_tests.sh

@@ -1,3 +1,6 @@
 #!/bin/bash
 set -e
+export DOCKER_UID=$(id -u)
+export DOCKER_GID=$(id -g)
+[[ "$@" =~ "--release" ]] && export PROFILE=release || export PROFILE=debug
 python integration_tests.py $@

integration/tests/behaviour.rs

@@ -1 +0,0 @@
-

integration/tests/fixtures/agent.rs

@@ -0,0 +1,35 @@
+use crate::helpers::ENV;
+use u_lib::{api::ClientHandler, messaging::Reportable, models::*};
+use uuid::Uuid;
+
+pub struct RegisteredAgent {
+    pub uid: Uuid,
+}
+
+impl RegisteredAgent {
+    pub async fn unregister(self) {
+        let cli = ClientHandler::new(&ENV.u_server, None);
+        cli.del(self.uid).await.unwrap();
+    }
+}
+
+#[fixture]
+pub async fn register_agent() -> RegisteredAgent {
+    let cli = ClientHandler::new(&ENV.u_server, None);
+    let agent_uid = Uuid::new_v4();
+    println!("registering agent {agent_uid}");
+    let resp = cli
+        .get_personal_jobs(agent_uid)
+        .await
+        .unwrap()
+        .pop()
+        .unwrap();
+    let job_id = resp.job_id;
+    let job = cli.get_jobs(Some(job_id)).await.unwrap().pop().unwrap();
+    assert_eq!(job.alias, Some("agent_hello".to_string()));
+    let mut agent_data = AssignedJob::from(&job);
+    agent_data.agent_id = agent_uid;
+    agent_data.set_result(&Agent::with_id(agent_uid));
+    cli.report(Reportable::Assigned(agent_data)).await.unwrap();
+    RegisteredAgent { uid: agent_uid }
+}

integration/tests/fixtures/mod.rs

@@ -0,0 +1 @@
+pub mod agent;

integration/tests/helpers/client.rs

@@ -1,48 +0,0 @@
-use reqwest::{Client, RequestBuilder, Url};
-use serde::Serialize;
-use serde_json::{from_str, json, Value};
-
-const SERVER: &str = "u_server";
-const PORT: &str = "63714";
-
-pub struct AgentClient {
-    client: Client,
-    base_url: Url,
-}
-
-impl AgentClient {
-    pub fn new() -> Self {
-        Self {
-            client: Client::new(),
-            base_url: Url::parse(&format!("http://{}:{}", SERVER, PORT)).unwrap(),
-        }
-    }
-
-    async fn process_request(&self, req: RequestBuilder, resp_needed: bool) -> Value {
-        let resp = req.send().await.unwrap();
-        if let Err(e) = resp.error_for_status_ref() {
-            panic!(
-                "Server responded with code {}\nError: {}",
-                e.status()
-                    .map(|s| s.to_string())
-                    .unwrap_or(String::from("<none>")),
-                e.to_string()
-            );
-        }
-        if !resp_needed {
-            return json!([]);
-        }
-        let resp: Value = from_str(&resp.text().await.unwrap()).unwrap();
-        resp.get("inner").unwrap().get(0).unwrap().clone()
-    }
-
-    pub async fn get<S: AsRef<str>>(&self, url: S) -> Value {
-        let req = self.client.get(self.base_url.join(url.as_ref()).unwrap());
-        self.process_request(req, true).await
-    }
-
-    pub async fn post<S: AsRef<str>, B: Serialize>(&self, url: S, body: &B) -> Value {
-        let req = self.client.post(self.base_url.join(url.as_ref()).unwrap());
-        self.process_request(req.json(body), false).await
-    }
-}

integration/tests/helpers/mod.rs

@@ -1,3 +1,8 @@
 pub mod panel;
 
 pub use panel::Panel;
+
+use once_cell::sync::Lazy;
+use u_lib::utils::{env::DefaultEnv, load_env_default};
+
+pub static ENV: Lazy<DefaultEnv> = Lazy::new(|| load_env_default().unwrap());

integration/tests/helpers/panel.rs

@@ -1,54 +1,67 @@
 use serde::de::DeserializeOwned;
-use serde_json::from_slice;
+use serde_json::{from_slice, Value};
-use shlex::split;
+use std::fmt::{Debug, Display};
 use std::process::{Command, Output};
-use u_lib::{datatypes::DataResult, messaging::AsMsg};
+use u_lib::{
+    datatypes::PanelResult,
+    utils::{bytes_to_string, ProcOutput},
+};
 
 const PANEL_BINARY: &str = "/u_panel";
 
-type PanelResult<T> = Result<DataResult<T>, String>;
-
 pub struct Panel;
 
 impl Panel {
     fn run(args: &[&str]) -> Output {
-        Command::new(PANEL_BINARY)
+        Command::new(PANEL_BINARY).args(args).output().unwrap()
-            .arg("--json")
-            .args(args)
-            .output()
-            .unwrap()
     }
 
-    pub fn output_argv<T: AsMsg + DeserializeOwned>(args: &[&str]) -> PanelResult<T> {
+    pub fn output_argv<T: DeserializeOwned>(argv: &[&str]) -> PanelResult<T> {
-        let result = Self::run(args);
+        let result = Self::run(argv);
-        assert!(result.status.success());
+        let output = ProcOutput::from_output(&result).into_vec();
-        from_slice(&result.stdout).map_err(|e| e.to_string())
+        from_slice(&output)
+            .map_err(|e| {
+                eprintln!(
+                    "Failed to decode panel response: '{}'",
+                    bytes_to_string(&output)
+                );
+                e.to_string()
+            })
+            .unwrap()
     }
 
-    pub fn output<T: AsMsg + DeserializeOwned>(args: impl Into<String>) -> PanelResult<T> {
+    pub fn output<T: DeserializeOwned + Debug>(
-        let splitted = split(args.into().as_ref()).unwrap();
+        args: impl Into<String> + Display,
-        Self::output_argv(
+    ) -> PanelResult<T> {
+        eprintln!(">>> {PANEL_BINARY} {}", &args);
+        let splitted = shlex::split(args.into().as_ref()).unwrap();
+        let result = Self::output_argv(
             splitted
                 .iter()
                 .map(|s| s.as_ref())
                 .collect::<Vec<&str>>()
                 .as_ref(),
-        )
+        );
+        match &result {
+            PanelResult::Ok(r) => eprintln!("<<<+ {r:02x?}"),
+            PanelResult::Err(e) => eprintln!("<<<! {e:02x?}"),
+        }
+        result
     }
 
-    fn status_is_ok<T: AsMsg + DeserializeOwned>(data: PanelResult<T>) -> T {
+    fn status_is_ok<T: DeserializeOwned + Debug>(data: PanelResult<T>) -> T {
-        match data.unwrap() {
+        match data {
-            DataResult::Ok(r) => r,
+            PanelResult::Ok(r) => r,
-            DataResult::Err(e) => panic!("Panel failed with erroneous status: {}", e),
+            PanelResult::Err(e) => panic!("Panel failed: {}", e),
         }
     }
 
-    pub fn check_status<'s, T: AsMsg + DeserializeOwned>(args: &'s str) {
+    pub fn check_status(args: impl Into<String> + Display) {
-        let result: PanelResult<T> = Self::output(args);
+        let result: PanelResult<Value> = Self::output(args);
         Self::status_is_ok(result);
     }
 
-    pub fn check_output<T: AsMsg + DeserializeOwned>(args: impl Into<String>) -> T {
+    pub fn check_output<T: DeserializeOwned + Debug>(args: impl Into<String> + Display) -> T {
         let result = Self::output(args);
         Self::status_is_ok(result)
     }

integration/tests/integration/behaviour.rs

@@ -0,0 +1,51 @@
+use crate::fixtures::agent::*;
+use crate::helpers::Panel;
+
+use rstest::rstest;
+use serde_json::{json, to_string};
+use std::error::Error;
+use std::time::Duration;
+use tokio::time::sleep;
+use u_lib::models::*;
+use uuid::Uuid;
+
+type TestResult<R = ()> = Result<R, Box<dyn Error>>;
+
+#[rstest]
+#[tokio::test]
+async fn test_registration(#[future] register_agent: RegisteredAgent) -> TestResult {
+    let agent = register_agent.await;
+    let agents: Vec<Agent> = Panel::check_output("agents read");
+    let found = agents.iter().find(|v| v.id == agent.uid);
+    assert!(found.is_some());
+    Panel::check_status(format!("agents delete {}", agent.uid));
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_setup_tasks() -> TestResult {
+    let agents: Vec<Agent> = Panel::check_output("agents read");
+    let agent_uid = match agents.get(0) {
+        Some(a) => a.id,
+        None => panic!("Some independent agents should present"),
+    };
+    let job_alias = "passwd_contents";
+    let job = json!(
+        {"alias": job_alias, "payload": b"cat /etc/passwd", "argv": "/bin/bash {}" }
+    );
+    let cmd = format!("jobs create '{}'", to_string(&job).unwrap());
+    Panel::check_status(cmd);
+    let cmd = format!("map create {} {}", agent_uid, job_alias);
+    let assigned_uids: Vec<Uuid> = Panel::check_output(cmd);
+    for _ in 0..3 {
+        let result: Vec<AssignedJob> =
+            Panel::check_output(format!("map read {}", assigned_uids[0]));
+        if result[0].state == JobState::Finished {
+            return Ok(());
+        } else {
+            sleep(Duration::from_secs(5)).await;
+            eprintln!("waiting for task");
+        }
+    }
+    panic!("Job didn't appear in the job map");
+}

integration/tests/integration/connection.rs

@@ -0,0 +1,22 @@
+use crate::helpers::ENV;
+use u_lib::config::MASTER_PORT;
+
+#[tokio::test]
+async fn test_non_auth_connection_dropped() {
+    let client = reqwest::ClientBuilder::new()
+        .danger_accept_invalid_certs(true)
+        .build()
+        .unwrap();
+    match client
+        .get(format!("https://{}:{}", &ENV.u_server, MASTER_PORT))
+        .send()
+        .await
+    {
+        Err(e) => {
+            let err = e.to_string();
+            println!("captured err: {err}");
+            assert!(err.contains("certificate required"));
+        }
+        _ => panic!("no error occured on foreign client connection"),
+    }
+}

integration/tests/integration/mod.rs

@@ -0,0 +1,2 @@
+mod behaviour;
+mod connection;

integration/tests/lib.rs

@@ -0,0 +1,6 @@
+mod fixtures;
+mod helpers;
+mod integration;
+
+#[macro_use]
+extern crate rstest;

integration/tests/tests.rs

@@ -1,67 +0,0 @@
-mod helpers;
-
-use helpers::Panel;
-
-use std::error::Error;
-use std::thread::sleep;
-use std::time::Duration;
-use u_lib::{api::ClientHandler, models::*};
-use uuid::Uuid;
-
-type TestResult<R = ()> = Result<R, Box<dyn Error>>;
-
-async fn register_agent() -> Uuid {
-    let cli = ClientHandler::new(None);
-    let agent_uid = Uuid::new_v4();
-    let resp = cli
-        .get_personal_jobs(Some(agent_uid))
-        .await
-        .unwrap()
-        .pop()
-        .unwrap();
-    let job_id = resp.job_id;
-    let resp = cli.get_jobs(Some(job_id)).await.unwrap().pop().unwrap();
-    assert_eq!(resp.alias, Some("agent_hello".to_string()));
-    let agent_data = Agent {
-        id: agent_uid,
-        ..Default::default()
-    };
-    cli.report(&vec![ExecResult::Agent(agent_data)])
-        .await
-        .unwrap();
-    agent_uid
-}
-
-#[tokio::test]
-async fn test_registration() -> TestResult {
-    let agent_uid = register_agent().await;
-    let agents: Vec<Agent> = Panel::check_output("agents list");
-    let found = agents.iter().find(|v| v.id == agent_uid);
-    assert!(found.is_some());
-    //teardown
-    Panel::check_status::<i32>(&format!("agents delete {}", agent_uid));
-    Ok(())
-}
-
-#[tokio::test]
-async fn test_setup_tasks() -> TestResult {
-    //some independent agents should present
-    let agents: Vec<Agent> = Panel::check_output("agents list");
-    let agent_uid = agents[0].id;
-    let job_alias = "passwd_contents";
-    let cmd = format!("jobs add --alias {} 'cat /etc/passwd'", job_alias);
-    Panel::check_status::<Empty>(&cmd);
-    let cmd = format!("jobmap add {} {}", agent_uid, job_alias);
-    let assigned_uids: Vec<Uuid> = Panel::check_output(cmd);
-    for _ in 0..3 {
-        let result: Vec<AssignedJob> =
-            Panel::check_output(format!("jobmap list {}", assigned_uids[0]));
-        if result[0].state == JobState::Finished {
-            return Ok(());
-        } else {
-            sleep(Duration::from_secs(5));
-            eprintln!("waiting for task");
-        }
-    }
-    panic!()
-}

lib/u_api_proc_macro/Cargo.toml

@@ -1,16 +0,0 @@
-[package]
-name = "u_api_proc_macro"
-version = "0.1.0"
-authors = ["plazmoid <kronos44@mail.ru>"]
-edition = "2018"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[lib]
-proc-macro = true
-
-[dependencies]
-syn = { version = "1.0", features = ["full", "extra-traits"] }
-quote = "1.0"
-strum = { version = "0.20", features = ["derive"] }
-proc-macro2 = "1.0"

lib/u_api_proc_macro/src/lib.rs

@@ -1,179 +0,0 @@
-use proc_macro::TokenStream;
-use proc_macro2::{Ident, TokenStream as TokenStream2};
-use quote::quote;
-use std::{collections::HashMap, str::FromStr};
-use strum::EnumString;
-use syn::{
-    parse_macro_input, punctuated::Punctuated, AttributeArgs, FnArg, ItemFn, Lit, NestedMeta,
-    ReturnType, Signature, Token, Type,
-};
-
-#[derive(EnumString, Debug)]
-enum ReqMethod {
-    GET,
-    POST,
-}
-
-#[derive(Debug)]
-struct Endpoint {
-    method: ReqMethod,
-}
-
-#[derive(Debug)]
-struct FnArgs {
-    url_param: Option<Type>,
-    payload: Option<Type>,
-}
-
-#[proc_macro_attribute]
-pub fn api_route(args: TokenStream, item: TokenStream) -> TokenStream {
-    let args: AttributeArgs = parse_macro_input!(args);
-    let input: ItemFn = parse_macro_input!(item);
-    let Signature {
-        ident,
-        inputs,
-        generics,
-        output,
-        ..
-    } = input.sig;
-    let (impl_generics, _, _) = generics.split_for_impl();
-    let FnArgs { url_param, payload } = parse_fn_args(inputs);
-    let Endpoint { method } = parse_attr_args(args);
-    let url_path = build_url_path(&ident, &url_param);
-    let return_ty = match output {
-        ReturnType::Type(_, ty) => quote!(#ty),
-        ReturnType::Default => quote!(()),
-    };
-    let request = match method {
-        ReqMethod::GET => build_get(url_path),
-        ReqMethod::POST => build_post(url_path, &payload),
-    };
-    let url_param = match url_param {
-        Some(p) => quote!(, param: #p),
-        None => TokenStream2::new(),
-    };
-    let payload = match payload {
-        Some(p) => quote!(, payload: #p),
-        None => TokenStream2::new(),
-    };
-    let q = quote! {
-        pub async fn #ident #impl_generics(
-            &self #url_param #payload
-        ) -> UResult<#return_ty> {
-            let request = {
-                #request
-            };
-            let response = request.send().await?;
-            let content_len = response.content_length();
-            let is_success = match response.error_for_status_ref() {
-                Ok(_) => Ok(()),
-                Err(e) => Err(UError::from(e))
-            };
-            match is_success {
-                Ok(_) => response.json::<BaseMessage<#return_ty>>()
-                    .await
-                    .map(|msg| msg.into_inner())
-                    .or_else(|e| {
-                        match content_len {
-                            Some(0) => Ok(Default::default()),
-                            _ => Err(UError::from(e))
-                        }
-                    }),
-                Err(UError::NetError(err_src, _)) => Err(
-                    UError::NetError(
-                        err_src,
-                        response.text().await.unwrap()
-                    )
-                ),
-                _ => unreachable!()
-            }
-        }
-    };
-    //eprintln!("#!#! RESULT:\n{}", q);
-    q.into()
-}
-
-fn parse_fn_args(raw: Punctuated<FnArg, Token![,]>) -> FnArgs {
-    let mut arg: HashMap<String, Type> = raw
-        .into_iter()
-        .filter_map(|arg| {
-            if let FnArg::Typed(argt) = arg {
-                let mut arg_name = String::new();
-                // did you think I won't overplay you? won't destroy?
-                |arg_ident| -> TokenStream {
-                    let q: TokenStream = quote!(#arg_ident).into();
-                    arg_name = parse_macro_input!(q as Ident).to_string();
-                    TokenStream::new()
-                }(argt.pat);
-                if &arg_name != "url_param" && &arg_name != "payload" {
-                    panic!("Wrong arg name: {}", &arg_name)
-                }
-                let arg_type = *argt.ty.clone();
-                Some((arg_name, arg_type))
-            } else {
-                None
-            }
-        })
-        .collect();
-    FnArgs {
-        url_param: arg.remove("url_param"),
-        payload: arg.remove("payload"),
-    }
-}
-
-fn build_get(url: TokenStream2) -> TokenStream2 {
-    quote! {
-        let request = self.build_get(#url);
-        request
-    }
-}
-
-fn build_post(url: TokenStream2, payload: &Option<Type>) -> TokenStream2 {
-    let pld = match payload {
-        Some(_) => quote! {
-            .json(&payload.as_message())
-        },
-        None => TokenStream2::new(),
-    };
-    quote! {
-        let request = self.build_post(#url);
-        request #pld
-    }
-}
-
-fn build_url_path(path: &Ident, url_param: &Option<Type>) -> TokenStream2 {
-    let url_param = match url_param {
-        Some(_) => quote! {
-            + &opt_to_string(param)
-        },
-        None => TokenStream2::new(),
-    };
-    quote! {
-        &format!(
-            "{}/{}",
-            stringify!(#path),
-            String::new() #url_param
-        )
-    }
-}
-
-fn parse_attr_args(args: AttributeArgs) -> Endpoint {
-    let mut args = args.into_iter();
-    let method = match args.next() {
-        Some(method) => match method {
-            NestedMeta::Lit(l) => {
-                if let Lit::Str(s) = l {
-                    match ReqMethod::from_str(&s.value()) {
-                        Ok(v) => v,
-                        Err(_) => panic!("Unknown method"),
-                    }
-                } else {
-                    panic!("Method must be a str")
-                }
-            }
-            _ => panic!("Method must be on the first place"),
-        },
-        None => panic!("Method required"),
-    };
-    Endpoint { method }
-}