Merge pull request 'Scheduling & stats' (#12) from 10-scheduling into master

Reviewed-on: #12

.cargo/config.toml

@@ -0,0 +1,15 @@
+[build]
+rustflags = [
+    "-L/usr/lib/musl/lib",
+    "-L/home/ortem/src/rust/unki/static/lib",
+    "--remap-path-prefix=/home/ortem/src/rust/unki=src",
+    "--remap-path-prefix=/home/ortem/.cargo=cargo"
+]
+target = "x86_64-unknown-linux-musl"
+
+[env]
+STATIC_PREFIX = "static"
+PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true"
+PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = { value = "static/bin/pg_config", relative = true }
+OPENSSL_STATIC = "true"
+OPENSSL_DIR = { value = "static", relative = true }

.env

@@ -1,4 +1,6 @@
-DB_HOST=u_db
+POSTGRES_HOST=u_db
-DB_NAME=u_db
+POSTGRES_DATABASE=u_db
-DB_USER=postgres
+POSTGRES_USER=u_ser
+POSTGRES_PORT=5432
 RUST_BACKTRACE=1
+U_SERVER=u_server

.gitignore

@@ -1,10 +1,14 @@
 target/
-**/*.rs.bk
 .idea/
 data/
+certs/
+static/
+.vscode/
+release/
+**/node_modules/
+
+**/*.rs.bk
 **/*.pyc
-certs/*
 *.log
 echoer
 .env.private
-*.lock

Cargo.toml

@@ -1,16 +1,34 @@
 [workspace]
 members = [
+    "bin/migrator",
     "bin/u_agent",
     "bin/u_panel",
     "bin/u_run",
     "bin/u_server",
     "lib/u_lib",
-    "lib/u_api_proc_macro",
+    "integration-tests",
-    "integration"
 ]
+resolver = "2"
+
+[workspace.dependencies]
+anyhow = "=1.0.63"
+deadpool-diesel = "0.4.0"
+diesel = { version = "2", features = ["postgres", "uuid"] }
+mime_guess = "2.0"
+openssl = "0.10"
+reqwest = { version = "0.11", features = ["json"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+thiserror = "=1.0.31"
+tokio = { version = "1.11", features = ["macros"] }
+tracing = "0.1.35"
+tracing-appender = "0.2.0"
+tracing-subscriber = { version = "0.3.0", features = ["env-filter"]}
+uuid = "1.2.1"
 
 [profile.release]
 panic = "abort"
+strip = "symbols"
 
 [profile.dev]
 debug = true  # Добавляет флаг `-g` для компилятора;

Makefile

@@ -1,26 +0,0 @@
-.PHONY: _pre_build debug release run clean unit integration test
-
-CARGO=./scripts/cargo_musl.sh
-
-clean:
-	${CARGO} clean
-
-_pre_build:
-	docker build -t unki/musllibs ./muslrust
-
-debug: _pre_build
-	${CARGO} build
-
-release: _pre_build
-	${CARGO} build --release
-
-run: build
-	${CARGO} run
-
-unit:
-	${CARGO} test --lib
-
-integration:
-	cd ./integration && ./integration_tests.sh
-
-test: unit integration

Makefile.toml

@@ -0,0 +1,96 @@
+# i need to preserve --release in args, not to pass cargo make -p release
+# due to cargo failing to parse "" argument
+env_scripts = ['''
+#!@duckscript
+args = array ${1} ${2} ${3} ${4} ${5} ${6} ${7}
+set_env PROFILE_OVERRIDE debug
+
+for arg in ${args}
+    e = eq ${arg} "--release"
+    if ${e}
+        set_env PROFILE_OVERRIDE release
+    end
+end
+
+profile = get_env PROFILE_OVERRIDE
+echo PROFILE_OVERRIDE=${profile}
+''']
+
+[config]
+default_to_workspace = false
+
+[env]
+TARGET = "x86_64-unknown-linux-musl"
+CARGO = "cargo"
+
+[tasks.build_static_libs]
+script = "./scripts/build_musl_libs.sh"
+
+[tasks.build_frontend]
+script = '''
+cd ./bin/u_panel/src/gui/fe
+ng build
+'''
+
+[tasks.clean]
+command = "${CARGO}"
+args = ["clean"]
+
+[tasks.cargo_build]
+dependencies = ["build_static_libs", "build_frontend"]
+command = "${CARGO}"
+args = ["build", "--target", "${TARGET}", "${@}"]
+
+[tasks.cargo_update]
+command = "${CARGO}"
+args = ["update"]
+
+[tasks.release_tasks]
+condition = { env = { PROFILE_OVERRIDE = "release"} }
+script = '''
+BINS=$(ls ./target/${TARGET}/${PROFILE_OVERRIDE}/u_* -1 | grep -v ".d")
+echo "Stripping..."
+strip $BINS
+echo "Packing..."
+upx -9 $BINS
+'''
+
+[tasks.build]
+dependencies = ["cargo_build", "release_tasks"]
+clear = true
+
+[tasks.run]
+disabled = true
+
+[tasks.run_front]
+script = '''
+cd ./bin/u_panel/src/gui/fe
+ng serve
+'''
+
+[tasks.unit-tests]
+command = "${CARGO}"
+args = ["test", "--target", "${TARGET}", "--lib", "--", "${@}"]
+
+[tasks.ut]
+alias = "unit-tests"
+
+[tasks.integration-tests]
+dependencies = ["cargo_update"]
+script = '''
+[[ ! -d "./target/${TARGET}/${PROFILE_OVERRIDE}" ]] && echo 'No target folder. Build project first' && exit 1
+cd ./integration-tests
+bash integration_tests.sh ${@}
+'''
+
+[tasks.it]
+alias = "integration-tests"
+
+[tasks.test]
+dependencies = ["unit", "integration-tests"]
+
+[tasks.gen_schema]
+script = './scripts/gen_schema.sh'
+
+[tasks.deploy]
+script = './scripts/deploy.sh'

bin/migrator/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "migrator"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+diesel = { workspace = true, features = ["postgres", "serde_json"] }
+diesel_migrations = { version = "2.0.0", features = ["postgres"] }
+openssl = { workspace = true }
+u_lib = { path = "../../lib/u_lib" }
+url = "2.3.1"

bin/migrator/src/database.rs

@@ -0,0 +1,140 @@
+use super::query_helper;
+use diesel::dsl::sql;
+use diesel::sql_types::Bool;
+use diesel::*;
+
+use std::env;
+use std::error::Error;
+
+type DatabaseResult<T> = Result<T, Box<dyn Error>>;
+
+pub enum Backend {
+    Pg,
+}
+
+impl Backend {
+    pub fn for_url(database_url: &str) -> Self {
+        match database_url {
+            _ if database_url.starts_with("postgres://")
+                || database_url.starts_with("postgresql://") =>
+            {
+                    Backend::Pg
+            }
+            _ => panic!(
+                "At least one backend must be specified for use with this crate. \
+                 You may omit the unneeded dependencies in the following command. \n\n \
+                 ex. `cargo install diesel_cli --no-default-features --features mysql postgres sqlite` \n"
+            ),
+        }
+    }
+}
+
+pub enum InferConnection {
+    Pg(PgConnection),
+}
+
+impl InferConnection {
+    pub fn establish(database_url: &str) -> DatabaseResult<Self> {
+        match Backend::for_url(database_url) {
+            Backend::Pg => PgConnection::establish(database_url).map(InferConnection::Pg),
+        }
+        .map_err(Into::into)
+    }
+}
+
+pub fn reset_database() -> DatabaseResult<()> {
+    drop_database(&database_url())?;
+    setup_database()
+}
+
+pub fn setup_database() -> DatabaseResult<()> {
+    let database_url = database_url();
+
+    create_database_if_needed(&database_url)?;
+    Ok(())
+}
+
+pub fn drop_database_command() -> DatabaseResult<()> {
+    drop_database(&database_url())
+}
+
+/// Creates the database specified in the connection url. It returns an error
+/// it was unable to create the database.
+fn create_database_if_needed(database_url: &str) -> DatabaseResult<()> {
+    match Backend::for_url(database_url) {
+        Backend::Pg => {
+            if PgConnection::establish(database_url).is_err() {
+                let (database, postgres_url) = change_database_of_url(database_url, "postgres");
+                println!("Creating database: {}", database);
+                let mut conn = PgConnection::establish(&postgres_url)?;
+                query_helper::create_database(&database).execute(&mut conn)?;
+            }
+        }
+    }
+
+    Ok(())
+}
+
+/// Drops the database specified in the connection url. It returns an error
+/// if it was unable to drop the database.
+fn drop_database(database_url: &str) -> DatabaseResult<()> {
+    match Backend::for_url(database_url) {
+        Backend::Pg => {
+            let (database, postgres_url) = change_database_of_url(database_url, "postgres");
+            let mut conn = PgConnection::establish(&postgres_url)?;
+            if pg_database_exists(&mut conn, &database)? {
+                println!("Dropping database: {}", database);
+                query_helper::drop_database(&database)
+                    .if_exists()
+                    .execute(&mut conn)?;
+            }
+        }
+    }
+    Ok(())
+}
+
+table! {
+    pg_database (datname) {
+        datname -> Text,
+        datistemplate -> Bool,
+    }
+}
+
+fn pg_database_exists(conn: &mut PgConnection, database_name: &str) -> QueryResult<bool> {
+    use self::pg_database::dsl::*;
+
+    pg_database
+        .select(datname)
+        .filter(datname.eq(database_name))
+        .filter(datistemplate.eq(false))
+        .get_result::<String>(conn)
+        .optional()
+        .map(|x| x.is_some())
+}
+
+/// Returns true if the `__diesel_schema_migrations` table exists in the
+/// database we connect to, returns false if it does not.
+pub fn schema_table_exists(database_url: &str) -> DatabaseResult<bool> {
+    match InferConnection::establish(database_url).unwrap() {
+        InferConnection::Pg(mut conn) => select(sql::<Bool>(
+            "EXISTS \
+             (SELECT 1 \
+             FROM information_schema.tables \
+             WHERE table_name = '__diesel_schema_migrations')",
+        ))
+        .get_result(&mut conn),
+    }
+    .map_err(Into::into)
+}
+
+pub fn database_url() -> String {
+    env::var("DATABASE_URL").unwrap()
+}
+
+fn change_database_of_url(database_url: &str, default_database: &str) -> (String, String) {
+    let base = ::url::Url::parse(database_url).unwrap();
+    let database = base.path_segments().unwrap().last().unwrap().to_owned();
+    let mut new_url = base.join(default_database).unwrap();
+    new_url.set_query(base.query());
+    (database, new_url.into())
+}

bin/migrator/src/main.rs

@@ -0,0 +1,88 @@
+// due to linking errors
+extern crate openssl;
+// don't touch anything
+extern crate diesel;
+// in this block
+
+pub mod database;
+pub mod query_helper;
+
+use diesel::migration::Migration;
+use diesel::{migration, pg::PgConnection, Connection};
+use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness};
+use std::error::Error;
+use u_lib::config::DBEnv;
+use u_lib::db::generate_postgres_url;
+
+const MIGRATIONS: EmbeddedMigrations = embed_migrations!();
+
+fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
+    let action = action::parse_command_line()?;
+    let dbconfig = DBEnv::load()?;
+
+    database::setup_database().unwrap();
+
+    let conn = PgConnection::establish(&generate_postgres_url(&dbconfig))?;
+    run(action, conn)
+}
+
+fn run(action: action::Action, mut conn: PgConnection) -> migration::Result<()> {
+    use action::Action::*;
+    match action {
+        ListPending => {
+            let list = conn.pending_migrations(MIGRATIONS)?;
+            if list.is_empty() {
+                println!("No pending migrations.");
+            }
+            for mig in list {
+                println!("Pending migration: {}", mig.name());
+            }
+        }
+        MigrateUp => {
+            let list = conn.run_pending_migrations(MIGRATIONS)?;
+            if list.is_empty() {
+                println!("No pending migrations.");
+            }
+            for mig in list {
+                println!("Applied migration: {}", mig);
+            }
+        }
+        MigrateDown => {
+            let mig = conn.revert_last_migration(MIGRATIONS)?;
+            println!("Reverted migration: {}", mig);
+        }
+    }
+    Ok(())
+}
+
+mod action {
+    pub enum Action {
+        ListPending,
+        MigrateUp,
+        MigrateDown,
+    }
+
+    impl TryFrom<&str> for Action {
+        type Error = ();
+
+        fn try_from(value: &str) -> Result<Self, Self::Error> {
+            match value {
+                "" | "list" => Ok(Action::ListPending),
+                "up" => Ok(Action::MigrateUp),
+                "down" => Ok(Action::MigrateDown),
+                _ => Err(()),
+            }
+        }
+    }
+
+    pub fn parse_command_line() -> Result<Action, String> {
+        let action_str = std::env::args().nth(1).unwrap_or_default();
+        let action = action_str.as_str().try_into().map_err(|_| {
+            format!(
+                "unrecognized command line argument: {} (expected 'up', 'down', 'list')",
+                action_str
+            )
+        })?;
+        Ok(action)
+    }
+}

bin/migrator/src/query_helper.rs

@@ -0,0 +1,82 @@
+use diesel::backend::Backend;
+use diesel::query_builder::*;
+use diesel::result::QueryResult;
+use diesel::RunQueryDsl;
+
+#[derive(Debug, Clone)]
+pub struct DropDatabaseStatement {
+    db_name: String,
+    if_exists: bool,
+}
+
+impl DropDatabaseStatement {
+    pub fn new(db_name: &str) -> Self {
+        DropDatabaseStatement {
+            db_name: db_name.to_owned(),
+            if_exists: false,
+        }
+    }
+
+    pub fn if_exists(self) -> Self {
+        DropDatabaseStatement {
+            if_exists: true,
+            ..self
+        }
+    }
+}
+
+impl<DB: Backend> QueryFragment<DB> for DropDatabaseStatement {
+    fn walk_ast<'b>(&'b self, mut out: AstPass<'_, 'b, DB>) -> QueryResult<()> {
+        out.push_sql("DROP DATABASE ");
+        if self.if_exists {
+            out.push_sql("IF EXISTS ");
+        }
+        out.push_identifier(&self.db_name)?;
+        Ok(())
+    }
+}
+
+impl<Conn> RunQueryDsl<Conn> for DropDatabaseStatement {}
+
+impl QueryId for DropDatabaseStatement {
+    type QueryId = ();
+
+    const HAS_STATIC_QUERY_ID: bool = false;
+}
+
+#[derive(Debug, Clone)]
+pub struct CreateDatabaseStatement {
+    db_name: String,
+}
+
+impl CreateDatabaseStatement {
+    pub fn new(db_name: &str) -> Self {
+        CreateDatabaseStatement {
+            db_name: db_name.to_owned(),
+        }
+    }
+}
+
+impl<DB: Backend> QueryFragment<DB> for CreateDatabaseStatement {
+    fn walk_ast<'b>(&'b self, mut out: AstPass<'_, 'b, DB>) -> QueryResult<()> {
+        out.push_sql("CREATE DATABASE ");
+        out.push_identifier(&self.db_name)?;
+        Ok(())
+    }
+}
+
+impl<Conn> RunQueryDsl<Conn> for CreateDatabaseStatement {}
+
+impl QueryId for CreateDatabaseStatement {
+    type QueryId = ();
+
+    const HAS_STATIC_QUERY_ID: bool = false;
+}
+
+pub fn drop_database(db_name: &str) -> DropDatabaseStatement {
+    DropDatabaseStatement::new(db_name)
+}
+
+pub fn create_database(db_name: &str) -> CreateDatabaseStatement {
+    CreateDatabaseStatement::new(db_name)
+}

bin/u_agent/Cargo.toml

@@ -2,19 +2,15 @@
 name = "u_agent"
 version = "0.1.0"
 authors = ["plazmoid <kronos44@mail.ru>"]
-edition = "2018"
+edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process", "time"] }
+log = { version = "0.4", features = ["release_max_level_off"] }
+reqwest = { workspace = true }
 sysinfo = "0.10.5"
-log = "^0.4"
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "time"] }
-env_logger = "0.8.3"
+uuid = { workspace = true }
-uuid = "0.6.5"
+u_lib = { path = "../../lib/u_lib", features = ["agent"] }
-reqwest = { version = "0.11", features = ["json"] }
-openssl = "*"
-u_lib = { version = "*", path = "../../lib/u_lib" }
 
-[build-dependencies]
-openssl = "*"

bin/u_agent/build.rs

@@ -3,6 +3,6 @@ use std::path::PathBuf;
 fn main() {
     let server_cert = PathBuf::from("../../certs/ca.crt");
     if !server_cert.exists() {
-        panic!("CA certificate doesn't exist. Create it first with certs/gen_certs.sh");
+        panic!("CA certificate doesn't exist. Create it first with scripts/gen_certs.sh");
     }
 }

bin/u_agent/src/lib.rs

@@ -1,86 +1,167 @@
-// TODO:
-// поддержка питона
-// резолв адреса управляющего сервера через DoT
-// кроссплатформенность (реализовать интерфейс для винды и никсов)
-
 #[macro_use]
 extern crate log;
-extern crate env_logger;
 
-use std::env;
+use tokio::runtime::Builder;
 use tokio::time::{sleep, Duration};
+use u_lib::models::PreparedJob;
+use u_lib::scheduler::SCHEDULER;
+use u_lib::u_runner::{IdentifiableFuture, URunner};
 use u_lib::{
-    api::ClientHandler,
+    api::HttpClient,
-    builder::JobBuilder,
     cache::JobCache,
-    executor::pop_completed,
+    config::{get_self_id, EndpointsEnv, AGENT_ITERATION_INTERVAL},
-    models::{AssignedJob, ExecResult},
+    error::ErrChan,
-    UID,
+    logging::init_logger,
-    //daemonize
+    messaging::Reportable,
+    models::AssignedJobById,
 };
 
-#[macro_export]
+async fn process_request(assigned_jobs: Vec<AssignedJobById>, client: &HttpClient) {
-macro_rules! retry_until_ok {
+    for asgn_job in assigned_jobs {
-    ( $body:expr ) => {
+        if !JobCache::contains(asgn_job.job_id) {
-        loop {
+            info!("Fetching job: {}", &asgn_job.job_id);
-            match $body {
+            let mut fetched_job = loop {
-                Ok(r) => break r,
+                //todo: use payload cache
-                Err(e) => error!("{:?}", e),
+                match client.get_full_job(asgn_job.job_id).await {
+                    Ok(result) => break result,
+                    Err(err) => {
+                        debug!("{:?} \nretrying...", err);
+                        sleep(AGENT_ITERATION_INTERVAL).await;
+                    }
+                }
             };
-            sleep(Duration::from_secs(5)).await;
+            if let Some(payload) = &mut fetched_job.payload {
+                if let Err(e) = payload.maybe_split_payload() {
+                    ErrChan::send(e, "pay").await;
+                    continue;
+                }
             }
+            JobCache::insert(fetched_job);
+        }
+
+        let job = match JobCache::get(asgn_job.job_id).as_deref() {
+            Some(job) => job.clone(),
+            None => continue,
         };
+
+        info!("Scheduling job {}", job.meta.id.to_string());
+
+        let schedule = match job.meta.schedule.clone() {
+            Some(sched) => {
+                if sched.is_empty() {
+                    None
+                } else {
+                    match sched.as_str().try_into() {
+                        Ok(s) => Some(s),
+                        Err(err) => {
+                            ErrChan::send(err, "sch").await;
+                            continue;
+                        }
+                    }
+                }
+            }
+            None => None,
+        };
+        SCHEDULER
+            .add_job(schedule, PreparedJob { job, ids: asgn_job })
+            .await;
+    }
 }
 
-pub async fn process_request(job_requests: Vec<AssignedJob>, client: &ClientHandler) {
+async fn error_reporting(client: HttpClient) {
-    if job_requests.len() > 0 {
+    while let Some(err) = ErrChan::recv().await {
-        for jr in &job_requests {
+        let _ = client.report([Reportable::Error(err.clone())]).await;
-            if !JobCache::contains(&jr.job_id) {
-                info!("Fetching job: {}", &jr.job_id);
-                let fetched_job = retry_until_ok!(client.get_jobs(Some(jr.job_id)).await)
-                    .pop()
-                    .unwrap();
-                JobCache::insert(fetched_job);
     }
 }
-        info!(
+
-            "Scheduling jobs: {}",
+async fn agent_loop(client: HttpClient) {
-            job_requests
+    let self_id = get_self_id();
-                .iter()
+
-                .map(|j| j.job_id.to_string())
+    match client.get_personal_jobs(self_id).await {
-                .collect::<Vec<String>>()
+        Ok(jobs) => {
-                .join(", ")
+            process_request(jobs, &client).await;
-        );
+        }
-        let mut builder = JobBuilder::from_request(job_requests);
+        Err(err) => ErrChan::send(err, "pro").await,
-        let errors = builder.pop_errors();
-        if errors.len() > 0 {
-            error!(
-                "Some errors encountered: \n{}",
-                errors
-                    .iter()
-                    .map(|j| j.to_string())
-                    .collect::<Vec<String>>()
-                    .join("\n")
-            );
     }
-        builder.unwrap_one().spawn().await;
+
+    let result: Vec<Reportable> = URunner::pop_completed()
+        .await
+        .into_iter()
+        .map(|result| match result {
+            Ok(r) => Reportable::Assigned(r),
+            Err(e) => Reportable::Error(e),
+        })
+        .collect();
+
+    if !result.is_empty() {
+        if let Err(err) = client.report(result).await {
+            ErrChan::send(err, "rep").await;
         }
     }
+}
+
+pub fn run_forever() -> ! {
+    let env = EndpointsEnv::load();
 
-pub async fn run_forever() {
+    if cfg!(debug_assertions) {
-    //daemonize();
+        let logfile_uid = format!(
-    env_logger::init();
+            "u_agent-{}",
-    let arg_ip = env::args().nth(1);
+            get_self_id()
-    let instance = ClientHandler::new(arg_ip.as_deref());
+                .hyphenated()
-    info!("Connecting to the server");
+                .to_string()
-    loop {
+                .split("-")
-        let job_requests: Vec<AssignedJob> =
+                .next()
-            retry_until_ok!(instance.get_personal_jobs(Some(*UID)).await).into_builtin_vec();
+                .unwrap()
-        process_request(job_requests, &instance).await;
+        );
-        let result: Vec<ExecResult> = pop_completed().await.into_iter().collect();
+        init_logger(Some(&logfile_uid));
-        if result.len() > 0 {
+    } else {
-            retry_until_ok!(instance.report(&result).await);
+        #[cfg(unix)]
+        u_lib::unix::daemonize()
     }
+
+    info!("Starting agent {}", get_self_id());
+
+    Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap()
+        .block_on(async {
+            let client = loop {
+                match HttpClient::new(&env.u_server, None).await {
+                    Ok(client) => break client,
+                    Err(e) => {
+                        error!("client init failed: {}", e);
                         sleep(Duration::from_secs(5)).await;
+                        continue;
+                    }
+                }
+            };
+            {
+                let client = client.clone();
+                SCHEDULER
+                    .add_job(Some("*/3 * * * * * *".try_into().unwrap()), move || {
+                        let client = client.clone();
+
+                        IdentifiableFuture::from_fut_with_ident("error_reporting", async move {
+                            error_reporting(client.clone()).await
+                        })
+                    })
+                    .await;
             }
+
+            {
+                let client = client.clone();
+                SCHEDULER
+                    .add_job(Some("*/3 * * * * * *".try_into().unwrap()), move || {
+                        let client = client.clone();
+
+                        IdentifiableFuture::from_fut_with_ident("agent_loop", async move {
+                            agent_loop(client).await
+                        })
+                    })
+                    .await;
+            }
+
+            SCHEDULER.start_blocking().await
+        })
 }

bin/u_agent/src/main.rs

@@ -1,7 +1,3 @@
-use tokio;
+fn main() {
-use u_agent::run_forever;
+    u_agent::run_forever();
-
-#[tokio::main]
-async fn main() {
-    run_forever().await;
 }

bin/u_panel/Cargo.toml

@@ -2,18 +2,27 @@
 name = "u_panel"
 version = "0.1.0"
 authors = ["plazmoid <kronos44@mail.ru>"]
-edition = "2018"
+edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-tokio = { version = "1.2.0", features = ["macros", "rt-multi-thread", "process"] }
+actix-cors = "0.6.1"
+actix-web = "4.1"
+anyhow = { workspace = true }
+futures-util = "0.3.21"
+mime_guess = { workspace = true }
+once_cell = "1.8.0"
+rust-embed = { version = "6.3.0", features = ["debug-embed", "compression"] }
+serde = { workspace = true }
+serde_json = { workspace = true }
+strum = { version = "0.22.0", features = ["derive"] }
+tokio = { workspace = true, features = ["rt", "rt-multi-thread"] }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
+tracing-appender = { workspace = true }
+shlex = "1.1.0"
 structopt = "0.3.21"
-log = "^0.4"
+thiserror = "1.0.31"
-env_logger = "0.7.1"
+uuid = { workspace = true }
-uuid = "0.6.5"
+u_lib = { version = "*", path = "../../lib/u_lib", features = ["panel"] }
-reqwest = { version = "0.11", features = ["json"] }
-openssl = "*"
-u_lib = { version = "*", path = "../../lib/u_lib" }
-serde_json = "1.0.4"
-serde = { version = "1.0.114", features = ["derive"] }

bin/u_panel/src/argparse.rs

@@ -0,0 +1,176 @@
+use serde_json::{from_str, to_value, Value};
+use structopt::StructOpt;
+use u_lib::{
+    api::HttpClient, messaging::AsMsg, models::*, types::Id, types::PanelResult, UError, UResult,
+};
+
+#[derive(StructOpt, Debug)]
+pub struct Args {
+    #[structopt(subcommand)]
+    cmd: Cmd,
+    #[structopt(short, long, default_value)]
+    brief: Brief,
+}
+
+#[derive(StructOpt, Debug)]
+enum Cmd {
+    Agents(RUD),
+    Jobs(CRUD),
+    Map(AssignedCRUD),
+    Payloads(PayloadCRUD),
+    Ping,
+    Serve,
+}
+
+#[derive(StructOpt, Debug)]
+enum CRUD {
+    Create {
+        item: String,
+    },
+    #[structopt(flatten)]
+    RUD(RUD),
+}
+
+#[derive(StructOpt, Debug)]
+enum AssignedCRUD {
+    Create {
+        item: String,
+    },
+    #[structopt(flatten)]
+    RUD(RUD),
+}
+
+#[derive(StructOpt, Debug)]
+enum PayloadCRUD {
+    Create {
+        item: String,
+    },
+    Read {
+        id: Option<String>,
+    },
+    Update {
+        item: String,
+    },
+    Delete {
+        #[structopt(parse(try_from_str = parse::uuid))]
+        id: Id,
+    },
+}
+
+#[derive(StructOpt, Debug)]
+enum RUD {
+    Read {
+        #[structopt(parse(try_from_str = parse::uuid))]
+        id: Option<Id>,
+    },
+    Update {
+        item: String,
+    },
+    Delete {
+        #[structopt(parse(try_from_str = parse::uuid))]
+        id: Id,
+    },
+}
+
+mod parse {
+    use super::*;
+
+    pub fn uuid(src: &str) -> Result<Id, String> {
+        Id::parse_str(src).map_err(|e| e.to_string())
+    }
+}
+
+pub fn into_value<M: AsMsg>(data: M) -> Value {
+    to_value(data).unwrap()
+}
+
+pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult<Value> {
+    let catcher: UResult<Value> = (|| async {
+        Ok(match args.cmd {
+            Cmd::Agents(action) => match action {
+                RUD::Read { id } => into_value(client.get_agents(id).await?),
+                RUD::Update { item } => {
+                    let agent = from_str::<Agent>(&item)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), item))?;
+                    into_value(client.update_agent(&agent).await?)
+                }
+                RUD::Delete { id } => into_value(client.del(id).await?),
+            },
+            Cmd::Jobs(action) => match action {
+                CRUD::Create { item: job } => {
+                    let raw_job = from_str::<RawJob>(&job)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), job))?;
+                    let mut job = raw_job.try_into_job()?;
+
+                    if let Some(payload) = &mut job.payload {
+                        payload.join_payload()?;
+                    }
+
+                    into_value(client.upload_jobs([&job]).await?)
+                }
+                CRUD::RUD(RUD::Read { id }) => match id {
+                    Some(id) => into_value(vec![client.get_job(id, args.brief).await?]),
+                    None => into_value(client.get_jobs().await?),
+                },
+                CRUD::RUD(RUD::Update { item }) => {
+                    let raw_job = from_str::<JobMeta>(&item)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), item))?;
+                    let job = raw_job.validate()?;
+
+                    // if let Some(payload) = &mut job.payload {
+                    //     payload.join_payload()?;
+                    // }
+
+                    into_value(client.update_job(&job).await?)
+                }
+                CRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?),
+            },
+            Cmd::Map(action) => match action {
+                AssignedCRUD::Create { item } => {
+                    let payload = serde_json::from_str::<Vec<AssignedJobById>>(&item)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), item))?;
+                    into_value(client.assign_jobs(&payload).await?)
+                }
+                AssignedCRUD::RUD(RUD::Read { id }) => {
+                    into_value(client.get_assigned_jobs(id).await?)
+                }
+                AssignedCRUD::RUD(RUD::Update { item }) => {
+                    let assigned = from_str::<AssignedJob>(&item)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), item))?;
+                    into_value(client.update_result(&assigned).await?)
+                }
+                AssignedCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?),
+            },
+            Cmd::Payloads(action) => match action {
+                PayloadCRUD::Create { item } => {
+                    let payload = from_str::<RawPayload>(&item)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), item))?;
+                    into_value(client.upload_payload(&payload).await?)
+                }
+                PayloadCRUD::Read { id } => match id {
+                    None => into_value(client.get_payloads().await?),
+                    Some(id) => into_value(vec![client.get_payload(id, args.brief).await?]),
+                },
+                PayloadCRUD::Update { item } => {
+                    let payload = from_str::<Payload>(&item)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), item))?;
+                    into_value(client.update_payload(&payload).await?)
+                }
+                PayloadCRUD::Delete { id } => into_value(client.del(id).await?),
+            },
+            Cmd::Ping => into_value(client.ping().await?),
+            Cmd::Serve => {
+                crate::gui::serve(client)
+                    .await
+                    .map_err(|e| UError::PanelError(format!("{e:?}")))?;
+                Value::Null
+            }
+        })
+    })()
+    .await;
+
+    match catcher {
+        Ok(r) => PanelResult::Ok(r),
+        Err(e) => PanelResult::Err(e),
+    }
+}

bin/u_panel/src/gui/error.rs

@@ -0,0 +1,17 @@
+use actix_web::http::StatusCode;
+use actix_web::ResponseError;
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("Arg parse error: {0}")]
+    ArgparseError(#[from] structopt::clap::Error),
+
+    #[error("Just an error: {0}")]
+    JustError(String),
+}
+
+impl ResponseError for Error {
+    fn status_code(&self) -> actix_web::http::StatusCode {
+        StatusCode::BAD_REQUEST
+    }
+}

bin/u_panel/src/gui/fe/.browserslistrc

@@ -0,0 +1,16 @@
+# This file is used by the build system to adjust CSS and JS output to support the specified browsers below.
+# For additional information regarding the format and rule options, please see:
+# https://github.com/browserslist/browserslist#queries
+
+# For the full list of supported browsers by the Angular framework, please see:
+# https://angular.io/guide/browser-support
+
+# You can see what browsers were selected by your queries by running:
+#   npx browserslist
+
+last 1 Chrome version
+last 1 Firefox version
+last 2 Edge major versions
+last 2 Safari major versions
+last 2 iOS major versions
+Firefox ESR

bin/u_panel/src/gui/fe/.editorconfig

@@ -0,0 +1,16 @@
+# Editor configuration, see https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.ts]
+quote_type = single
+
+[*.md]
+max_line_length = off
+trim_trailing_whitespace = false

bin/u_panel/src/gui/fe/.gitignore

@@ -0,0 +1,48 @@
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+
+# compiled output
+/dist
+/tmp
+/out-tsc
+# Only exists if Bazel was run
+/bazel-out
+
+# dependencies
+/node_modules
+
+# profiling files
+chrome-profiler-events*.json
+
+# IDEs and editors
+/.idea
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# IDE - VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+.history/*
+
+# misc
+/.angular/cache
+/.sass-cache
+/connect.lock
+/coverage
+/libpeerconnection.log
+npm-debug.log
+yarn-error.log
+testem.log
+/typings
+
+# System Files
+.DS_Store
+Thumbs.db
+
+package-lock.json

bin/u_panel/src/gui/fe/README.md

@@ -0,0 +1,27 @@
+# Fe
+
+This project was generated with [Angular CLI](https://github.com/angular/angular-cli) version 13.1.2.
+
+## Development server
+
+Run `ng serve` for a dev server. Navigate to `http://localhost:4200/`. The app will automatically reload if you change any of the source files.
+
+## Code scaffolding
+
+Run `ng generate component component-name` to generate a new component. You can also use `ng generate directive|pipe|service|class|guard|interface|enum|module`.
+
+## Build
+
+Run `ng build` to build the project. The build artifacts will be stored in the `dist/` directory.
+
+## Running unit tests
+
+Run `ng test` to execute the unit tests via [Karma](https://karma-runner.github.io).
+
+## Running end-to-end tests
+
+Run `ng e2e` to execute the end-to-end tests via a platform of your choice. To use this command, you need to first add a package that implements end-to-end testing capabilities.
+
+## Further help
+
+To get more help on the Angular CLI use `ng help` or go check out the [Angular CLI Overview and Command Reference](https://angular.io/cli) page.

bin/u_panel/src/gui/fe/angular.json

@@ -0,0 +1,114 @@
+{
+  "$schema": "./node_modules/@angular/cli/lib/config/schema.json",
+  "version": 1,
+  "newProjectRoot": "projects",
+  "projects": {
+    "fe": {
+      "projectType": "application",
+      "schematics": {
+        "@schematics/angular:component": {
+          "style": "less"
+        },
+        "@schematics/angular:application": {
+          "strict": true
+        }
+      },
+      "root": "",
+      "sourceRoot": "src",
+      "prefix": "app",
+      "architect": {
+        "build": {
+          "builder": "@angular-devkit/build-angular:browser",
+          "options": {
+            "outputPath": "dist/fe",
+            "index": "src/index.html",
+            "main": "src/main.ts",
+            "polyfills": "src/polyfills.ts",
+            "tsConfig": "tsconfig.app.json",
+            "inlineStyleLanguage": "less",
+            "assets": [
+              "src/favicon.ico",
+              "src/assets"
+            ],
+            "styles": [
+              "./node_modules/@angular/material/prebuilt-themes/deeppurple-amber.css",
+              "src/styles.less"
+            ],
+            "scripts": []
+          },
+          "configurations": {
+            "production": {
+              "baseHref": "/core/",
+              "budgets": [
+                {
+                  "type": "initial",
+                  "maximumWarning": "500kb",
+                  "maximumError": "1mb"
+                },
+                {
+                  "type": "anyComponentStyle",
+                  "maximumWarning": "2kb",
+                  "maximumError": "4kb"
+                }
+              ],
+              "fileReplacements": [
+                {
+                  "replace": "src/environments/environment.ts",
+                  "with": "src/environments/environment.prod.ts"
+                }
+              ],
+              "outputHashing": "all"
+            },
+            "development": {
+              "buildOptimizer": false,
+              "optimization": false,
+              "vendorChunk": true,
+              "extractLicenses": false,
+              "sourceMap": true,
+              "namedChunks": true
+            }
+          },
+          "defaultConfiguration": "production"
+        },
+        "serve": {
+          "builder": "@angular-devkit/build-angular:dev-server",
+          "configurations": {
+            "production": {
+              "browserTarget": "fe:build:production"
+            },
+            "development": {
+              "browserTarget": "fe:build:development"
+            }
+          },
+          "defaultConfiguration": "development"
+        },
+        "extract-i18n": {
+          "builder": "@angular-devkit/build-angular:extract-i18n",
+          "options": {
+            "browserTarget": "fe:build"
+          }
+        },
+        "test": {
+          "builder": "@angular-devkit/build-angular:karma",
+          "options": {
+            "main": "src/test.ts",
+            "polyfills": "src/polyfills.ts",
+            "tsConfig": "tsconfig.spec.json",
+            "karmaConfig": "karma.conf.js",
+            "inlineStyleLanguage": "less",
+            "assets": [
+              "src/favicon.ico",
+              "src/assets"
+            ],
+            "styles": [
+              "./node_modules/@angular/material/prebuilt-themes/deeppurple-amber.css",
+              "src/styles.less"
+            ],
+            "scripts": []
+          }
+        }
+      }
+    }
+  },
+  "defaultProject": "fe"
+}

bin/u_panel/src/gui/fe/karma.conf.js

@@ -0,0 +1,44 @@
+// Karma configuration file, see link for more information
+// https://karma-runner.github.io/1.0/config/configuration-file.html
+
+module.exports = function (config) {
+  config.set({
+    basePath: '',
+    frameworks: ['jasmine', '@angular-devkit/build-angular'],
+    plugins: [
+      require('karma-jasmine'),
+      require('karma-chrome-launcher'),
+      require('karma-jasmine-html-reporter'),
+      require('karma-coverage'),
+      require('@angular-devkit/build-angular/plugins/karma')
+    ],
+    client: {
+      jasmine: {
+        // you can add configuration options for Jasmine here
+        // the possible options are listed at https://jasmine.github.io/api/edge/Configuration.html
+        // for example, you can disable the random execution with `random: false`
+        // or set a specific seed with `seed: 4321`
+      },
+      clearContext: false // leave Jasmine Spec Runner output visible in browser
+    },
+    jasmineHtmlReporter: {
+      suppressAll: true // removes the duplicated traces
+    },
+    coverageReporter: {
+      dir: require('path').join(__dirname, './coverage/fe'),
+      subdir: '.',
+      reporters: [
+        { type: 'html' },
+        { type: 'text-summary' }
+      ]
+    },
+    reporters: ['progress', 'kjhtml'],
+    port: 9876,
+    colors: true,
+    logLevel: config.LOG_INFO,
+    autoWatch: true,
+    browsers: ['Chrome'],
+    singleRun: false,
+    restartOnFileChange: true
+  });
+};

bin/u_panel/src/gui/fe/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "fe",
+  "version": "0.0.0",
+  "scripts": {
+    "ng": "ng",
+    "start": "ng serve",
+    "build": "ng build",
+    "watch": "ng build --watch --configuration development",
+    "test": "ng test"
+  },
+  "private": true,
+  "dependencies": {
+    "@angular/animations": "~13.1.0",
+    "@angular/cdk": "^13.3.9",
+    "@angular/common": "~13.1.0",
+    "@angular/compiler": "~13.1.0",
+    "@angular/core": "~13.1.0",
+    "@angular/forms": "~13.1.0",
+    "@angular/material": "^13.3.9",
+    "@angular/platform-browser": "~13.1.0",
+    "@angular/platform-browser-dynamic": "~13.1.0",
+    "@angular/router": "~13.1.0",
+    "@types/uuid": "^8.3.4",
+    "rxjs": "~7.4.0",
+    "tslib": "^2.3.0",
+    "uuid": "^8.3.2",
+    "zone.js": "~0.11.4"
+  },
+  "devDependencies": {
+    "@angular-devkit/build-angular": "^13.3.9",
+    "@angular/cli": "~13.1.2",
+    "@angular/compiler-cli": "~13.1.0",
+    "@types/jasmine": "~3.10.0",
+    "@types/node": "^12.11.1",
+    "jasmine-core": "~3.10.0",
+    "karma": "~6.3.0",
+    "karma-chrome-launcher": "~3.1.0",
+    "karma-coverage": "~2.1.0",
+    "karma-jasmine": "~4.0.0",
+    "karma-jasmine-html-reporter": "~1.7.0",
+    "typescript": "~4.5.2"
+  }
+}

bin/u_panel/src/gui/fe/src/app/app-routing.module.ts

@@ -0,0 +1,18 @@
+import { NgModule } from '@angular/core';
+import { RouterModule, Routes } from '@angular/router';
+import { JobComponent, ResultComponent, AgentComponent, PayloadComponent } from './components/tables';
+//import { AgentInfoDialogComponent } from './core/tables/dialogs/agent-info-dialog.component';
+
+const routes: Routes = [
+  { path: '', redirectTo: 'agents', pathMatch: 'full' },
+  { path: 'agents', component: AgentComponent },
+  { path: 'jobs', component: JobComponent },
+  { path: 'payloads', component: PayloadComponent },
+  { path: 'results', component: ResultComponent },
+];
+
+@NgModule({
+  imports: [RouterModule.forRoot(routes)],
+  exports: [RouterModule]
+})
+export class AppRoutingModule { }

bin/u_panel/src/gui/fe/src/app/app.component.html

@@ -0,0 +1,6 @@
+<nav mat-tab-nav-bar animationDuration="0ms" mat-align-tabs="center">
+  <a mat-tab-link *ngFor="let tab of tabs" routerLink={{tab.link}} routerLinkActive #rla="routerLinkActive"
+    [active]="rla.isActive" [routerLinkActiveOptions]="{ exact: true }">{{tab.name}}</a>
+</nav>
+<router-outlet></router-outlet>
+<global-error></global-error>

bin/u_panel/src/gui/fe/src/app/app.component.spec.ts

@@ -0,0 +1,35 @@
+import { TestBed } from '@angular/core/testing';
+import { RouterTestingModule } from '@angular/router/testing';
+import { AppComponent } from './app.component';
+
+describe('AppComponent', () => {
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [
+        RouterTestingModule
+      ],
+      declarations: [
+        AppComponent
+      ],
+    }).compileComponents();
+  });
+
+  it('should create the app', () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    const app = fixture.componentInstance;
+    expect(app).toBeTruthy();
+  });
+
+  it(`should have as title 'fe'`, () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    const app = fixture.componentInstance;
+    expect(app.title).toEqual('fe');
+  });
+
+  it('should render title', () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    fixture.detectChanges();
+    const compiled = fixture.nativeElement as HTMLElement;
+    expect(compiled.querySelector('.content span')?.textContent).toContain('fe app is running!');
+  });
+});

bin/u_panel/src/gui/fe/src/app/app.component.ts

@@ -0,0 +1,15 @@
+import { Component } from '@angular/core';
+
+@Component({
+  selector: 'app-root',
+  templateUrl: './app.component.html',
+  styleUrls: ['./app.component.less']
+})
+export class AppComponent {
+  tabs = [
+    { name: 'Agents', link: '/agents' },
+    { name: 'Jobs', link: '/jobs' },
+    { name: 'Results', link: '/results' },
+    { name: 'Payloads', link: '/payloads' }
+  ];
+}

bin/u_panel/src/gui/fe/src/app/app.module.ts

@@ -0,0 +1,73 @@
+import { NgModule } from '@angular/core';
+import { BrowserModule } from '@angular/platform-browser';
+import { AppRoutingModule } from './app-routing.module';
+import { AppComponent } from './app.component';
+import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
+import { MatTabsModule } from '@angular/material/tabs';
+import { MatTableModule } from '@angular/material/table';
+import { MatFormFieldModule } from '@angular/material/form-field';
+import { MatButtonModule } from '@angular/material/button'
+import { MatInputModule } from '@angular/material/input';
+import { MatSelectModule } from '@angular/material/select';
+import { MatProgressSpinnerModule } from '@angular/material/progress-spinner';
+import { HttpClientModule } from '@angular/common/http';
+import { MatDialogModule } from '@angular/material/dialog';
+import { MatGridListModule } from '@angular/material/grid-list';
+import { MatIconModule } from '@angular/material/icon';
+import { FormsModule } from '@angular/forms';
+import { AgentComponent, JobComponent, ResultComponent, PayloadComponent } from './components/tables';
+import {
+  AgentInfoDialogComponent,
+  AssignJobDialogComponent,
+  JobInfoDialogComponent,
+  ResultInfoDialogComponent,
+  PayloadInfoDialogComponent
+} from './components/dialogs';
+import { APP_BASE_HREF } from '@angular/common';
+import { MatTooltipModule } from '@angular/material/tooltip';
+import { MatSnackBarModule } from '@angular/material/snack-bar';
+import { MatListModule } from '@angular/material/list';
+import { GlobalErrorComponent } from './components/global-error/global-error.component';
+import { PayloadOverviewComponent } from './components/payload-overview/payload-overview.component';
+import { NewPayloadDialogComponent } from './components/dialogs/new-payload-dialog/new-payload-dialog.component';
+
+@NgModule({
+  declarations: [
+    AppComponent,
+    AgentComponent,
+    JobComponent,
+    ResultComponent,
+    AgentInfoDialogComponent,
+    JobInfoDialogComponent,
+    ResultInfoDialogComponent,
+    AssignJobDialogComponent,
+    PayloadComponent,
+    PayloadInfoDialogComponent,
+    GlobalErrorComponent,
+    PayloadOverviewComponent,
+    NewPayloadDialogComponent
+  ],
+  imports: [
+    BrowserModule,
+    HttpClientModule,
+    AppRoutingModule,
+    MatTabsModule,
+    MatTableModule,
+    MatButtonModule,
+    MatFormFieldModule,
+    MatInputModule,
+    MatDialogModule,
+    MatProgressSpinnerModule,
+    MatIconModule,
+    MatTooltipModule,
+    MatSnackBarModule,
+    MatSelectModule,
+    MatListModule,
+    MatGridListModule,
+    FormsModule,
+    BrowserAnimationsModule
+  ],
+  providers: [{ provide: APP_BASE_HREF, useValue: '/' }],
+  bootstrap: [AppComponent]
+})
+export class AppModule { }

bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.html

@@ -0,0 +1,64 @@
+<h2 mat-dialog-title *ngIf="is_preview">Agent info</h2>
+<h2 mat-dialog-title *ngIf="!is_preview">Editing agent info</h2>
+<mat-dialog-content>
+    <p>
+        <mat-form-field class="info-dlg-field" cdkFocusInitial>
+            <mat-label>ID</mat-label>
+            <input matInput disabled value="{{data.id}}">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Alias</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.alias">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Username</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.username">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Hostname</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.hostname">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Host info</mat-label>
+            <textarea matInput cdkTextareaAutosize [readonly]="is_preview" [(ngModel)]="data.host_info">
+            </textarea>
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Platform</mat-label>
+            <input matInput [readonly]="is_preview" [(ngModel)]="data.platform">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Is root</mat-label>
+            <input matInput disabled value="{{data.is_root}}">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Registration time</mat-label>
+            <input matInput disabled value="{{data.regtime.secs_since_epoch * 1000 | date:'long'}}">
+        </mat-form-field>
+    </p>
+    <p>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Last active time</mat-label>
+            <input matInput disabled value="{{data.last_active.secs_since_epoch * 1000 | date:'long'}}">
+        </mat-form-field>
+    </p>
+</mat-dialog-content>
+<mat-dialog-actions align="end">
+    <button mat-raised-button *ngIf="is_preview" (click)="is_preview = false">Edit</button>
+    <button mat-raised-button *ngIf="!is_preview" (click)="updateAgent()">Save</button>
+    <button mat-button mat-dialog-close>Cancel</button>
+</mat-dialog-actions>

bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.ts

@@ -0,0 +1,21 @@
+import { Component, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { AgentModel } from '../../../models/agent.model';
+import { EventEmitter } from '@angular/core';
+
+@Component({
+    selector: 'agent-info-dialog',
+    templateUrl: 'agent-info-dialog.component.html',
+    styleUrls: ['../base-info-dialog.component.less']
+})
+export class AgentInfoDialogComponent {
+    is_preview = true;
+    onSave = new EventEmitter();
+
+    constructor(@Inject(MAT_DIALOG_DATA) public data: AgentModel) { }
+
+    updateAgent() {
+        console.log(this.data);
+        this.onSave.emit(this.data);
+    }
+}

bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.html

@@ -0,0 +1,12 @@
+<h2 mat-dialog-title>Assign job</h2>
+<mat-dialog-content>
+    <mat-selection-list #jobsList [(ngModel)]="selected_rows">
+        <mat-list-option *ngFor="let row of rows" [value]="row">
+            {{row}}
+        </mat-list-option>
+    </mat-selection-list>
+</mat-dialog-content>
+<mat-dialog-actions align="end">
+    <button mat-raised-button mat-dialog-close (click)="assignSelectedJobs()">Assign</button>
+    <button mat-button mat-dialog-close>Cancel</button>
+</mat-dialog-actions>

bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts

@@ -0,0 +1,36 @@
+import { Component, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { AssignedJobByIdModel } from 'src/app/models';
+import { ApiTableService } from '../../../services';
+
+@Component({
+    selector: 'assign-job-dialog',
+    templateUrl: 'assign-job-dialog.component.html',
+    styleUrls: []
+})
+export class AssignJobDialogComponent {
+    rows: string[] = [];
+    selected_rows: string[] = [];
+
+    constructor(
+        @Inject(MAT_DIALOG_DATA) public agent_id: string,
+        private dataSource: ApiTableService,
+    ) {
+        dataSource.getJobs().subscribe(resp => {
+            this.rows = resp.map(j => `${j.id} ${j.alias}`)
+        })
+    }
+
+    assignSelectedJobs() {
+        const assigned_jobs: AssignedJobByIdModel[] = this.selected_rows.map(row => {
+            const job_id = row.split(' ', 1)[0];
+            return {
+                job_id: job_id,
+                agent_id: this.agent_id
+            }
+        });
+        this.dataSource.createResult(assigned_jobs).subscribe(_ => {
+            alert("Created")
+        });
+    }
+}

bin/u_panel/src/gui/fe/src/app/components/dialogs/base-info-dialog.component.less

@@ -0,0 +1,18 @@
+.info-dlg-field {
+    width: 100%;
+}
+
+div.info-dialog-forms-box {
+    width: 100%;
+    margin-right: 10px;
+}
+
+div.info-dialog-forms-box-smol {
+    width: 30%;
+    float: left;
+    margin-right: 10px;
+}
+
+.code {
+    font-family: "Roboto Mono", monospace;
+}

bin/u_panel/src/gui/fe/src/app/components/dialogs/index.ts

@@ -0,0 +1,5 @@
+export * from './agent-info-dialog/agent-info-dialog.component';
+export * from './result-info-dialog/result-info-dialog.component';
+export * from './job-info-dialog/job-info-dialog.component';
+export * from './assign-job-dialog/assign-job-dialog.component';
+export * from './payload-info-dialog/payload-info-dialog.component';

bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html

@@ -0,0 +1,48 @@
+<h2 mat-dialog-title *ngIf="isPreview">Job info</h2>
+<h2 mat-dialog-title *ngIf="!isPreview">Editing job info</h2>
+<mat-dialog-content>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field">
+            <mat-label>ID</mat-label>
+            <input matInput disabled value="{{data.meta.id}}">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Alias</mat-label>
+            <input matInput [readonly]="isPreview" [(ngModel)]="data.meta.alias">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Args</mat-label>
+            <input matInput [readonly]="isPreview" [(ngModel)]="data.meta.argv">
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Type</mat-label>
+            <input matInput [readonly]="isPreview" [(ngModel)]="data.meta.exec_type">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Platform</mat-label>
+            <input matInput [readonly]="isPreview" [(ngModel)]="data.meta.target_platforms">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Schedule</mat-label>
+            <input matInput [readonly]="isPreview" [(ngModel)]="data.meta.schedule">
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box">
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Payload</mat-label>
+            <mat-select [disabled]="isPreview" [(value)]="data.meta.payload_id">
+                <mat-option *ngFor="let pld of allPayloads" [value]="pld[0]">{{ pld[1] }}</mat-option>
+            </mat-select>
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box">
+        <payload-overview *ngIf="data.payload" [preview]="true" [payload]="data.payload.data"></payload-overview>
+    </div>
+</mat-dialog-content>
+<mat-dialog-actions align="end">
+    <button mat-raised-button *ngIf="isPreview" (click)="isPreview = false">Edit</button>
+    <button mat-raised-button *ngIf="!isPreview" (click)="updateJob()">Save</button>
+    <button mat-button mat-dialog-close>Cancel</button>
+</mat-dialog-actions>

bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts

@@ -0,0 +1,28 @@
+import { Component, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { EventEmitter } from '@angular/core';
+import { Job, JobModel } from '../../../models/job.model';
+import { ApiTableService } from 'src/app/services';
+
+@Component({
+    selector: 'job-info-dialog',
+    templateUrl: 'job-info-dialog.component.html',
+    styleUrls: ['../base-info-dialog.component.less']
+})
+export class JobInfoDialogComponent {
+    //[id, name]
+    isPreview = true;
+    allPayloads: [string | null, string][] = [[null, "none"]];
+
+    onSave = new EventEmitter<JobModel>();
+
+    constructor(@Inject(MAT_DIALOG_DATA) public data: Job, dataSource: ApiTableService) {
+        dataSource.getPayloads().subscribe(resp => {
+            this.allPayloads = this.allPayloads.concat(resp.map(r => [r.id, r.name]))
+        })
+    }
+
+    updateJob() {
+        this.onSave.emit(this.data.meta);
+    }
+}

bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.html

@@ -0,0 +1,20 @@
+<h2 mat-dialog-title>New payload</h2>
+<mat-dialog-content>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field" cdkFocusInitial>
+            <mat-label>Name</mat-label>
+            <input matInput [(ngModel)]="payload.name">
+        </mat-form-field>
+        <input type="file" class="file-input" (change)="onFileSelected($event)" #fileUpload>
+    </div>
+    <div class="info-dialog-forms-box">
+        <mat-form-field class="info-dlg-field" *ngIf="!uploadMode">
+            <mat-label>Data</mat-label>
+            <textarea matInput [(ngModel)]="decodedPayload"></textarea>
+        </mat-form-field>
+    </div>
+</mat-dialog-content>
+<mat-dialog-actions align="end">
+    <button mat-raised-button (click)="save()">Save</button>
+    <button mat-button mat-dialog-close>Close</button>
+</mat-dialog-actions>

bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.ts

@@ -0,0 +1,43 @@
+import { Component, EventEmitter, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { NewPayloadModel } from 'src/app/models/payload.model';
+
+@Component({
+  selector: 'new-payload-dialog',
+  templateUrl: 'new-payload-dialog.component.html',
+  styleUrls: ['../base-info-dialog.component.less']
+})
+export class NewPayloadDialogComponent {
+  decodedPayload = "";
+  uploadMode = false;
+  onSave = new EventEmitter<NewPayloadModel>();
+
+  constructor(@Inject(MAT_DIALOG_DATA) public payload: NewPayloadModel) { }
+
+  save() {
+    if (this.payload.data.length == 0) {
+      this.payload.data = Array.from(new TextEncoder().encode(this.decodedPayload));
+    }
+    this.onSave.emit(this.payload);
+  }
+
+  onFileSelected(event: any) {
+    const file: File = event.target.files[0];
+    if (file) {
+      this.uploadMode = true
+      const reader = new FileReader();
+      reader.onload = e => {
+        this.payload.name = file.name;
+        const result = e.target?.result;
+        if (result instanceof ArrayBuffer) {
+          const d = Array.from(new Uint8Array(result));
+          this.payload.data = d;
+          console.log(this.payload.data)
+        } else {
+          alert!("no file")
+        }
+      }
+      reader.readAsArrayBuffer(file)
+    }
+  }
+}

bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html

@@ -0,0 +1,34 @@
+<h2 mat-dialog-title *ngIf="isPreview">Payload</h2>
+<h2 mat-dialog-title *ngIf="!isPreview">Editing payload</h2>
+<mat-dialog-content>
+    <div class="info-dialog-forms-box">
+        <div class="info-dialog-forms-box-smol">
+            <mat-form-field class="info-dlg-field" cdkFocusInitial>
+                <mat-label>ID</mat-label>
+                <input matInput disabled value="{{payload.id}}">
+            </mat-form-field>
+            <mat-form-field class="info-dlg-field">
+                <mat-label>Name</mat-label>
+                <input matInput [readonly]="isPreview" [(ngModel)]="payload.name">
+            </mat-form-field>
+        </div>
+        <div class="info-dialog-forms-box-smol">
+            <mat-form-field class="info-dlg-field">
+                <mat-label>MIME-type</mat-label>
+                <input matInput disabled value="{{payload.mime_type}}">
+            </mat-form-field>
+            <mat-form-field class="info-dlg-field">
+                <mat-label>Size</mat-label>
+                <input matInput disabled value="{{payload.size}}">
+            </mat-form-field>
+        </div>
+    </div>
+    <div class="info-dialog-forms-box">
+        <payload-overview [preview]="isPreview" [payload]="payload.data"></payload-overview>
+    </div>
+</mat-dialog-content>
+<mat-dialog-actions align="end">
+    <button mat-raised-button *ngIf="isPreview" (click)="isPreview = false">Edit</button>
+    <button mat-raised-button *ngIf="!isPreview" (click)="updatePayload()">Save</button>
+    <button mat-button mat-dialog-close>Close</button>
+</mat-dialog-actions>

bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts

@@ -0,0 +1,20 @@
+import { Component, EventEmitter, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { PayloadModel } from 'src/app/models/payload.model';
+
+@Component({
+  selector: 'payload-info-dialog',
+  templateUrl: 'payload-info-dialog.component.html',
+  styleUrls: ['../base-info-dialog.component.less']
+})
+export class PayloadInfoDialogComponent {
+  isPreview = true;
+
+  onSave = new EventEmitter<PayloadModel>();
+
+  constructor(@Inject(MAT_DIALOG_DATA) public payload: PayloadModel) { }
+
+  updatePayload() {
+    this.onSave.emit(this.payload);
+  }
+}

bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.html

@@ -0,0 +1,47 @@
+<h2 mat-dialog-title>Result</h2>
+<mat-dialog-content>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field" cdkFocusInitial>
+            <mat-label>ID</mat-label>
+            <input matInput readonly value="{{data.id}}">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Job ID</mat-label>
+            <input matInput readonly value="{{data.job_id}}">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Agent ID</mat-label>
+            <input matInput readonly value="{{data.agent_id}}">
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Alias</mat-label>
+            <input matInput readonly value="{{data.alias}}">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>State</mat-label>
+            <input matInput readonly value="{{data.state}}">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Return code</mat-label>
+            <input matInput readonly value="{{data.retcode}}">
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Created</mat-label>
+            <input matInput readonly value="{{data.created.secs_since_epoch * 1000 | date:'long'}}">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Updated</mat-label>
+            <input matInput readonly value="{{data.updated.secs_since_epoch * 1000 | date:'long'}}">
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box">
+        <payload-overview [preview]="true" [payload]="data.result"></payload-overview>
+    </div>
+</mat-dialog-content>
+<mat-dialog-actions align="end">
+    <button mat-button mat-dialog-close>Close</button>
+</mat-dialog-actions>

bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.ts

@@ -0,0 +1,20 @@
+import { Component, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { ResultModel } from '../../../models/result.model';
+
+@Component({
+    selector: 'result-info-dialog',
+    templateUrl: 'result-info-dialog.component.html',
+    styleUrls: ['../base-info-dialog.component.less']
+})
+export class ResultInfoDialogComponent {
+    decodedResult: string;
+
+    constructor(@Inject(MAT_DIALOG_DATA) public data: ResultModel) {
+        if (data.result !== null) {
+            this.decodedResult = new TextDecoder().decode(new Uint8Array(data.result))
+        } else {
+            this.decodedResult = ""
+        }
+    }
+}

bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.ts

@@ -0,0 +1,34 @@
+import { Component, OnInit } from '@angular/core';
+import { MatSnackBar, MatSnackBarConfig } from '@angular/material/snack-bar';
+import { ErrorService } from 'src/app/services/error.service';
+
+@Component({
+  selector: 'global-error',
+  templateUrl: './global-error.component.html',
+  styleUrls: ['./global-error.component.less']
+})
+export class GlobalErrorComponent implements OnInit {
+
+  constructor(
+    private snackBar: MatSnackBar,
+    private errorService: ErrorService
+  ) { }
+
+  ngOnInit() {
+    this.errorService.error$.subscribe(err => {
+      const _config = (duration: number): MatSnackBarConfig => {
+        return {
+          horizontalPosition: 'right',
+          verticalPosition: 'bottom',
+          duration
+        }
+      }
+      const error = true;
+      const cfg = error ? _config(0) : _config(2000)
+
+      if (err != '') {
+        this.snackBar.open(err, 'Ok', cfg)
+      }
+    })
+  }
+}

bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html

@@ -0,0 +1,7 @@
+<mat-form-field style="box-sizing:border-box; width:100%" class="info-dlg-field" floatLabel="always">
+    <mat-label>Payload data</mat-label>
+    <textarea class="code" matInput cdkTextareaAutosize="true" *ngIf="!isTooBigPayload" [readonly]="isPreview"
+        [(ngModel)]="decodedPayload">
+    </textarea>
+    <input matInput *ngIf="isTooBigPayload" disabled placeholder="Payload is too big to display">
+</mat-form-field>

bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts

@@ -0,0 +1,21 @@
+import { Component, Input, OnInit } from '@angular/core';
+
+@Component({
+  selector: 'payload-overview',
+  templateUrl: './payload-overview.component.html',
+  styleUrls: ['./payload-overview.component.less']
+})
+export class PayloadOverviewComponent implements OnInit {
+  @Input() payload: number[] | null = null;
+  @Input("preview") isPreview = true;
+  isTooBigPayload = false;
+  decodedPayload = "";
+
+  ngOnInit() {
+    if (this.payload !== null) {
+      this.decodedPayload = new TextDecoder().decode(new Uint8Array(this.payload))
+    } else {
+      this.isTooBigPayload = true
+    }
+  }
+}

bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.html

@@ -0,0 +1,78 @@
+<div class="mat-elevation-z8">
+
+    <div class="table-container">
+        <div class="loading-shade" *ngIf="isLoadingResults">
+            <mat-spinner *ngIf="isLoadingResults"></mat-spinner>
+        </div>
+        <mat-form-field appearance="standard">
+            <mat-label>Filter</mat-label>
+            <input matInput (keyup)="applyFilter($event)" #input>
+        </mat-form-field>
+        <button id="refresh_btn" mat-raised-button color="primary" (click)="loadTableData()">Refresh</button>
+
+        <table mat-table fixedLayout="true" [dataSource]="table_data" class="data-table" matSort matSortActive="id"
+            matSortDisableClear matSortDirection="desc">
+
+            <ng-container matColumnDef="id">
+                <th mat-header-cell *matHeaderCellDef>ID</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.id}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="alias">
+                <th mat-header-cell *matHeaderCellDef>Alias</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.alias}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="username">
+                <th mat-header-cell *matHeaderCellDef>User</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.username}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="hostname">
+                <th mat-header-cell *matHeaderCellDef>Hostname</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.hostname}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="last_active">
+                <th mat-header-cell *matHeaderCellDef>Last active</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.last_active.secs_since_epoch * 1000 | date:'long'}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="actions">
+                <th mat-header-cell *matHeaderCellDef></th>
+                <td mat-cell *matCellDef="let row">
+                    <button mat-icon-button (click)="assignJobs(row.id)">
+                        <mat-icon>add_task</mat-icon>
+                    </button>
+                    |
+                    <button mat-icon-button routerLink='.' [queryParams]="{id: row.id}">
+                        <mat-icon>more_horiz</mat-icon>
+                    </button>
+                    |
+                    <button mat-icon-button (click)="deleteItem(row.id)">
+                        <mat-icon>delete</mat-icon>
+                    </button>
+                </td>
+            </ng-container>
+
+            <tr mat-header-row *matHeaderRowDef="displayedColumns"></tr>
+            <tr mat-row class="data-table-row" *matRowDef="let row; columns: displayedColumns;"></tr>
+            <tr class="mat-row" *matNoDataRow>
+                <td class="mat-cell">No data</td>
+            </tr>
+        </table>
+    </div>
+
+    <!-- <mat-paginator [length]="resultsLength" [pageSize]="30" aria-label="Select page of GitHub search results">
+    </mat-paginator> -->
+</div>

bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts

@@ -0,0 +1,42 @@
+import { Component, OnInit } from '@angular/core';
+import { TableComponent } from '../base-table/base-table.component';
+import { AgentModel, Area } from '../../../models';
+import { AssignJobDialogComponent, AgentInfoDialogComponent } from '../../dialogs';
+
+@Component({
+  selector: 'agent-table',
+  templateUrl: './agent-table.component.html',
+  styleUrls: ['../base-table/base-table.component.less'],
+})
+export class AgentComponent extends TableComponent<AgentModel> implements OnInit {
+  area = 'agents' as Area
+  displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions']
+
+  showItemDialog(id: string) {
+    this.dataSource.getAgent(id).subscribe(resp => {
+      const dialog = this.infoDialog.open(AgentInfoDialogComponent, {
+        data: resp,
+        width: '1000px',
+      });
+
+      const saveSub = dialog.componentInstance.onSave.subscribe(result => {
+        this.dataSource.updateAgent(result).subscribe(_ => {
+          alert('Saved')
+          this.loadTableData()
+        })
+      })
+
+      dialog.afterClosed().subscribe(result => {
+        saveSub.unsubscribe()
+        this.router.navigate(['.'], { relativeTo: this.route })
+      })
+    })
+  }
+
+  assignJobs(id: string) {
+    const dialog = this.infoDialog.open(AssignJobDialogComponent, {
+      data: id,
+      width: '1000px',
+    });
+  }
+}

bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.less

@@ -0,0 +1,32 @@
+.data-table {
+    width: 100%;
+}
+
+.table-container {
+    margin: 50px;
+}
+
+.loading-shade {
+    position: absolute;
+    top: 0;
+    left: 0;
+    bottom: 56px;
+    right: 0;
+    //background: rgba(0, 0, 0, 0.15);
+    z-index: 1;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+}
+
+#refresh_btn {
+    margin-left: 10px;
+}
+
+.data-table-row {
+    height: 30px;
+}
+
+.data-table-row:hover {
+    background: whitesmoke;
+}

bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts

@@ -0,0 +1,59 @@
+import { OnInit, Directive, Component } from '@angular/core';
+import { ApiTableService } from '../../..';
+import { MatTableDataSource } from '@angular/material/table';
+import { MatDialog } from '@angular/material/dialog';
+import { ApiModel, Area } from '../../../models';
+import { ActivatedRoute, Router } from '@angular/router';
+
+@Directive()
+export abstract class TableComponent<T extends ApiModel> implements OnInit {
+  abstract area: Area;
+  table_data: MatTableDataSource<T> = new MatTableDataSource;
+  isLoadingResults = true;
+
+  constructor(
+    public dataSource: ApiTableService,
+    public infoDialog: MatDialog,
+    public route: ActivatedRoute,
+    public router: Router,
+  ) { }
+
+  ngOnInit() {
+    this.loadTableData();
+    this.route.queryParams.subscribe(params => {
+      const id = params['id']
+      const new_item = params['new']
+      if (id) {
+        this.showItemDialog(id);
+      }
+      if (new_item) {
+        this.showItemDialog(null);
+      }
+    })
+    //interval(10000).subscribe(_ => this.loadTableData());
+  }
+
+  loadTableData() {
+    this.isLoadingResults = true;
+
+    this.dataSource.getMany(this.area).subscribe(resp => {
+      this.isLoadingResults = false;
+      this.table_data.data = resp;
+    })
+  }
+
+  applyFilter(event: Event) {
+    const filterValue = (event.target as HTMLInputElement).value;
+    this.table_data.filter = filterValue.trim().toLowerCase();
+  }
+
+  deleteItem(id: string) {
+    if (confirm(`Delete ${id}?`)) {
+      this.dataSource.delete(id, this.area).subscribe(_ => { })
+      this.loadTableData()
+    }
+  }
+
+  abstract displayedColumns: string[];
+  abstract showItemDialog(id: string | null): void;
+}

bin/u_panel/src/gui/fe/src/app/components/tables/index.ts

@@ -0,0 +1,5 @@
+export * from './agent-table/agent-table.component';
+export * from './base-table/base-table.component';
+export * from './job-table/job-table.component';
+export * from './payload-table/payload-table.component';
+export * from './result-table/result-table.component';

bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.html

@@ -0,0 +1,83 @@
+<div class="mat-elevation-z8">
+
+    <div class="table-container">
+        <div class="loading-shade" *ngIf="isLoadingResults">
+            <mat-spinner *ngIf="isLoadingResults"></mat-spinner>
+        </div>
+        <mat-form-field appearance="standard">
+            <mat-label>Filter</mat-label>
+            <input matInput (keyup)="applyFilter($event)" #input>
+        </mat-form-field>
+        <button id="refresh_btn" mat-raised-button color="basic" (click)="loadTableData()">Refresh</button>
+        <button id="new_btn" mat-raised-button color="primary" routerLink='.' [queryParams]="{new: true}">Add
+            job</button>
+
+        <table mat-table fixedLayout="true" [dataSource]="table_data" class="data-table" matSort matSortActive="id"
+            matSortDisableClear matSortDirection="desc">
+
+            <ng-container matColumnDef="id">
+                <th mat-header-cell *matHeaderCellDef>ID</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.id}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="alias">
+                <th mat-header-cell *matHeaderCellDef>Alias</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.alias}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="argv">
+                <th mat-header-cell *matHeaderCellDef>Cmd-line args</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.argv}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="platform">
+                <th mat-header-cell *matHeaderCellDef>Platform</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.target_platforms}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="schedule">
+                <th mat-header-cell *matHeaderCellDef>Schedule</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.schedule}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="exec_type">
+                <th mat-header-cell *matHeaderCellDef>Type</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.exec_type}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="actions">
+                <th mat-header-cell *matHeaderCellDef></th>
+                <td mat-cell *matCellDef="let row">
+                    <button mat-icon-button routerLink='.' [queryParams]="{id: row.id}">
+                        <mat-icon>more_horiz</mat-icon>
+                    </button>
+                    |
+                    <button mat-icon-button (click)="deleteItem(row.id)">
+                        <mat-icon>delete</mat-icon>
+                    </button>
+                </td>
+            </ng-container>
+
+            <tr mat-header-row *matHeaderRowDef="displayedColumns"></tr>
+            <tr mat-row class="data-table-row" *matRowDef="let row; columns: displayedColumns;"></tr>
+            <tr class="mat-row" *matNoDataRow>
+                <td class="mat-cell">No data</td>
+            </tr>
+        </table>
+    </div>
+
+    <!-- <mat-paginator [length]="resultsLength" [pageSize]="30" aria-label="Select page of GitHub search results">
+    </mat-paginator> -->
+</div>

bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.ts

@@ -0,0 +1,72 @@
+import { Component, OnInit } from '@angular/core';
+import { TableComponent } from '../base-table/base-table.component';
+import { Area, JobModel, Job } from '../../../models';
+import { JobInfoDialogComponent } from '../../dialogs';
+import { Observable } from 'rxjs';
+
+@Component({
+  selector: 'job-table',
+  templateUrl: './job-table.component.html',
+  styleUrls: ['../base-table/base-table.component.less'],
+  providers: [{ provide: 'area', useValue: 'jobs' }]
+})
+export class JobComponent extends TableComponent<JobModel> {
+  area = 'jobs' as Area;
+  displayedColumns = ['id', 'alias', 'platform', 'schedule', 'exec_type', 'actions']
+
+  showItemDialog(id: string | null) {
+    const is_new_job = id === null;
+
+    var dialogData$: Observable<Job>;
+
+    if (is_new_job) {
+      dialogData$ = new Observable(subscriber => {
+        var defaultJob: Job = {
+          meta: {
+            alias: null,
+            argv: '',
+            exec_type: 'shell',
+            target_platforms: '*',
+            payload_id: null,
+            schedule: null
+          },
+          payload: null
+        };
+        subscriber.next(defaultJob)
+      })
+    } else {
+      dialogData$ = this.dataSource.getJob(id)
+    }
+
+    dialogData$.subscribe(dialogData => {
+      const dialog = this.infoDialog.open(JobInfoDialogComponent, {
+        data: dialogData,
+        width: '1000px',
+      });
+
+      dialog.componentInstance.isPreview = !is_new_job;
+
+      const saveSub = dialog.componentInstance.onSave.subscribe(result => {
+        if (is_new_job) {
+          this.dataSource.create(dialogData.meta, this.area)
+            .subscribe(_ => {
+              alert("Created")
+              this.loadTableData()
+            })
+        } else {
+          this.dataSource.updateJob(result)
+            .subscribe(_ => {
+              alert("Updated")
+              this.loadTableData()
+            })
+        }
+        dialog.close()
+      })
+
+      dialog.afterClosed().subscribe(result => {
+        saveSub.unsubscribe()
+        this.router.navigate(['.'], { relativeTo: this.route })
+      })
+    })
+  }
+}

bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.html

@@ -0,0 +1,63 @@
+<div class="mat-elevation-z8">
+
+    <div class="table-container">
+        <div class="loading-shade" *ngIf="isLoadingResults">
+            <mat-spinner *ngIf="isLoadingResults"></mat-spinner>
+        </div>
+        <mat-form-field appearance="standard">
+            <mat-label>Filter</mat-label>
+            <input matInput (keyup)="applyFilter($event)" #input>
+        </mat-form-field>
+        <button id="refresh_btn" mat-raised-button color="primary" (click)="loadTableData()">Refresh</button>
+        <button id="new_btn" mat-raised-button color="primary" routerLink='.' [queryParams]="{new: true}">Add
+            payload</button>
+
+
+        <table mat-table fixedLayout="true" [dataSource]="table_data" class="data-table" matSort matSortActive="id"
+            matSortDisableClear matSortDirection="desc">
+
+            <ng-container matColumnDef="name">
+                <th mat-header-cell *matHeaderCellDef>Name</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.name}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="mime_type">
+                <th mat-header-cell *matHeaderCellDef>MIME-type</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.mime_type}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="size">
+                <th mat-header-cell *matHeaderCellDef>Size</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.size}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="actions">
+                <th mat-header-cell *matHeaderCellDef></th>
+                <td mat-cell *matCellDef="let row">
+                    <button mat-icon-button routerLink='.' [queryParams]="{id: row.id}">
+                        <mat-icon>more_horiz</mat-icon>
+                    </button>
+                    |
+                    <button mat-icon-button (click)="deleteItem(row.id)">
+                        <mat-icon>delete</mat-icon>
+                    </button>
+                </td>
+            </ng-container>
+
+            <tr mat-header-row *matHeaderRowDef="displayedColumns"></tr>
+            <tr mat-row class="data-table-row" *matRowDef="let row; columns: displayedColumns;"></tr>
+            <tr class="mat-row" *matNoDataRow>
+                <td class="mat-cell">No data</td>
+            </tr>
+        </table>
+    </div>
+
+    <!-- <mat-paginator [length]="resultsLength" [pageSize]="30" aria-label="Select page of GitHub search results">
+    </mat-paginator> -->
+</div>

bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts

@@ -0,0 +1,67 @@
+import { Component } from '@angular/core';
+import { Area } from 'src/app/models';
+import { NewPayloadModel, PayloadModel } from 'src/app/models/payload.model';
+import { PayloadInfoDialogComponent } from '../../dialogs';
+import { NewPayloadDialogComponent } from '../../dialogs/new-payload-dialog/new-payload-dialog.component';
+import { TableComponent } from '../base-table/base-table.component';
+
+@Component({
+  selector: 'payload-table',
+  templateUrl: './payload-table.component.html',
+  styleUrls: ['../base-table/base-table.component.less'],
+  providers: [{ provide: 'area', useValue: 'payloads' }]
+})
+export class PayloadComponent extends TableComponent<PayloadModel> {
+  area = 'payloads' as Area
+  displayedColumns = ["name", "mime_type", "size", 'actions'];
+
+  showItemDialog(id: string | null) {
+    if (id === null) {
+      const payload: NewPayloadModel = {
+        name: "",
+        data: []
+      }
+      const dialog = this.infoDialog.open(NewPayloadDialogComponent, {
+        data: payload,
+        width: '1000px',
+      });
+
+      dialog.componentInstance.onSave.subscribe(result => {
+        this.dataSource.createPayload(result)
+          .subscribe(_ => {
+            alert("Created")
+            this.loadTableData()
+          })
+
+        dialog.close()
+      })
+
+      dialog.afterClosed().subscribe(_ => {
+        this.router.navigate(['.'], { relativeTo: this.route })
+      })
+
+    } else {
+      this.dataSource.getPayload(id as string).subscribe(resp => {
+        const dialog = this.infoDialog.open(PayloadInfoDialogComponent, {
+          data: resp,
+          width: '1000px',
+        });
+
+        const saveSub = dialog.componentInstance.onSave.subscribe(result => {
+          this.dataSource.updatePayload(result)
+            .subscribe(_ => {
+              alert("Updated")
+              this.loadTableData()
+            })
+          dialog.close()
+        })
+
+        dialog.afterClosed().subscribe(_ => {
+          saveSub.unsubscribe()
+          this.router.navigate(['.'], { relativeTo: this.route })
+        })
+      })
+    }
+  }
+
+}

bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.html

@@ -0,0 +1,81 @@
+<div class="mat-elevation-z8">
+
+    <div class="table-container">
+        <div class="loading-shade" *ngIf="isLoadingResults">
+            <mat-spinner *ngIf="isLoadingResults"></mat-spinner>
+        </div>
+        <mat-form-field appearance="standard">
+            <mat-label>Filter</mat-label>
+            <input matInput (keyup)="applyFilter($event)" #input>
+        </mat-form-field>
+        <button id="refresh_btn" mat-raised-button color="primary" (click)="loadTableData()">Refresh</button>
+
+        <table mat-table fixedLayout="true" [dataSource]="table_data" class="data-table" matSort matSortActive="id"
+            matSortDisableClear matSortDirection="desc">
+
+            <ng-container matColumnDef="id">
+                <th mat-header-cell *matHeaderCellDef>ID</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.id}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="alias">
+                <th mat-header-cell *matHeaderCellDef>Alias</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.alias}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="agent_id">
+                <th mat-header-cell *matHeaderCellDef>Agent</th>
+                <td mat-cell *matCellDef="let row">
+                    <a routerLink='/agents' [queryParams]="{id: row.agent_id}">{{row.agent_id}}</a>
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="job_id">
+                <th mat-header-cell *matHeaderCellDef>Job</th>
+                <td mat-cell *matCellDef="let row">
+                    <a routerLink='/jobs' [queryParams]="{id: row.job_id}">{{row.job_id}}</a>
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="state">
+                <th mat-header-cell *matHeaderCellDef>State</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.state}} {{(row.state === "Finished") ? '(' + row.retcode + ')' : ''}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="last_updated">
+                <th mat-header-cell *matHeaderCellDef>Last updated</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.updated.secs_since_epoch * 1000| date:'long'}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="actions">
+                <th mat-header-cell *matHeaderCellDef></th>
+                <td mat-cell *matCellDef="let row">
+                    <button mat-icon-button routerLink='.' [queryParams]="{id: row.id}">
+                        <mat-icon>more_horiz</mat-icon>
+                    </button>
+                    |
+                    <button mat-icon-button (click)="deleteItem(row.id)">
+                        <mat-icon>delete</mat-icon>
+                    </button>
+                </td>
+            </ng-container>
+
+            <tr mat-header-row *matHeaderRowDef="displayedColumns"></tr>
+            <tr mat-row class="data-table-row" *matRowDef="let row; columns: displayedColumns;"></tr>
+            <tr class="mat-row" *matNoDataRow>
+                <td class="mat-cell">No data</td>
+            </tr>
+        </table>
+    </div>
+
+    <!-- <mat-paginator [length]="resultsLength" [pageSize]="30" aria-label="Select page of GitHub search results">
+    </mat-paginator> -->
+</div>

bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts

@@ -0,0 +1,36 @@
+import { Component, OnInit } from '@angular/core';
+import { TableComponent } from '../base-table/base-table.component';
+import { Area, ResultModel } from '../../../models';
+import { ResultInfoDialogComponent } from '../../dialogs';
+
+@Component({
+  selector: 'results-table',
+  templateUrl: './result-table.component.html',
+  styleUrls: ['../base-table/base-table.component.less'],
+  providers: [{ provide: 'area', useValue: 'map' }]
+})
+export class ResultComponent extends TableComponent<ResultModel> {
+  area = 'map' as Area
+  displayedColumns = [
+    'id',
+    'alias',
+    'agent_id',
+    'job_id',
+    'state',
+    'last_updated',
+    'actions'
+  ];
+
+  showItemDialog(id: string) {
+    this.dataSource.getResult(id).subscribe(resp => {
+      const dialog = this.infoDialog.open(ResultInfoDialogComponent, {
+        data: resp,
+        width: '1000px',
+      });
+
+      dialog.afterClosed().subscribe(_ => {
+        this.router.navigate(['.'], { relativeTo: this.route })
+      })
+    })
+  }
+}

bin/u_panel/src/gui/fe/src/app/index.ts

@@ -0,0 +1 @@
+export * from './services'; 

bin/u_panel/src/gui/fe/src/app/models/agent.model.ts

@@ -0,0 +1,16 @@
+import { UTCDate } from ".";
+
+export interface AgentModel {
+    alias: string | null,
+    hostname: string,
+    host_info: string,
+    id: string,
+    is_root: boolean,
+    is_root_allowed: boolean,
+    last_active: UTCDate,
+    platform: string,
+    regtime: UTCDate,
+    state: "new" | "active" | "banned",
+    token: string | null,
+    username: string,
+}

bin/u_panel/src/gui/fe/src/app/models/index.ts

@@ -0,0 +1,20 @@
+import { AgentModel } from './agent.model';
+import { JobModel } from './job.model';
+import { PayloadModel } from './payload.model';
+import { ResultModel } from './result.model';
+
+export * from './agent.model';
+export * from './result.model';
+export * from './job.model';
+export * from './payload.model';
+
+export interface UTCDate {
+    secs_since_epoch: number,
+    nanos_since_epoch: number
+}
+
+export type Area = "agents" | "jobs" | "map" | "payloads";
+
+export type ApiModel = AgentModel | JobModel | ResultModel | PayloadModel | Empty;
+
+export interface Empty { }

bin/u_panel/src/gui/fe/src/app/models/job.model.ts

@@ -0,0 +1,16 @@
+import { PayloadModel } from './'
+
+export interface JobModel {
+    alias: string | null,
+    argv: string,
+    id?: string,
+    exec_type: string,
+    target_platforms: string,
+    payload_id: string | null,
+    schedule: string | null,
+}
+
+export interface Job {
+    meta: JobModel,
+    payload: PayloadModel | null,
+}

bin/u_panel/src/gui/fe/src/app/models/payload.model.ts

@@ -0,0 +1,12 @@
+export interface PayloadModel {
+    id: string,
+    mime_type: string,
+    name: string,
+    size: number,
+    data: number[] | null
+}
+
+export interface NewPayloadModel {
+    name: string,
+    data: number[]
+}

bin/u_panel/src/gui/fe/src/app/models/result.model.ts

@@ -0,0 +1,18 @@
+import { UTCDate } from ".";
+
+export interface ResultModel {
+    agent_id: string,
+    alias: string,
+    created: UTCDate,
+    id: string,
+    job_id: string,
+    result: number[] | null,
+    state: "Queued" | "Running" | "Finished",
+    retcode: number | null,
+    updated: UTCDate,
+}
+
+export interface AssignedJobByIdModel {
+    job_id: string,
+    agent_id: string
+}

bin/u_panel/src/gui/fe/src/app/services/api.service.ts

@@ -0,0 +1,142 @@
+import { environment } from 'src/environments/environment';
+import { HttpClient, HttpErrorResponse } from '@angular/common/http';
+import { Observable, map, catchError, throwError } from 'rxjs';
+import { ApiModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, Job, NewPayloadModel, AssignedJobByIdModel } from '../models';
+import { Injectable, Inject } from '@angular/core';
+import { ErrorService } from './error.service';
+
+type Status = "ok" | "err";
+
+interface ServerResponse<T extends ApiModel> {
+  status: Status,
+  data: T | string
+}
+
+@Injectable({
+  providedIn: 'root'
+})
+export class ApiTableService {
+
+  constructor(
+    private http: HttpClient,
+    private errorService: ErrorService
+  ) {
+  }
+
+  requestUrl = `${environment.server}/cmd/`;
+
+  req<R extends ApiModel>(cmd: string): Observable<ServerResponse<R>> {
+    return this.http.post<ServerResponse<R>>(this.requestUrl, cmd)
+  }
+
+  getOne<T extends ApiModel>(id: string, area: Area, brief: 'yes' | 'no' | 'auto' | null = null): Observable<T> {
+    const request = `${area} read ${id}` + (brief !== null ? `-b=${brief}` : '')
+    const resp = this.req<T[]>(request).pipe(
+      map(resp => {
+        if (resp.data.length === 0) {
+          return {
+            status: 'err' as Status,
+            data: `${id} not found in ${area}`
+          }
+        }
+        return {
+          status: resp.status,
+          data: resp.data[0]
+        }
+      }));
+    return this.filterErrStatus(resp)
+  }
+
+  getAgent(id: string): Observable<AgentModel> {
+    return this.getOne(id, 'agents')
+  }
+
+  getJob(id: string): Observable<Job> {
+    return this.getOne(id, 'jobs')
+  }
+
+  getResult(id: string): Observable<ResultModel> {
+    return this.getOne(id, 'map')
+  }
+
+  getPayload(id: string): Observable<PayloadModel> {
+    return this.getOne(id, 'payloads')
+  }
+
+  getMany(area: Area): Observable<any[]> {
+    return this.filterErrStatus(this.req(`${area} read`))
+  }
+
+  getAgents(): Observable<AgentModel[]> {
+    return this.getMany('agents')
+  }
+
+  getJobs(): Observable<JobModel[]> {
+    return this.getMany('jobs')
+  }
+
+  getResults(): Observable<ResultModel[]> {
+    return this.getMany('map')
+  }
+
+  getPayloads(): Observable<PayloadModel[]> {
+    return this.getMany('payloads')
+  }
+
+  update<T extends ApiModel>(item: T, area: Area): Observable<Empty> {
+    return this.filterErrStatus(this.req(`${area} update '${JSON.stringify(item)}'`))
+  }
+
+  updateAgent(item: AgentModel): Observable<Empty> {
+    return this.update(item, 'agents')
+  }
+
+  updateJob(item: JobModel): Observable<Empty> {
+    return this.update(item, 'jobs')
+  }
+
+  updateResult(item: ResultModel): Observable<Empty> {
+    return this.update(item, 'map')
+  }
+
+  updatePayload(item: PayloadModel): Observable<Empty> {
+    return this.update(item, 'payloads')
+  }
+
+  delete(id: string, area: Area): Observable<Empty> {
+    return this.filterErrStatus(this.req(`${area} delete ${id}`))
+  }
+
+  create<T extends ApiModel>(item: T | null, area: Area): Observable<string[]> {
+    var serialized = '"{}"'
+    if (item) {
+      serialized = JSON.stringify(item);
+    }
+    return this.filterErrStatus(this.req(`${area} create '${serialized}'`))
+  }
+
+  createResult(item: AssignedJobByIdModel[]): Observable<string[]> {
+    return this.create(item, 'map')
+  }
+
+  createPayload(item: NewPayloadModel): Observable<string[]> {
+    return this.create(item, 'payloads')
+  }
+
+  filterErrStatus<R extends ApiModel>(obs: Observable<ServerResponse<R>>): Observable<R> {
+    return obs.pipe(
+      map(r => {
+        if (r.status == 'err') {
+          throw new Error(r.data as string)
+        }
+        return r.data as R
+      }),
+      catchError(this.errorHandler.bind(this)))
+  }
+
+  errorHandler(err: HttpErrorResponse, caught: any) {
+    var error = err.error.data !== undefined ? JSON.stringify(err.error.data) : err.message;
+    this.errorService.handle(error);
+    return throwError(() => new Error());
+  }
+}

bin/u_panel/src/gui/fe/src/app/services/error.service.ts

@@ -0,0 +1,17 @@
+import { Injectable } from '@angular/core';
+import { Subject } from 'rxjs';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class ErrorService {
+  error$ = new Subject<string>();
+
+  handle(msg: string) {
+    this.error$.next(msg)
+  }
+
+  clear() {
+    this.handle('')
+  }
+}

bin/u_panel/src/gui/fe/src/app/services/index.ts

@@ -0,0 +1 @@
+export * from './api.service'

bin/u_panel/src/gui/fe/src/app/utils.ts

@@ -0,0 +1,3 @@
+export function epochToStr(epoch: number): string {
+    return new Date(epoch * 1000).toLocaleString('en-GB')
+}

bin/u_panel/src/gui/fe/src/environments/environment.prod.ts

@@ -0,0 +1,4 @@
+export const environment = {
+  production: true,
+  server: "",
+};

bin/u_panel/src/gui/fe/src/environments/environment.ts

@@ -0,0 +1,17 @@
+// This file can be replaced during build by using the `fileReplacements` array.
+// `ng build` replaces `environment.ts` with `environment.prod.ts`.
+// The list of file replacements can be found in `angular.json`.
+
+export const environment = {
+  production: false,
+  server: "http://127.0.0.1:7799",
+};
+
+/*
+ * For easier debugging in development mode, you can import the following file
+ * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`.
+ *
+ * This import should be commented out in production mode because it will have a negative impact
+ * on performance if an error is thrown.
+ */
+// import 'zone.js/plugins/zone-error';  // Included with Angular CLI.

bin/u_panel/src/gui/fe/src/index.html

@@ -0,0 +1,18 @@
+<!doctype html>
+<html lang="en">
+
+<head>
+  <meta charset="utf-8">
+  <title>Fe</title>
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <link rel="icon" type="image/x-icon" href="favicon.ico">
+  <link rel="preconnect" href="https://fonts.gstatic.com">
+  <link href="https://fonts.googleapis.com/css2?family=Roboto+Mono:wght@300;400;500&display=swap" rel="stylesheet">
+  <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
+</head>
+
+<body class="mat-typography">
+  <app-root></app-root>
+</body>
+
+</html>

bin/u_panel/src/gui/fe/src/main.ts

@@ -0,0 +1,12 @@
+import { enableProdMode } from '@angular/core';
+import { platformBrowserDynamic } from '@angular/platform-browser-dynamic';
+
+import { AppModule } from './app/app.module';
+import { environment } from './environments/environment';
+
+if (environment.production) {
+  enableProdMode();
+}
+
+platformBrowserDynamic().bootstrapModule(AppModule)
+  .catch(err => console.error(err));

bin/u_panel/src/gui/fe/src/polyfills.ts

@@ -0,0 +1,53 @@
+/**
+ * This file includes polyfills needed by Angular and is loaded before the app.
+ * You can add your own extra polyfills to this file.
+ *
+ * This file is divided into 2 sections:
+ *   1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers.
+ *   2. Application imports. Files imported after ZoneJS that should be loaded before your main
+ *      file.
+ *
+ * The current setup is for so-called "evergreen" browsers; the last versions of browsers that
+ * automatically update themselves. This includes recent versions of Safari, Chrome (including
+ * Opera), Edge on the desktop, and iOS and Chrome on mobile.
+ *
+ * Learn more in https://angular.io/guide/browser-support
+ */
+
+/***************************************************************************************************
+ * BROWSER POLYFILLS
+ */
+
+/**
+ * By default, zone.js will patch all possible macroTask and DomEvents
+ * user can disable parts of macroTask/DomEvents patch by setting following flags
+ * because those flags need to be set before `zone.js` being loaded, and webpack
+ * will put import in the top of bundle, so user need to create a separate file
+ * in this directory (for example: zone-flags.ts), and put the following flags
+ * into that file, and then add the following code before importing zone.js.
+ * import './zone-flags';
+ *
+ * The flags allowed in zone-flags.ts are listed here.
+ *
+ * The following flags will work for all browsers.
+ *
+ * (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame
+ * (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick
+ * (window as any).__zone_symbol__UNPATCHED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames
+ *
+ *  in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js
+ *  with the following flag, it will bypass `zone.js` patch for IE/Edge
+ *
+ *  (window as any).__Zone_enable_cross_context_check = true;
+ *
+ */
+
+/***************************************************************************************************
+ * Zone JS is required by default for Angular itself.
+ */
+import 'zone.js';  // Included with Angular CLI.
+
+
+/***************************************************************************************************
+ * APPLICATION IMPORTS
+ */

bin/u_panel/src/gui/fe/src/styles.less

@@ -0,0 +1,4 @@
+/* You can add global styles to this file, and also import other style files */
+
+html, body { height: 100%; }
+body { margin: 0; font-family: Roboto, "Helvetica Neue", sans-serif; }

bin/u_panel/src/gui/fe/src/test.ts

@@ -0,0 +1,26 @@
+// This file is required by karma.conf.js and loads recursively all the .spec and framework files
+
+import 'zone.js/testing';
+import { getTestBed } from '@angular/core/testing';
+import {
+  BrowserDynamicTestingModule,
+  platformBrowserDynamicTesting
+} from '@angular/platform-browser-dynamic/testing';
+
+declare const require: {
+  context(path: string, deep?: boolean, filter?: RegExp): {
+    <T>(id: string): T;
+    keys(): string[];
+  };
+};
+
+// First, initialize the Angular testing environment.
+getTestBed().initTestEnvironment(
+  BrowserDynamicTestingModule,
+  platformBrowserDynamicTesting(),
+);
+
+// Then we find all the tests.
+const context = require.context('./', true, /\.spec\.ts$/);
+// And load the modules.
+context.keys().map(context);

bin/u_panel/src/gui/fe/tsconfig.app.json

@@ -0,0 +1,15 @@
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./out-tsc/app",
+    "types": []
+  },
+  "files": [
+    "src/main.ts",
+    "src/polyfills.ts"
+  ],
+  "include": [
+    "src/**/*.d.ts"
+  ]
+}

bin/u_panel/src/gui/fe/tsconfig.json

@@ -0,0 +1,32 @@
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+  "compileOnSave": false,
+  "compilerOptions": {
+    "baseUrl": "./",
+    "outDir": "./dist/out-tsc",
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "noImplicitOverride": true,
+    "noPropertyAccessFromIndexSignature": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "sourceMap": true,
+    "declaration": false,
+    "downlevelIteration": true,
+    "experimentalDecorators": true,
+    "moduleResolution": "node",
+    "importHelpers": true,
+    "target": "es2017",
+    "module": "es2020",
+    "lib": [
+      "es2020",
+      "dom"
+    ]
+  },
+  "angularCompilerOptions": {
+    "enableI18nLegacyMessageIdFormat": false,
+    "strictInjectionParameters": true,
+    "strictInputAccessModifiers": true,
+    "strictTemplates": true
+  }
+}

bin/u_panel/src/gui/fe/tsconfig.spec.json

@@ -0,0 +1,18 @@
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./out-tsc/spec",
+    "types": [
+      "jasmine"
+    ]
+  },
+  "files": [
+    "src/test.ts",
+    "src/polyfills.ts"
+  ],
+  "include": [
+    "src/**/*.spec.ts",
+    "src/**/*.d.ts"
+  ]
+}

bin/u_panel/src/gui/mod.rs

@@ -0,0 +1,96 @@
+mod error;
+
+use crate::{process_cmd, Args};
+use actix_cors::Cors;
+use actix_web::{get, middleware::Logger, post, web, App, HttpResponse, HttpServer, Responder};
+use error::Error;
+use futures_util::StreamExt;
+use rust_embed::RustEmbed;
+use std::borrow::Cow;
+use structopt::StructOpt;
+use u_lib::{api::HttpClient, unwrap_enum};
+
+#[derive(RustEmbed)]
+#[folder = "./src/gui/fe/dist/fe/"]
+struct Files;
+
+impl Files {
+    pub fn get_static(path: impl AsRef<str>) -> Option<&'static [u8]> {
+        let file = Self::get(path.as_ref())?.data;
+        Some(unwrap_enum!(file, Cow::Borrowed))
+    }
+}
+
+async fn spa_main() -> impl Responder {
+    let index = Files::get_static("index.html").unwrap();
+    HttpResponse::Ok().body(index)
+}
+
+#[get("/core/{path}")]
+async fn resources_adapter(path: web::Path<(String,)>) -> impl Responder {
+    let path = path.into_inner().0;
+    let mimetype = mime_guess::from_path(&path).first_or_octet_stream();
+
+    match Files::get_static(path) {
+        Some(data) => HttpResponse::Ok()
+            .content_type(mimetype.to_string())
+            .body(data),
+        None => HttpResponse::NotFound().finish(),
+    }
+}
+
+#[post("/cmd/")]
+async fn send_cmd(
+    mut body: web::Payload,
+    client: web::Data<HttpClient>,
+) -> Result<impl Responder, Error> {
+    let mut bytes = web::BytesMut::new();
+
+    while let Some(item) = body.next().await {
+        bytes.extend_from_slice(
+            &item.map_err(|e| Error::JustError(format!("payload loading failure: {e}")))?,
+        );
+    }
+
+    let cmd = String::from_utf8(bytes.to_vec())
+        .map_err(|_| Error::JustError("cmd contains non-utf8 data".to_string()))?;
+    let mut cmd = shlex::split(&cmd).ok_or(Error::JustError("argparse failed".to_string()))?;
+
+    info!("cmd: {:?}", cmd);
+    cmd.insert(0, String::from("u_panel"));
+
+    let parsed_cmd = Args::from_iter_safe(cmd)?;
+    let result = process_cmd(client.as_ref().clone(), parsed_cmd).await;
+    let result_string = result.to_string();
+
+    let response = if result.is_ok() {
+        HttpResponse::Ok().body(result_string)
+    } else {
+        HttpResponse::BadRequest().body(result_string)
+    };
+
+    Ok(response)
+}
+
+pub async fn serve(client: HttpClient) -> anyhow::Result<()> {
+    info!("Connecting to u_server...");
+    client.ping().await?;
+
+    let addr = "127.0.0.1:7799";
+    info!("Connected, instanciating u_panel at http://{}", addr);
+
+    HttpServer::new(move || {
+        App::new()
+            .wrap(Logger::default())
+            .wrap(Cors::permissive())
+            .app_data(web::Data::new(client.clone()))
+            .service(send_cmd)
+            .service(resources_adapter)
+            .service(web::resource("/").to(spa_main))
+            .service(web::resource("/{_}").to(spa_main))
+    })
+    .bind(addr)?
+    .run()
+    .await?;
+    Ok(())
+}

bin/u_panel/src/main.rs

@@ -1,143 +1,25 @@
-use std::env;
+mod argparse;
-use std::fmt;
+mod gui;
-use structopt::StructOpt;
-use u_lib::{
-    api::ClientHandler, datatypes::DataResult, messaging::AsMsg, models::JobMeta, utils::init_env,
-    UResult,
-};
-use uuid::Uuid;
-
-#[derive(StructOpt, Debug)]
-struct Args {
-    #[structopt(subcommand)]
-    cmd: Cmd,
-    #[structopt(long)]
-    json: bool,
-}
-
-#[derive(StructOpt, Debug)]
-enum Cmd {
-    Agents(LD),
-    Jobs(JobALD),
-    Jobmap(JobMapALD),
-}
-
-#[derive(StructOpt, Debug)]
-enum JobALD {
-    Add {
-        #[structopt(long, parse(try_from_str = parse_uuid))]
-        agent: Option<Uuid>,
-
-        #[structopt(long)]
-        alias: String,
-
-        #[structopt(subcommand)]
-        cmd: JobCmd,
-    },
-    #[structopt(flatten)]
-    LD(LD),
-}
-
-#[derive(StructOpt, Debug)]
-enum JobCmd {
-    #[structopt(external_subcommand)]
-    Cmd(Vec<String>),
-}
-
-#[derive(StructOpt, Debug)]
-enum JobMapALD {
-    Add {
-        #[structopt(parse(try_from_str = parse_uuid))]
-        agent_uid: Uuid,
-
-        job_idents: Vec<String>,
-    },
-    List {
-        #[structopt(parse(try_from_str = parse_uuid))]
-        uid: Option<Uuid>,
-    },
-    Delete {
-        #[structopt(parse(try_from_str = parse_uuid))]
-        uid: Uuid,
-    },
-}
 
-#[derive(StructOpt, Debug)]
+#[macro_use]
-enum LD {
+extern crate tracing;
-    List {
-        #[structopt(parse(try_from_str = parse_uuid))]
-        uid: Option<Uuid>,
-    },
-    Delete {
-        #[structopt(parse(try_from_str = parse_uuid))]
-        uid: Uuid,
-    },
-}
 
-fn parse_uuid(src: &str) -> Result<Uuid, String> {
+use anyhow::Result as AnyResult;
-    Uuid::parse_str(src).map_err(|e| e.to_string())
+use argparse::{process_cmd, Args};
-}
+use structopt::StructOpt;
+use u_lib::api::HttpClient;
+use u_lib::config::AccessEnv;
+use u_lib::logging::init_logger;
 
-async fn process_cmd(args: Args) {
+#[actix_web::main]
-    struct Printer {
+async fn main() -> AnyResult<()> {
-        json: bool,
+    init_logger(None);
-    }
 
-    impl Printer {
+    let env = AccessEnv::load()?;
-        pub fn print<Msg: AsMsg + fmt::Display>(&self, data: UResult<Msg>) {
+    let client = HttpClient::new(&env.u_server, Some(env.admin_auth_token)).await?;
-            if self.json {
+    let args = Args::from_args();
-                let data = match data {
+    let result = process_cmd(client, args).await.to_string();
-                    Ok(r) => DataResult::Ok(r),
-                    Err(e) => DataResult::Err(e),
-                };
-                println!("{}", serde_json::to_string_pretty(&data).unwrap());
-            } else {
-                match data {
-                    Ok(r) => println!("{}", r),
-                    Err(e) => eprintln!("Error: {}", e),
-                }
-            }
-        }
-    }
-
-    let token = env::var("ADMIN_AUTH_TOKEN").expect("Authentication token is not set");
-    let cli_handler = ClientHandler::new(None).password(token);
-    let printer = Printer { json: args.json };
-    match args.cmd {
-        Cmd::Agents(action) => match action {
-            LD::List { uid } => printer.print(cli_handler.get_agents(uid).await),
-            LD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await),
-        },
-        Cmd::Jobs(action) => match action {
-            JobALD::Add {
-                cmd: JobCmd::Cmd(cmd),
-                alias,
-                agent: _agent,
-            } => {
-                let job = JobMeta::builder()
-                    .with_shell(cmd.join(" "))
-                    .with_alias(alias)
-                    .build()
-                    .unwrap();
-                printer.print(cli_handler.upload_jobs(&[job]).await);
-            }
-            JobALD::LD(LD::List { uid }) => printer.print(cli_handler.get_jobs(uid).await),
-            JobALD::LD(LD::Delete { uid }) => printer.print(cli_handler.del(Some(uid)).await),
-        },
-        Cmd::Jobmap(action) => match action {
-            JobMapALD::Add {
-                agent_uid,
-                job_idents,
-            } => printer.print(cli_handler.set_jobs(Some(agent_uid), &job_idents).await),
-            JobMapALD::List { uid } => printer.print(cli_handler.get_agent_jobs(uid).await),
-            JobMapALD::Delete { uid } => printer.print(cli_handler.del(Some(uid)).await),
-        },
-    }
-}
 
-#[tokio::main]
+    println!("{result}");
-async fn main() {
+    Ok(())
-    init_env();
-    let args: Args = Args::from_args();
-    process_cmd(args).await;
 }

bin/u_run/Cargo.toml

@@ -7,5 +7,5 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-nix = "0.17"
 libc = "^0.2"
+nix = "0.17"

bin/u_server/Cargo.toml

@@ -1,39 +1,29 @@
 [package]
 authors = ["plazmoid <kronos44@mail.ru>"]
-edition = "2018"
+edition = "2021"
 name = "u_server"
 version = "0.1.0"
 
 [dependencies]
-log = "0.4.11"
+anyhow = { workspace = true }
-simplelog = "0.10"
+diesel = { workspace = true }
-thiserror = "*"
+deadpool-diesel = { workspace = true }
-warp = { version = "0.3.1", features = ["tls"] }
-uuid = { version = "0.6.5", features = ["serde", "v4"] }
-once_cell = "1.7.2"
 hyper = "0.14"
-mockall = "0.9.1"
+mime_guess = { workspace = true }
-mockall_double = "0.2"
+once_cell = "1.7.2"
-openssl = "*"
+openssl = { workspace = true }
-
+serde = { workspace = true }
-[dependencies.diesel]
+serde_json = { workspace = true }
-features = ["postgres", "uuid"]
+thiserror = { workspace = true }
-version = "1.4.5"
+tracing = { workspace = true }
-
+tokio = { workspace = true, features = ["macros"] }
-[dependencies.serde]
+uuid = { workspace = true, features = ["serde", "v4"] }
-features = ["derive"]
+u_lib = { path = "../../lib/u_lib", features = ["server"] }
-version = "1.0.114"
+warp = { version = "0.3.1", features = ["tls"] }
-
+serde_qs = { version = "0.12.0", features = ["warp"] }
-[dependencies.tokio]
-features = ["macros"]
-version = "1.9"
-
-[dependencies.u_lib]
-path = "../../lib/u_lib"
-version = "*"
 
 [dev-dependencies]
-test-case = "1.1.0"
+rstest = "0.12"
 
 [lib]
 name = "u_server_lib"

bin/u_server/src/db.rs

@@ -1,234 +1,377 @@
-use diesel::{pg::PgConnection, prelude::*, result::Error as DslError};
+use crate::error::Error;
-use once_cell::sync::OnceCell;
+use diesel::{pg::PgConnection, prelude::*, result::Error as DslError, Connection};
-use std::{
+use std::collections::{HashMap, HashSet};
-    env,
+use std::mem::drop;
-    sync::{Arc, Mutex, MutexGuard},
-};
 use u_lib::{
-    models::{schema, Agent, AssignedJob, JobMeta, JobState},
+    db::PgAsyncPool,
-    ULocalError, ULocalResult,
+    models::{schema, Agent, AssignedJob, AssignedJobById, Job, JobMeta, JobState, Payload},
+    platform::Platform,
+    types::Id,
 };
 use uuid::Uuid;
 
-pub struct UDB {
+type Result<T> = std::result::Result<T, Error>;
-    pub conn: PgConnection,
+
+pub struct PgRepo {
+    pool: PgAsyncPool,
 }
 
-static DB: OnceCell<Arc<Mutex<UDB>>> = OnceCell::new();
+impl PgRepo {
+    pub fn new(pool: PgAsyncPool) -> PgRepo {
+        PgRepo { pool }
+    }
 
-#[cfg_attr(test, automock)]
+    pub async fn interact<F, R>(&self, f: F) -> Result<R>
-impl UDB {
+    where
-    pub fn lock_db() -> MutexGuard<'static, UDB> {
+        F: for<'c> FnOnce(UDB<'c>) -> Result<R>,
-        DB.get_or_init(|| {
+        F: Send + 'static,
-            let _getenv = |v| env::var(v).unwrap();
+        R: Send + 'static,
-            let db_host = _getenv("DB_HOST");
+    {
-            let db_name = _getenv("DB_NAME");
+        let connection = self.pool.get().await?;
-            let db_user = _getenv("DB_USER");
+        connection
-            let db_password = _getenv("DB_PASSWORD");
+            .interact(|conn| f(UDB { conn }))
-            let db_url = format!(
+            .await
-                "postgres://{}:{}@{}/{}",
+            .expect("deadpool interaction failed")
-                db_user, db_password, db_host, db_name
-            );
-            let conn = PgConnection::establish(&db_url).unwrap();
-            let instance = UDB { conn };
-            Arc::new(Mutex::new(instance))
-        })
-        .lock()
-        .unwrap()
     }
 
-    pub fn insert_jobs(&self, job_metas: &[JobMeta]) -> ULocalResult<()> {
+    pub async fn transaction<F, R>(&self, f: F) -> Result<R>
+    where
+        F: for<'c> FnOnce(UDB<'c>) -> Result<R>,
+        F: Send + 'static,
+        R: Send + 'static,
+    {
+        let conn = self.pool.get().await?;
+        conn.interact(|c| c.transaction(|conn| f(UDB { conn })))
+            .await
+            .expect("deadpool interaction failed")
+    }
+}
+
+pub struct UDB<'c> {
+    conn: &'c mut PgConnection,
+}
+
+impl UDB<'_> {
+    pub fn insert_jobs(&mut self, jobs: &[JobMeta]) -> Result<()> {
         use schema::jobs;
+
         diesel::insert_into(jobs::table)
-            .values(job_metas)
+            .values(jobs)
-            .execute(&self.conn)?;
+            .execute(self.conn)
-        Ok(())
+            .map(drop)
+            .map_err(with_err_ctx("Can't insert jobs"))
+    }
+
+    pub fn insert_payload(&mut self, payload: &Payload) -> Result<()> {
+        use schema::payloads;
+
+        diesel::insert_into(payloads::table)
+            .values(payload)
+            .execute(self.conn)
+            .map(drop)
+            .map_err(with_err_ctx(format!("Can't insert payload {payload:?}")))
     }
 
-    pub fn get_jobs(&self, uid: Option<Uuid>) -> ULocalResult<Vec<JobMeta>> {
+    pub fn get_job(&mut self, id: Id) -> Result<Option<Job>> {
+        use schema::{jobs, payloads};
+
+        let maybe_job_with_payload = jobs::table
+            .left_join(payloads::table)
+            .filter(jobs::id.eq(id))
+            .first::<(JobMeta, Option<Payload>)>(self.conn)
+            .optional()
+            .map_err(with_err_ctx(format!("Can't get job {id}")))?;
+
+        Ok(maybe_job_with_payload.map(|(job, payload)| Job { meta: job, payload }))
+    }
+
+    pub fn get_jobs(&mut self) -> Result<Vec<JobMeta>> {
         use schema::jobs;
-        let result = if uid.is_some() {
+
         jobs::table
-                .filter(jobs::id.eq(uid.unwrap()))
+            .load(self.conn)
-                .get_results::<JobMeta>(&self.conn)?
+            .map_err(with_err_ctx("Can't get jobs"))
-        } else {
-            jobs::table.load::<JobMeta>(&self.conn)?
-        };
-        Ok(result)
     }
 
-    pub fn find_job_by_alias(&self, alias: &str) -> ULocalResult<JobMeta> {
+    pub fn get_payload(&mut self, id: Id) -> Result<Option<Payload>> {
-        use schema::jobs;
+        use schema::payloads;
-        let result = jobs::table
+
+        payloads::table
+            .filter(payloads::id.eq(id))
+            .first(self.conn)
+            .optional()
+            .map_err(with_err_ctx(format!("Can't get payload {id}")))
+    }
+
+    pub fn get_payload_by_name(&mut self, name: String) -> Result<Option<Payload>> {
+        use schema::payloads;
+
+        payloads::table
+            .filter(payloads::name.eq(&name))
+            .first(self.conn)
+            .optional()
+            .map_err(with_err_ctx(format!("Can't get payload by name {name}")))
+    }
+
+    pub fn get_payloads(&mut self) -> Result<Vec<Payload>> {
+        use schema::payloads;
+
+        payloads::table
+            .load(self.conn)
+            .map_err(with_err_ctx("Can't get payloads"))
+    }
+
+    pub fn payload_exists(&mut self, payload_id: Id) -> Result<bool> {
+        use schema::payloads;
+
+        payloads::table
+            .find(payload_id)
+            .first::<Payload>(self.conn)
+            .optional()
+            .map(|r| r.is_some())
+            .map_err(with_err_ctx("Can't check payload {payload_id}"))
+    }
+
+    pub fn get_job_by_alias(&mut self, alias: &str) -> Result<Option<Job>> {
+        use schema::{jobs, payloads};
+
+        let maybe_job_with_payload = jobs::table
+            .left_join(payloads::table)
             .filter(jobs::alias.eq(alias))
-            .first::<JobMeta>(&self.conn)?;
+            .first::<(JobMeta, Option<Payload>)>(self.conn)
-        Ok(result)
+            .optional()
+            .map_err(with_err_ctx(format!("Can't get job by alias {alias}")))?;
+
+        Ok(maybe_job_with_payload.map(|(job, payload_meta)| Job {
+            meta: job,
+            payload: payload_meta,
+        }))
     }
 
-    pub fn insert_agent(&self, agent: &Agent) -> ULocalResult<()> {
+    pub fn insert_result(&mut self, result: &AssignedJob) -> Result<()> {
-        use schema::agents;
+        use schema::results;
-        diesel::insert_into(agents::table)
+
-            .values(agent)
+        diesel::insert_into(results::table)
-            .on_conflict(agents::id)
+            .values(result)
-            .do_update()
+            .execute(self.conn)
-            .set(agent)
+            .map_err(with_err_ctx(format!("Can't insert result {result:?}")))?;
-            .execute(&self.conn)?;
         Ok(())
     }
 
-    pub fn get_agents(&self, uid: Option<Uuid>) -> ULocalResult<Vec<Agent>> {
+    pub fn get_agent(&mut self, id: Id) -> Result<Option<Agent>> {
         use schema::agents;
-        let result = if uid.is_some() {
+
         agents::table
-                .filter(agents::id.eq(uid.unwrap()))
+            .filter(agents::id.eq(id))
-                .load::<Agent>(&self.conn)?
+            .first(self.conn)
-        } else {
+            .optional()
-            agents::table.load::<Agent>(&self.conn)?
+            .map_err(with_err_ctx(format!("Can't get agent {id:?}")))
-        };
+    }
-        Ok(result)
+
+    pub fn get_agents(&mut self) -> Result<Vec<Agent>> {
+        use schema::agents;
+
+        agents::table
+            .load::<Agent>(self.conn)
+            .map_err(with_err_ctx(format!("Can't get agents")))
     }
 
-    pub fn update_job_status(&self, uid: Uuid, status: JobState) -> ULocalResult<()> {
+    pub fn update_job_status(&mut self, id: Id, status: JobState) -> Result<()> {
         use schema::results;
+
         diesel::update(results::table)
-            .filter(results::id.eq(uid))
+            .filter(results::id.eq(id))
             .set(results::state.eq(status))
-            .execute(&self.conn)?;
+            .execute(self.conn)
+            .map_err(with_err_ctx(format!("Can't update status of job {id}")))?;
         Ok(())
     }
 
     //TODO: filters possibly could work in a wrong way, check
-    pub fn get_exact_jobs(
+    pub fn get_assigned_jobs(
-        &self,
+        &mut self,
-        uid: Option<Uuid>,
+        id: Option<Id>,
         personal: bool,
-    ) -> ULocalResult<Vec<AssignedJob>> {
+    ) -> Result<Vec<AssignedJob>> {
         use schema::results;
+
         let mut q = results::table.into_boxed();
-        /*if uid.is_some() {
+        /*if id.is_some() {
-            q = q.filter(results::agent_id.eq(uid.unwrap()))
+            q = q.filter(results::agent_id.eq(id.unwrap()))
         }*/
         if personal {
             q = q.filter(
                 results::state
                     .eq(JobState::Queued)
-                    .and(results::agent_id.eq(uid.unwrap())),
+                    .and(results::agent_id.eq(id.unwrap())),
             )
-        } else if uid.is_some() {
+        } else if id.is_some() {
             q = q
-                .filter(results::agent_id.eq(uid.unwrap()))
+                .filter(results::agent_id.eq(id.unwrap()))
-                .or_filter(results::job_id.eq(uid.unwrap()))
+                .or_filter(results::job_id.eq(id.unwrap()))
-                .or_filter(results::id.eq(uid.unwrap()))
+                .or_filter(results::id.eq(id.unwrap()))
         }
-        let result = q.load::<AssignedJob>(&self.conn)?;
+        let result = q
+            .load::<AssignedJob>(self.conn)
+            .map_err(with_err_ctx("Can't get exact jobs"))?;
         Ok(result)
     }
 
-    pub fn set_jobs_for_agent(
+    // todo: move to handlers
-        &self,
+    pub fn assign_jobs(&mut self, assigned_jobs: &[AssignedJobById]) -> Result<()> {
-        agent_uid: &Uuid,
+        use schema::{jobs, results};
-        job_uids: &[Uuid],
+
-    ) -> ULocalResult<Vec<Uuid>> {
+        struct JobBriefMeta {
-        use schema::{agents::dsl::agents, jobs::dsl::jobs, results};
+            alias: Option<String>,
-        if let Err(DslError::NotFound) = agents.find(agent_uid).first::<Agent>(&self.conn) {
+            target_platform: String,
-            return Err(ULocalError::NotFound(agent_uid.to_string()));
+        }
+
+        let assigned_job_ids = HashSet::<Uuid>::from_iter(assigned_jobs.iter().map(|a| a.job_id));
+
+        let jobs_meta = HashMap::<Id, JobBriefMeta>::from_iter(
+            jobs::table
+                .select((jobs::id, jobs::alias, jobs::target_platforms))
+                .filter(jobs::id.eq_any(&assigned_job_ids))
+                .load::<(Id, Option<String>, String)>(self.conn)
+                .map_err(with_err_ctx(format!(
+                    "Can't find jobs {:?}",
+                    assigned_job_ids
+                )))?
+                .into_iter()
+                .map(|(id, alias, target_platform)| {
+                    (
+                        id,
+                        JobBriefMeta {
+                            alias,
+                            target_platform,
+                        },
+                    )
+                }),
+        );
+
+        let existing_job_ids = HashSet::from_iter(jobs_meta.keys().copied());
+
+        if assigned_job_ids != existing_job_ids {
+            return Err(Error::ProcessingError(format!(
+                "Jobs not found: {:?}",
+                assigned_job_ids.difference(&existing_job_ids),
+            )));
+        }
+
+        for ajob in assigned_jobs {
+            let meta = &jobs_meta[&ajob.job_id];
+            let agent_platform = match self.get_agent(ajob.agent_id)? {
+                Some(agent) => Platform::new(&agent.platform),
+                None => {
+                    return Err(Error::ProcessingError(format!(
+                        "Agent {} not found",
+                        ajob.agent_id
+                    )))
                 }
-        let not_found_jobs = job_uids
+            };
-            .iter()
+            if !agent_platform.matches(&meta.target_platform) {
-            .filter_map(|job_uid| {
+                return Err(Error::InsuitablePlatform(
-                if let Err(DslError::NotFound) = jobs.find(job_uid).first::<JobMeta>(&self.conn) {
+                    agent_platform.into_string(),
-                    Some(job_uid.to_string())
+                    meta.target_platform.clone(),
-                } else {
+                ));
-                    None
             }
-            })
-            .collect::<Vec<String>>();
-        if not_found_jobs.len() > 0 {
-            return Err(ULocalError::NotFound(not_found_jobs.join(", ")));
-        }
-        let job_requests = job_uids
-            .iter()
-            .map(|job_uid| {
-                info!("set_jobs_for_agent: set {} for {}", job_uid, agent_uid);
-                AssignedJob {
-                    job_id: *job_uid,
-                    agent_id: *agent_uid,
-                    ..Default::default()
         }
+
+        let job_requests = assigned_jobs
+            .into_iter()
+            .map(|a| AssignedJob {
+                job_id: a.job_id,
+                agent_id: a.agent_id,
+                alias: jobs_meta[&a.job_id].alias.clone(),
+                ..Default::default()
             })
             .collect::<Vec<AssignedJob>>();
+
         diesel::insert_into(results::table)
             .values(&job_requests)
-            .execute(&self.conn)?;
+            .execute(self.conn)
-        let assigned_uids = job_requests.iter().map(|aj| aj.id).collect();
+            .map(drop)
-        Ok(assigned_uids)
+            .map_err(with_err_ctx("Can't assign jobs"))
     }
 
-    pub fn del_jobs(&self, uids: &Vec<Uuid>) -> ULocalResult<usize> {
+    pub fn del_jobs(&mut self, ids: &[Id]) -> Result<()> {
         use schema::jobs;
-        let mut affected = 0;
+
-        for &uid in uids {
+        diesel::delete(jobs::table)
-            let deleted = diesel::delete(jobs::table)
+            .filter(jobs::id.eq_any(ids))
-                .filter(jobs::id.eq(uid))
+            .execute(self.conn)
-                .execute(&self.conn)?;
+            .map(drop)
-            affected += deleted;
+            .map_err(with_err_ctx("Can't delete jobs"))
-        }
-        Ok(affected)
     }
 
-    pub fn del_results(&self, uids: &Vec<Uuid>) -> ULocalResult<usize> {
+    pub fn del_results(&mut self, ids: &[Id]) -> Result<()> {
         use schema::results;
-        let mut affected = 0;
+
-        for &uid in uids {
+        diesel::delete(results::table)
-            let deleted = diesel::delete(results::table)
+            .filter(results::id.eq_any(ids))
-                .filter(results::id.eq(uid))
+            .execute(self.conn)
-                .execute(&self.conn)?;
+            .map(drop)
-            affected += deleted;
+            .map_err(with_err_ctx("Can't delete results"))
+    }
+
+    pub fn del_agents(&mut self, ids: &[Id]) -> Result<()> {
+        use schema::agents;
+
+        diesel::delete(agents::table)
+            .filter(agents::id.eq_any(ids))
+            .execute(self.conn)
+            .map(drop)
+            .map_err(with_err_ctx("Can't delete agents"))
     }
-        Ok(affected)
+
+    pub fn del_payloads(&mut self, ids: &[Id]) -> Result<()> {
+        use schema::payloads;
+
+        diesel::delete(payloads::table)
+            .filter(payloads::id.eq_any(ids))
+            .execute(self.conn)
+            .map(drop)
+            .map_err(with_err_ctx("Can't delete payloads"))
     }
 
-    pub fn del_agents(&self, uids: &Vec<Uuid>) -> ULocalResult<usize> {
+    pub fn upsert_agent(&mut self, agent: &Agent) -> Result<()> {
         use schema::agents;
-        let mut affected = 0;
+
-        for &uid in uids {
+        diesel::insert_into(agents::table)
-            let deleted = diesel::delete(agents::table)
+            .values(agent)
-                .filter(agents::id.eq(uid))
+            .on_conflict(agents::id)
-                .execute(&self.conn)?;
+            .do_update()
-            affected += deleted;
+            .set(agent)
-        }
+            .execute(self.conn)
-        Ok(affected)
+            .map_err(with_err_ctx(format!("Can't insert agent {agent:?}")))?;
-    }
+        Ok(())
-}
+    }
-/*
+
-#[cfg(test)]
+    pub fn update_job(&mut self, job: &JobMeta) -> Result<()> {
-mod tests {
+        job.save_changes::<JobMeta>(self.conn)
-    use super::*;
+            .map_err(with_err_ctx(format!("Can't update job {job:?}")))?;
-
+        Ok(())
-    fn setup_db() -> Storage {
+    }
-        return UDB::new().unwrap();
+
-    }
+    pub fn update_payload(&mut self, payload: &Payload) -> Result<()> {
-
+        payload
-    #[tokio::test]
+            .save_changes::<Payload>(self.conn)
-    async fn test_add_agent() {
+            .map_err(with_err_ctx(format!("Can't update payload {payload:?}")))?;
-        let db = setup_db();
+        Ok(())
-        let agent = IAgent {
-            alias: None,
-            id: "000-000".to_string(),
-            hostname: "test".to_string(),
-            is_root: false,
-            is_root_allowed: false,
-            platform: "linux".to_string(),
-            status: None,
-            token: None,
-            username: "test".to_string()
-        };
-        db.lock().unwrap().new_agent(agent).unwrap();
-        let result = db.lock().unwrap().get_agents().unwrap();
-        assert_eq!(
-            result[0].username,
-            "test".to_string()
-        )
     }
+
+    pub fn update_result(&mut self, result: &AssignedJob) -> Result<()> {
+        debug!(
+            "updating result: id = {}, job_id = {}, agent_id = {}",
+            result.id, result.job_id, result.agent_id
+        );
+        result
+            .save_changes::<AssignedJob>(self.conn)
+            .map_err(with_err_ctx(format!("Can't update result {result:?}")))?;
+        Ok(())
+    }
+}
+
+fn with_err_ctx(msg: impl AsRef<str>) -> impl Fn(DslError) -> Error {
+    move |err| Error::DBErrorCtx(format!("{}, reason: {err}", msg.as_ref()))
 }
-*/

bin/u_server/src/error.rs

@@ -0,0 +1,98 @@
+use diesel::result::Error as DslError;
+use thiserror::Error;
+use u_lib::{ufs, UError};
+use warp::{
+    http::StatusCode,
+    reject::Reject,
+    reply::{with_status, Response},
+    Reply,
+};
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("Configs error: {0}")]
+    ConfigError(#[from] u_lib::config::Error),
+
+    #[error("Processing error: {0}")]
+    ProcessingError(String),
+
+    #[error(transparent)]
+    DBError(#[from] DslError),
+
+    #[error("DB error: {0}")]
+    DBErrorCtx(String),
+
+    #[error("Deadpool error: {0}")]
+    DeadpoolError(#[from] deadpool_diesel::PoolError),
+
+    #[error(transparent)]
+    FSError(#[from] ufs::Error),
+
+    #[error("Job cannot be ran on this platform. Expected: {0}, got: {1}")]
+    InsuitablePlatform(String, String),
+
+    #[error("{0}\nContext: {1}")]
+    Contexted(Box<Error>, String),
+
+    #[error(transparent)]
+    UError(#[from] UError),
+
+    #[error("Runtime error: {0}")]
+    Runtime(String),
+}
+
+impl Reject for Error {}
+
+pub struct RejResponse {
+    message: String,
+    status: StatusCode,
+}
+
+impl RejResponse {
+    pub fn not_found(msg: impl Into<String>) -> Self {
+        Self {
+            message: msg.into(),
+            status: StatusCode::NOT_FOUND,
+        }
+    }
+
+    pub fn bad_request(msg: impl Into<String>) -> Self {
+        Self {
+            message: msg.into(),
+            status: StatusCode::BAD_REQUEST,
+        }
+    }
+
+    pub fn internal() -> Self {
+        Self {
+            message: "INTERNAL SERVER ERROR".to_string(),
+            status: StatusCode::INTERNAL_SERVER_ERROR,
+        }
+    }
+}
+
+impl Reply for RejResponse {
+    fn into_response(self) -> Response {
+        with_status(self.message, self.status).into_response()
+    }
+}
+
+impl From<anyhow::Error> for Error {
+    fn from(e: anyhow::Error) -> Self {
+        let ctx = e
+            .chain()
+            .rev()
+            .skip(1)
+            .map(|cause| format!("ctx: {}", cause))
+            .collect::<Vec<_>>()
+            .join("\n");
+
+        match e.downcast::<Error>() {
+            Ok(err) => Error::Contexted(Box::new(err), ctx),
+            Err(err) => match err.downcast::<ufs::Error>() {
+                Ok(err) => Error::Contexted(Box::new(Error::FSError(err)), ctx),
+                Err(err) => Error::Runtime(err.to_string()),
+            },
+        }
+    }
+}

bin/u_server/src/filters.rs

@@ -1,87 +0,0 @@
-use crate::handlers::Endpoints;
-use serde::de::DeserializeOwned;
-use std::env;
-use u_lib::{
-    messaging::{AsMsg, BaseMessage},
-    models::*,
-};
-use uuid::Uuid;
-use warp::{body, Filter, Rejection, Reply};
-
-fn get_content<M>() -> impl Filter<Extract = (BaseMessage<'static, M>,), Error = Rejection> + Clone
-where
-    M: AsMsg + Sync + Send + DeserializeOwned + 'static,
-{
-    body::content_length_limit(1024 * 64).and(body::json::<BaseMessage<M>>())
-}
-
-pub fn make_filters() -> impl Filter<Extract = (impl Reply,), Error = Rejection> + Clone {
-    let infallible_none = |_| async { Ok::<(Option<Uuid>,), std::convert::Infallible>((None,)) };
-
-    let get_agents = warp::get()
-        .and(warp::path("get_agents"))
-        .and(
-            warp::path::param::<Uuid>()
-                .map(Some)
-                .or_else(infallible_none),
-        )
-        .and_then(Endpoints::get_agents);
-
-    let upload_jobs = warp::post()
-        .and(warp::path("upload_jobs"))
-        .and(get_content::<Vec<JobMeta>>())
-        .and_then(Endpoints::upload_jobs);
-
-    let get_jobs = warp::get()
-        .and(warp::path("get_jobs"))
-        .and(
-            warp::path::param::<Uuid>()
-                .map(Some)
-                .or_else(infallible_none),
-        )
-        .and_then(Endpoints::get_jobs);
-
-    let get_agent_jobs = warp::get()
-        .and(warp::path("get_agent_jobs"))
-        .and(
-            warp::path::param::<Uuid>()
-                .map(Some)
-                .or_else(infallible_none),
-        )
-        .and_then(|uid| Endpoints::get_agent_jobs(uid));
-
-    let get_personal_jobs = warp::get()
-        .and(warp::path("get_personal_jobs"))
-        .and(warp::path::param::<Uuid>().map(Some))
-        .and_then(|uid| Endpoints::get_personal_jobs(uid));
-
-    let del = warp::get()
-        .and(warp::path("del"))
-        .and(warp::path::param::<Uuid>())
-        .and_then(Endpoints::del);
-
-    let set_jobs = warp::post()
-        .and(warp::path("set_jobs"))
-        .and(warp::path::param::<Uuid>())
-        .and(get_content::<Vec<String>>())
-        .and_then(Endpoints::set_jobs);
-
-    let report = warp::post()
-        .and(warp::path("report"))
-        .and(get_content::<Vec<ExecResult>>().and_then(Endpoints::report));
-
-    let auth_token = format!("Bearer {}", env::var("ADMIN_AUTH_TOKEN").unwrap()).into_boxed_str();
-    let auth_header = warp::header::exact("authorization", Box::leak(auth_token));
-
-    let auth_zone = (get_agents
-        .or(get_jobs)
-        .or(upload_jobs)
-        .or(del)
-        .or(set_jobs)
-        .or(get_agent_jobs))
-    .and(auth_header);
-
-    let agent_zone = get_jobs.clone().or(get_personal_jobs).or(report);
-
-    auth_zone.or(agent_zone)
-}

bin/u_server/src/handlers.rs

@@ -1,171 +1,343 @@
-use crate::db::UDB;
+use std::sync::Arc;
-use diesel::SaveChangesDsl;
+
-use hyper::Body;
+use crate::db::{PgRepo, UDB};
-use serde::Serialize;
+use crate::error::Error;
-use u_lib::{
+use serde::Deserialize;
-    messaging::{AsMsg, BaseMessage},
+use u_lib::{api::api_types, messaging::Reportable, models::*, types::Id};
-    models::{Agent, AgentState, AssignedJob, ExecResult, JobMeta, JobState},
+use warp::reject::not_found;
-    ULocalError,
+use warp::Rejection;
-};
+
-use uuid::Uuid;
+type EndpResult<T> = Result<T, Rejection>;
-use warp::{
+
-    http::{Response, StatusCode},
+#[derive(Deserialize)]
-    Rejection, Reply,
+pub struct PayloadFlags {
+    brief: Brief,
+}
+
+pub struct Endpoints;
+
+impl Endpoints {
+    pub async fn get_agents(repo: Arc<PgRepo>, id: Option<Id>) -> EndpResult<api_types::GetAgents> {
+        repo.interact(move |mut db| {
+            Ok(match id {
+                Some(id) => {
+                    if let Some(agent) = db.get_agent(id)? {
+                        vec![agent]
+                    } else {
+                        vec![]
+                    }
+                }
+                None => db.get_agents()?,
+            })
+        })
+        .await
+        .map_err(From::from)
+    }
+
+    pub async fn get_job(
+        repo: Arc<PgRepo>,
+        id: Id,
+        params: Option<PayloadFlags>,
+    ) -> EndpResult<api_types::GetJob> {
+        let Some(mut job) = repo.interact(move |mut db| db.get_job(id)).await? else {
+            return Err(not_found());
         };
 
-pub fn build_response<S: Into<Body>>(code: StatusCode, body: S) -> Response<Body> {
+        Ok(match params.map(|p| p.brief) {
-    Response::builder().status(code).body(body.into()).unwrap()
+            Some(Brief::Yes) => job,
+            Some(Brief::Auto) | None => {
+                if let Some(payload) = &mut job.payload {
+                    payload.maybe_join_payload().map_err(Error::from)?;
+                }
+                job
+            }
+            Some(Brief::No) => {
+                if let Some(payload) = &mut job.payload {
+                    payload.join_payload().map_err(Error::from)?;
+                }
+                job
+            }
+        })
     }
 
-pub fn build_ok<S: Into<Body>>(body: S) -> Response<Body> {
+    pub async fn get_jobs(repo: Arc<PgRepo>) -> EndpResult<api_types::GetJobs> {
-    build_response(StatusCode::OK, body)
+        repo.interact(move |mut db| db.get_jobs())
+            .await
+            .map_err(From::from)
     }
 
-pub fn build_err<S: ToString>(body: S) -> Response<Body> {
+    pub async fn get_assigned_jobs(
-    build_response(StatusCode::BAD_REQUEST, body.to_string())
+        repo: Arc<PgRepo>,
+        id: Option<Id>,
+    ) -> EndpResult<api_types::GetAgentJobs> {
+        repo.interact(move |mut db| db.get_assigned_jobs(id, false))
+            .await
+            .map_err(From::from)
     }
 
-pub fn build_message<M: AsMsg + Serialize>(m: M) -> Response<Body> {
+    pub async fn get_payloads(repo: Arc<PgRepo>) -> EndpResult<api_types::GetPayloads> {
-    warp::reply::json(&m.as_message()).into_response()
+        repo.interact(move |mut db| db.get_payloads())
+            .await
+            .map_err(From::from)
     }
 
-pub struct Endpoints;
+    pub async fn get_payload(
+        repo: Arc<PgRepo>,
+        name_or_id: String,
+        params: Option<PayloadFlags>,
+    ) -> EndpResult<api_types::GetPayload> {
+        let mut payload = match repo
+            .interact(move |mut db| match Id::parse_str(&name_or_id) {
+                Ok(id) => db.get_payload(id),
+                Err(_) => db.get_payload_by_name(name_or_id),
+            })
+            .await?
+        {
+            Some(p) => p,
+            None => return Err(not_found()),
+        };
 
-#[cfg_attr(test, automock)]
+        Ok(match params.map(|p| p.brief) {
-impl Endpoints {
+            Some(Brief::Yes) => {
-    pub async fn add_agent(msg: Agent) -> Result<Response<Body>, Rejection> {
+                payload.data = None;
-        info!("hnd: add_agent");
+                payload
-        UDB::lock_db()
+            }
-            .insert_agent(&msg)
+            None | Some(Brief::Auto) => {
-            .map(|_| build_ok(""))
+                payload.maybe_join_payload().map_err(Error::from)?;
-            .or_else(|e| Ok(build_err(e)))
+                payload
+            }
+            _ => {
+                payload.join_payload().map_err(Error::from)?;
+                payload
+            }
+        })
     }
 
-    pub async fn get_agents(uid: Option<Uuid>) -> Result<Response<Body>, Rejection> {
+    pub async fn get_personal_jobs(
-        info!("hnd: get_agents");
+        repo: Arc<PgRepo>,
-        UDB::lock_db()
+        agent_id: Id,
-            .get_agents(uid)
+    ) -> EndpResult<api_types::GetPersonalJobs> {
-            .map(|m| build_message(m))
+        repo.transaction(move |mut db| {
-            .or_else(|e| Ok(build_err(e)))
+            let agent = db.get_agent(agent_id)?;
+            match agent {
+                Some(mut agent) => {
+                    agent.touch();
+                    db.upsert_agent(&agent)?;
                 }
+                None => {
+                    let mut new_agent = Agent::empty();
+                    new_agent.id = agent_id;
+
+                    db.upsert_agent(&new_agent)?;
 
-    pub async fn get_jobs(uid: Option<Uuid>) -> Result<Response<Body>, Rejection> {
+                    let job = db
-        info!("hnd: get_jobs");
+                        .get_job_by_alias("agent_hello")?
-        UDB::lock_db()
+                        .expect("agent_hello job not found");
-            .get_jobs(uid)
+
-            .map(|m| build_message(m))
+                    let assigned_job = AssignedJobById {
-            .or_else(|e| Ok(build_err(e)))
+                        agent_id,
+                        job_id: job.meta.id,
+                        ..Default::default()
+                    };
+                    db.assign_jobs(&[assigned_job])?;
                 }
+            }
+
+            let assigned_jobs = db.get_assigned_jobs(Some(agent_id), true)?;
 
-    pub async fn get_agent_jobs(uid: Option<Uuid>) -> Result<Response<Body>, Rejection> {
+            for job in &assigned_jobs {
-        info!("hnd: get_agent_jobs");
+                db.update_job_status(job.id, JobState::Running)?;
-        UDB::lock_db()
-            .get_exact_jobs(uid, false)
-            .map(|m| build_message(m))
-            .or_else(|e| Ok(build_err(e)))
             }
 
-    pub async fn get_personal_jobs(uid: Option<Uuid>) -> Result<Response<Body>, Rejection> {
+            Ok(assigned_jobs
-        info!("hnd: get_personal_jobs");
+                .into_iter()
-        let agents = UDB::lock_db().get_agents(uid).unwrap();
+                .map(|j| AssignedJobById::from(&j))
-        if agents.len() == 0 {
+                .collect())
-            let db = UDB::lock_db();
+        })
-            db.insert_agent(&Agent::with_id(uid.unwrap())).unwrap();
+        .await
-            let job = db.find_job_by_alias("agent_hello").unwrap();
+        .map_err(From::from)
-            if let Err(e) = db.set_jobs_for_agent(&uid.unwrap(), &[job.id]) {
-                return Ok(build_err(e));
     }
+
+    pub async fn upload_jobs(
+        repo: Arc<PgRepo>,
+        jobs: Vec<Job>,
+    ) -> EndpResult<api_types::UploadJobs> {
+        let mut checked_jobs = vec![];
+        for mut job in jobs {
+            if let Some(payload) = &mut job.payload {
+                payload.maybe_split_payload().map_err(Error::from)?;
+            } else if let Some(pld_id) = job.meta.payload_id {
+                if !repo
+                    .interact(move |mut db| db.payload_exists(pld_id))
+                    .await?
+                {
+                    Err(Error::ProcessingError(format!(
+                        "Payload {pld_id} not found"
+                    )))?
                 }
-        let result = UDB::lock_db().get_exact_jobs(uid, true);
-        match result {
-            Ok(r) => {
-                let db = UDB::lock_db();
-                for j in r.iter() {
-                    db.update_job_status(j.id, JobState::Running).unwrap();
             }
-                Ok(build_message(r))
+            checked_jobs.push(job)
+        }
+
+        let (jobs, payloads_opt): (Vec<_>, Vec<_>) = checked_jobs
+            .into_iter()
+            .map(|j| (j.meta, j.payload))
+            .unzip();
+
+        let payloads = payloads_opt
+            .into_iter()
+            .filter_map(|p| p)
+            .collect::<Vec<_>>();
+
+        repo.transaction(move |mut db| {
+            for payload in payloads {
+                db.insert_payload(&payload)?;
             }
-            Err(e) => Ok(build_err(e)),
+            db.insert_jobs(&jobs)
+        })
+        .await
+        .map_err(From::from)
     }
+
+    pub async fn upload_payload(
+        repo: Arc<PgRepo>,
+        raw_payload: RawPayload,
+    ) -> EndpResult<api_types::UploadPayloads> {
+        let payloads = raw_payload.into_payload().map_err(Error::from)?;
+
+        repo.interact(move |mut db| db.insert_payload(&payloads))
+            .await
+            .map_err(From::from)
     }
 
-    pub async fn upload_jobs(
+    pub async fn del(repo: Arc<PgRepo>, id: Id) -> EndpResult<()> {
-        msg: BaseMessage<'static, Vec<JobMeta>>,
+        repo.transaction(move |mut db| {
-    ) -> Result<Response<Body>, Rejection> {
+            [
-        info!("hnd: upload_jobs");
+                UDB::del_agents,
-        UDB::lock_db()
+                UDB::del_jobs,
-            .insert_jobs(&msg.into_inner())
+                UDB::del_results,
-            .map(|_| build_ok(""))
+                UDB::del_payloads,
-            .or_else(|e| Ok(build_err(e)))
+            ]
-    }
+            .iter()
-
+            .map(|f| f(&mut db, &[id]))
-    pub async fn del(uid: Uuid) -> Result<Response<Body>, Rejection> {
+            .collect::<Result<(), Error>>()
-        info!("hnd: del");
-        let db = UDB::lock_db();
-        let del_fns = &[UDB::del_agents, UDB::del_jobs, UDB::del_results];
-        for del_fn in del_fns {
-            let affected = del_fn(&db, &vec![uid]).unwrap();
-            if affected > 0 {
-                return Ok(build_message(affected as i32));
-            }
-        }
-        Ok(build_message(0))
-    }
-
-    pub async fn set_jobs(
-        agent_uid: Uuid,
-        msg: BaseMessage<'static, Vec<String>>,
-    ) -> Result<Response<Body>, Rejection> {
-        info!("hnd: set_jobs_by_alias, agent: {}", agent_uid);
-        let jobs: Result<Vec<Uuid>, ULocalError> = msg
-            .into_inner()
-            .into_iter()
-            .map(|ident| {
-                info!("hnd: set_jobs_by_alias, job: {}", ident);
-                Uuid::parse_str(&ident)
-                    .or_else(|_| UDB::lock_db().find_job_by_alias(&ident).map(|j| j.id))
         })
-            .collect();
+        .await
-        match jobs {
+        .map_err(From::from)
-            Ok(j) => UDB::lock_db()
-                .set_jobs_for_agent(&agent_uid, &j)
-                .map(|assigned_uids| build_message(assigned_uids))
-                .or_else(|e| Ok(build_err(e))),
-            Err(e) => Ok(build_err(e)),
     }
+
+    pub async fn assign_jobs(
+        repo: Arc<PgRepo>,
+        assigned_jobs: Vec<AssignedJobById>,
+    ) -> EndpResult<()> {
+        repo.transaction(move |mut db| {
+            db.assign_jobs(&assigned_jobs)?;
+            Ok(())
+        })
+        .await
+        .map_err(From::from)
     }
 
     pub async fn report(
-        msg: BaseMessage<'static, Vec<ExecResult>>,
+        repo: Arc<PgRepo>,
-    ) -> Result<Response<Body>, Rejection> {
+        msg: Vec<Reportable>,
-        info!("hnd: report");
+        agent_id: Id,
-        let id = msg.id;
+    ) -> EndpResult<api_types::Report> {
-        let mut failed = vec![];
+        repo.transaction(move |mut db| {
-        for entry in msg.into_inner() {
+            for entry in msg {
                 match entry {
-                ExecResult::Assigned(res) => {
+                    Reportable::Assigned(mut result) => {
-                    if id != res.agent_id {
+                        let result_agent_id = &result.agent_id;
+                        if agent_id != *result_agent_id {
+                            warn!(
+                                "Agent ids are not equal! actual id: {agent_id}, \
+                                id from job: {result_agent_id}"
+                            );
                             continue;
                         }
-                    let db = UDB::lock_db();
+                        result.touch();
-                    if let Err(e) = res
+
-                        .save_changes::<AssignedJob>(&db.conn)
+                        info!("agent {agent_id} updated job {}", result.id);
-                        .map_err(ULocalError::from)
+
-                    {
+                        match result.exec_type {
-                        failed.push(e.to_string())
+                            JobType::Init => {
+                                result.state = JobState::Finished;
+
+                                let mut agent: Agent = match result.deserialize() {
+                                    Ok(a) => a,
+                                    Err(e) => {
+                                        error!(
+                                            "Error deserializing agent \
+                                                    data from {agent_id}: {e}"
+                                        );
+                                        continue;
+                                    }
+                                };
+                                agent.state = AgentState::Active;
+                                db.upsert_agent(&agent)?;
                             }
+                            JobType::Shell => result.state = JobState::Finished,
+                            JobType::Stats => result.state = JobState::Finished,
+                            JobType::Terminate => (),
+                            JobType::Update => (),
                         }
-                ExecResult::Agent(mut a) => {
+                        db.update_result(&result)?;
-                    a.state = AgentState::Active;
-                    Self::add_agent(a).await?;
                     }
-                ExecResult::Dummy => (),
+                    Reportable::Error(e) => {
+                        error!("agent {agent_id} reported: {e}");
+                    }
+                    Reportable::Dummy => (),
+                }
+            }
+            Ok(())
+        })
+        .await
+        .map_err(From::from)
     }
+
+    pub async fn update_agent(
+        repo: Arc<PgRepo>,
+        agent: Agent,
+    ) -> EndpResult<api_types::UpdateAgent> {
+        repo.interact(move |mut db| db.upsert_agent(&agent))
+            .await
+            .map_err(From::from)
+    }
+
+    pub async fn update_job(repo: Arc<PgRepo>, job: JobMeta) -> EndpResult<api_types::UpdateJob> {
+        repo.interact(move |mut db| db.update_job(&job.validate()?))
+            .await
+            .map_err(From::from)
+    }
+
+    pub async fn update_assigned_job(
+        repo: Arc<PgRepo>,
+        assigned: AssignedJob,
+    ) -> EndpResult<api_types::UpdateResult> {
+        repo.interact(move |mut db| db.update_result(&assigned))
+            .await
+            .map_err(From::from)
+    }
+
+    pub async fn update_payload(
+        repo: Arc<PgRepo>,
+        payload: Payload,
+    ) -> EndpResult<api_types::UpdatePayload> {
+        debug!("update payload: {payload:?}");
+        match payload.data {
+            Some(data) => {
+                let mut well_formed_payload =
+                    Payload::from_data(data, Some(payload.name)).map_err(Error::from)?;
+                well_formed_payload.id = payload.id;
+
+                repo.interact(move |mut db| db.update_payload(&well_formed_payload))
+                    .await
+                    .map_err(From::from)
             }
-        if failed.len() > 0 {
+            None => repo
-            let err_msg = ULocalError::ProcessingError(failed.join(", "));
+                .interact(move |mut db| db.update_payload(&payload))
-            return Ok(build_err(err_msg));
+                .await
+                .map_err(From::from),
         }
-        Ok(build_ok(""))
     }
 }

bin/u_server/src/main.rs

@@ -1,6 +1,17 @@
-use u_server_lib::serve;
+// due to linking errors
+extern crate openssl;
+// don't touch anything
+extern crate diesel;
+// in this block
+
+#[macro_use]
+extern crate tracing;
 
 #[tokio::main]
 async fn main() {
-    serve().await;
+    u_lib::logging::init_logger(Some("u_server"));
+
+    if let Err(e) = u_server_lib::serve().await {
+        error!("U_SERVER error: {}", e);
+    }
 }

bin/u_server/src/u_server.rs

@@ -1,117 +1,300 @@
 #[macro_use]
-extern crate log;
+extern crate tracing;
 
-#[macro_use]
-extern crate mockall;
-#[macro_use]
-extern crate mockall_double;
-
-// because of linking errors
-extern crate openssl;
-#[macro_use]
-extern crate diesel;
-//
 mod db;
-mod filters;
+mod error;
 mod handlers;
 
-use db::UDB;
+use crate::handlers::{Endpoints, PayloadFlags};
-use filters::make_filters;
+use db::PgRepo;
-use u_lib::{config::MASTER_PORT, models::*, utils::init_env};
+use error::{Error as ServerError, RejResponse};
-use warp::Filter;
+use std::{convert::Infallible, sync::Arc};
+use u_lib::{
+    config,
+    db::async_pool,
+    messaging::{AsMsg, Reportable},
+    models::*,
+    types::Id,
+};
+use warp::{
+    body,
+    log::{custom, Info},
+    reply::{json, Json, Response},
+    Filter, Rejection, Reply,
+};
 
-const LOGFILE: &str = "u_server.log";
+const DEFAULT_RESPONSE: &str = "null";
 
-fn prefill_jobs() {
+fn into_message<M: AsMsg>(msg: M) -> Json {
-    let agent_hello = JobMeta::builder()
+    json(&msg)
-        .with_type(misc::JobType::Manage)
-        .with_alias("agent_hello")
-        .build()
-        .unwrap();
-    UDB::lock_db().insert_jobs(&[agent_hello]).ok();
 }
 
-fn init_logger() {
+pub fn init_endpoints(
-    use simplelog::*;
+    auth_token: &str,
-    use std::fs::OpenOptions;
+    db: PgRepo,
-    let log_cfg = ConfigBuilder::new()
+) -> impl Filter<Extract = (impl Reply,), Error = Rejection> + Clone {
-        .set_time_format_str("%x %X")
+    fn make_optional<T>(
-        .set_time_to_local(true)
+        f: impl Filter<Extract = (T,), Error = Rejection> + Clone,
-        .build();
+    ) -> impl Filter<Extract = (Option<T>,), Error = Infallible> + Clone {
-    let logfile = OpenOptions::new()
+        f.map(Some)
-        .append(true)
+            .or_else(|_| async { Ok::<(Option<T>,), Infallible>((None,)) })
-        .create(true)
+    }
-        .open(LOGFILE)
+
-        .unwrap();
+    let path = |p: &'static str| warp::post().and(warp::path(p));
-    let level = LevelFilter::Info;
+    let create_qs_cfg = || serde_qs::Config::new(1, true);
-    let loggers = vec![
+
-        WriteLogger::new(level, log_cfg.clone(), logfile) as Box<dyn SharedLogger>,
+    let with_db = {
-        TermLogger::new(level, log_cfg, TerminalMode::Stderr, ColorChoice::Auto),
+        let adb = Arc::new(db);
-    ];
+        warp::any().map(move || adb.clone())
-    CombinedLogger::init(loggers).unwrap();
+    };
+
+    let get_agents = path("get_agents")
+        .and(with_db.clone())
+        .and(make_optional(warp::path::param::<Id>()))
+        .and_then(Endpoints::get_agents)
+        .map(into_message);
+
+    let upload_jobs = path("upload_jobs")
+        .and(with_db.clone())
+        .and(body::json::<Vec<Job>>())
+        .and_then(Endpoints::upload_jobs)
+        .map(into_message);
+
+    let get_job = path("get_job")
+        .and(with_db.clone())
+        .and(warp::path::param::<Id>())
+        .and(make_optional(serde_qs::warp::query::<PayloadFlags>(
+            create_qs_cfg(),
+        )))
+        .and_then(Endpoints::get_job)
+        .map(into_message);
+
+    let get_jobs = path("get_jobs")
+        .and(with_db.clone())
+        .and_then(Endpoints::get_jobs)
+        .map(into_message);
+
+    let get_assigned_jobs = path("get_assigned_jobs")
+        .and(with_db.clone())
+        .and(make_optional(warp::path::param::<Id>()))
+        .and_then(Endpoints::get_assigned_jobs)
+        .map(into_message);
+
+    let get_personal_jobs = path("get_personal_jobs")
+        .and(with_db.clone())
+        .and(warp::path::param::<Id>())
+        .and_then(Endpoints::get_personal_jobs)
+        .map(into_message);
+
+    let del = path("del")
+        .and(with_db.clone())
+        .and(warp::path::param::<Id>())
+        .and_then(Endpoints::del)
+        .map(ok);
+
+    let assign_jobs = path("assign_jobs")
+        .and(with_db.clone())
+        .and(body::json::<Vec<AssignedJobById>>())
+        .and_then(Endpoints::assign_jobs)
+        .map(into_message);
+
+    let report = path("report")
+        .and(with_db.clone())
+        .and(body::json::<Vec<Reportable>>())
+        .and(warp::header("User-Agent"))
+        .and_then(Endpoints::report)
+        .map(ok);
+
+    let update_agent = path("update_agent")
+        .and(with_db.clone())
+        .and(body::json::<Agent>())
+        .and_then(Endpoints::update_agent)
+        .map(ok);
+
+    let update_job = path("update_job")
+        .and(with_db.clone())
+        .and(body::json::<JobMeta>())
+        .and_then(Endpoints::update_job)
+        .map(ok);
+
+    let update_assigned_job = path("update_result")
+        .and(with_db.clone())
+        .and(body::json::<AssignedJob>())
+        .and_then(Endpoints::update_assigned_job)
+        .map(ok);
+
+    let get_payloads = path("get_payloads")
+        .and(with_db.clone())
+        .and_then(Endpoints::get_payloads)
+        .map(into_message);
+
+    let get_payload = path("get_payload")
+        .and(with_db.clone())
+        .and(warp::path::param::<String>())
+        .and(make_optional(serde_qs::warp::query::<PayloadFlags>(
+            create_qs_cfg(),
+        )))
+        .and_then(Endpoints::get_payload)
+        .map(into_message);
+
+    let upload_payload = path("upload_payload")
+        .and(with_db.clone())
+        .and(body::json::<RawPayload>())
+        .and_then(Endpoints::upload_payload)
+        .map(ok);
+
+    let update_payload = path("update_payload")
+        .and(with_db.clone())
+        .and(body::json::<Payload>())
+        .and_then(Endpoints::update_payload)
+        .map(ok);
+
+    let ping = path("ping").map(|| DEFAULT_RESPONSE);
+
+    let auth_token = format!("Bearer {auth_token}",).into_boxed_str();
+    let auth_header = warp::header::exact("authorization", Box::leak(auth_token));
+
+    let auth_zone = (get_agents
+        .or(get_job.clone())
+        .or(get_jobs.clone())
+        .or(get_payloads)
+        .or(get_payload)
+        .or(upload_jobs)
+        .or(upload_payload)
+        .or(del)
+        .or(assign_jobs)
+        .or(get_assigned_jobs)
+        .or(update_agent)
+        .or(update_job)
+        .or(update_assigned_job)
+        .or(update_payload)
+        .or(ping))
+    .and(auth_header);
+
+    let agent_zone = get_job.or(get_jobs).or(get_personal_jobs).or(report);
+
+    auth_zone.or(agent_zone)
 }
 
-fn init_all() {
+pub async fn preload_jobs(repo: &PgRepo) -> Result<(), ServerError> {
-    init_logger();
+    repo.interact(|mut db| {
-    init_env();
+        let job_alias = "agent_hello";
-    prefill_jobs();
+        let if_job_exists = db.get_job_by_alias(job_alias)?;
+        if if_job_exists.is_none() {
+            let agent_hello = RawJob::default()
+                .with_type(JobType::Init)
+                .with_alias(job_alias)
+                .try_into_job()
+                .unwrap();
+            db.insert_jobs(&[agent_hello.meta])?;
+        }
+        Ok(())
+    })
+    .await
 }
 
-pub async fn serve() {
+pub async fn serve() -> Result<(), ServerError> {
-    init_all();
+    let env = config::DBEnv::load()?;
-    let routes = make_filters();
+    let pool = async_pool(&env);
-    warp::serve(routes.with(warp::log("warp")))
+    let db = PgRepo::new(pool);
+
+    preload_jobs(&db).await?;
+
+    let env = config::AccessEnv::load()?;
+    let routes = init_endpoints(&env.admin_auth_token, db)
+        .recover(handle_rejection)
+        .with(custom(logger));
+
+    let server_cert = include_bytes!("../../../certs/server.crt");
+    let server_key = include_bytes!("../../../certs/server.key");
+    let ca = include_bytes!("../../../certs/ca.crt");
+
+    warp::serve(routes)
         .tls()
-        .cert_path("./certs/server.crt")
+        .cert(server_cert)
-        .key_path("./certs/server.key")
+        .key(server_key)
-        .client_auth_required_path("./certs/ca.crt")
+        .client_auth_required(ca)
-        .run(([0, 0, 0, 0], MASTER_PORT))
+        .run(([0, 0, 0, 0], config::MASTER_PORT))
         .await;
+    Ok(())
+}
+
+async fn handle_rejection(rej: Rejection) -> Result<Response, Infallible> {
+    let resp = if let Some(err) = rej.find::<ServerError>() {
+        error!("{:?}", err);
+        RejResponse::bad_request(err.to_string())
+    } else if rej.is_not_found() {
+        RejResponse::not_found("not found placeholder")
+    } else {
+        error!("{:?}", rej);
+        RejResponse::internal()
+    };
+    Ok(resp.into_response())
 }
 
+fn logger(info: Info<'_>) {
+    info!(target: "warp",
+        "{raddr} {agent_id} \"{path}\" {status}",
+        raddr = info.remote_addr().unwrap_or(([0, 0, 0, 0], 0).into()),
+        path = info.path(),
+        agent_id = info.user_agent()
+            .map(|id: &str| id.splitn(3, '-')
+                .take(2)
+                .collect::<String>()
+            )
+            .unwrap_or_else(|| "NO_AGENT_UID".to_string()),
+        status = info.status()
+    );
+}
+
+fn ok<T>(_: T) -> impl Reply {
+    DEFAULT_RESPONSE
+}
+
+/*
 #[cfg(test)]
 mod tests {
     use super::*;
-    #[double]
     use crate::handlers::Endpoints;
     use handlers::build_ok;
-    use mockall::predicate::*;
+    use u_lib::messaging::{AsMsg, BaseMessage, Reportable};
-    use test_case::test_case;
-    use u_lib::messaging::{AsMsg, BaseMessage};
     use uuid::Uuid;
-    use warp::test::request;
+    use warp::test;
 
-    #[test_case(Some(Uuid::new_v4()))]
+    #[rstest]
-    #[test_case(None => panics)]
+    #[case(Some(Uuid::new_v4()))]
+    #[should_panic]
+    #[case(None)]
     #[tokio::test]
-    async fn test_get_agent_jobs_unauthorized(uid: Option<Uuid>) {
+    async fn test_get_agent_jobs_unauthorized(#[case] uid: Option<Uuid>) {
-        let mock = Endpoints::get_agent_jobs_context();
+        let mock = Endpoints::faux();
-        mock.expect().with(eq(uid)).returning(|_| Ok(build_ok("")));
+        when!(mock.get_agent_jobs).then_return(Ok(build_ok("")));
-        request()
+        //mock.expect().with(eq(uid)).returning(|_| Ok(build_ok("")));
+        test::request()
             .path(&format!(
                 "/get_agent_jobs/{}",
                 uid.map(|u| u.simple().to_string()).unwrap_or(String::new())
             ))
             .method("GET")
-            .filter(&make_filters())
+            .filter(&init_filters(""))
             .await
             .unwrap();
-        mock.checkpoint();
     }
 
     #[tokio::test]
     async fn test_report_unauth_successful() {
-        let mock = Endpoints::report_context();
+        let mock = Endpoints::report();
         mock.expect()
-            .withf(|msg: &BaseMessage<'_, Vec<ExecResult>>| msg.inner_ref()[0] == ExecResult::Dummy)
+            .withf(|msg: &BaseMessage<'_, Vec<Reportable>>| msg.inner_ref()[0] == Reportable::Dummy)
             .returning(|_| Ok(build_ok("")));
-        request()
+        test::request()
             .path("/report/")
             .method("POST")
-            .json(&vec![ExecResult::Dummy].as_message())
+            .json(&vec![Reportable::Dummy].as_message())
-            .filter(&make_filters())
+            .filter(&init_filters(""))
             .await
             .unwrap();
         mock.checkpoint();
     }
 }
+*/

deploy/podman-compose.yml

@@ -0,0 +1,52 @@
+version: "3.4"
+
+networks:
+  u_net:
+
+services:
+
+  u_server:
+    image: localhost/unki/u_server
+    networks:
+      - u_net
+    volumes:
+      - ./u_server:/unki/u_server
+      - ./logs:/unki/logs:rw
+    working_dir: /unki
+    command: /unki/u_server
+    depends_on:
+      u_db:
+        condition: service_healthy
+    ports:
+      - 9990:9990
+    env_file:
+      - ./.env
+      - ./.env.private
+    environment:
+      RUST_LOG: warp=info,u_server_lib=debug
+    healthcheck:
+        test: ss -tlpn | grep 9990
+        interval: 5s
+        timeout: 2s
+        retries: 2
+
+  u_db:
+    image: localhost/unki/u_db
+    networks:
+      - u_net
+    env_file:
+      - ./.env
+      - ./.env.private
+    volumes:
+      - ./migrator:/migrator
+      - ./data:/var/lib/postgresql/data
+      - type: bind
+        source: ./u_db_entrypoint.sh
+        target: /u_db_entrypoint.sh
+    command: /u_db_entrypoint.sh
+    healthcheck:
+        # test if db's port is open and db is created
+        test: ss -tlpn | grep 5432 && psql -lqt -U $${POSTGRES_USER} | grep -qw $${POSTGRES_DATABASE}
+        interval: 5s
+        timeout: 5s
+        retries: 3

deploy/start_server.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+export DOCKER_UID=$(id -u)
+export DOCKER_GID=$(id -g)
+
+docker build -t localhost/unki/u_db -f u_db.Dockerfile .
+docker build -t localhost/unki/u_server -f u_server.Dockerfile .
+podman-compose down -v
+podman-compose up -d

images/musl-libs.Dockerfile

@@ -0,0 +1,95 @@
+FROM ubuntu:xenial
+LABEL maintainer="Eirik Albrigtsen <sszynrae@gmail.com>"
+
+# Required packages:
+# - musl-dev, musl-tools - the musl toolchain
+# - curl, g++, make, pkgconf, cmake - for fetching and building third party libs
+# - ca-certificates - openssl + curl + peer verification of downloads
+# - xutils-dev - for openssl makedepend
+# - libssl-dev and libpq-dev - for dynamic linking during diesel_codegen build process
+# - git - cargo builds in user projects
+# - linux-headers-amd64 - needed for building openssl 1.1 (stretch only)
+# - file - needed by rustup.sh install
+# - automake autoconf libtool - support crates building C deps as part cargo build
+# recently removed:
+# cmake (not used), nano, zlib1g-dev
+RUN apt-get update && apt-get install -y \
+  musl-dev \
+  musl-tools \
+  git \
+  file \
+  openssh-client \
+  make \
+  g++ \
+  curl \
+  pkgconf \
+  ca-certificates \
+  xutils-dev \
+  libssl-dev \
+  libpq-dev \
+  automake \
+  autoconf \
+  libtool \
+  python3 \
+  --no-install-recommends && \
+  rm -rf /var/lib/apt/lists/*
+
+# Convenience list of versions and variables for compilation later on
+# This helps continuing manually if anything breaks.
+ENV SSL_VER="1.0.2u" \
+    CURL_VER="7.77.0" \
+    ZLIB_VER="1.2.13" \
+    PQ_VER="11.12" \
+    SQLITE_VER="3350500" \
+    CC=musl-gcc \
+    PREFIX=/musl \
+    PATH=/usr/local/bin:/root/.cargo/bin:$PATH \
+    PKG_CONFIG_PATH=/usr/local/lib/pkgconfig \
+    LD_LIBRARY_PATH=$PREFIX
+
+# Set up a prefix for musl build libraries, make the linker's job of finding them easier
+# Primarily for the benefit of postgres.
+# Lastly, link some linux-headers for openssl 1.1 (not used herein)
+RUN mkdir $PREFIX && \
+    echo "$PREFIX/lib" >> /etc/ld-musl-x86_64.path && \
+    ln -s /usr/include/x86_64-linux-gnu/asm /usr/include/x86_64-linux-musl/asm && \
+    ln -s /usr/include/asm-generic /usr/include/x86_64-linux-musl/asm-generic && \
+    ln -s /usr/include/linux /usr/include/x86_64-linux-musl/linux
+
+# Build zlib (used in openssl and pq)
+RUN curl -sSL https://zlib.net/zlib-$ZLIB_VER.tar.gz | tar xz && \
+    cd zlib-$ZLIB_VER && \
+    CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure --static --prefix=$PREFIX && \
+    make -j$(nproc) && make install && \
+    cd .. && rm -rf zlib-$ZLIB_VER
+
+# Build openssl (used in curl and pq)
+# Would like to use zlib here, but can't seem to get it to work properly
+RUN curl -sSL https://www.openssl.org/source/old/1.0.2/openssl-$SSL_VER.tar.gz | tar xz && \
+    cd openssl-$SSL_VER && \
+    ./Configure no-zlib no-shared -fPIC --prefix=$PREFIX --openssldir=$PREFIX/ssl linux-x86_64 && \
+    env C_INCLUDE_PATH=$PREFIX/include make depend 2> /dev/null && \
+    make -j$(nproc) && make install && \
+    cd .. && rm -rf openssl-$SSL_VER
+
+# Build curl (needs with-zlib and all this stuff to allow https)
+# curl_LDFLAGS needed on stretch to avoid fPIC errors - though not sure from what
+RUN curl -sSL https://curl.se/download/curl-$CURL_VER.tar.gz | tar xz && \
+    cd curl-$CURL_VER && \
+    CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \
+      --enable-shared=no --with-zlib --enable-static=ssl --enable-optimize --prefix=$PREFIX \
+      --with-ca-path=/etc/ssl/certs/ --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt --without-ca-fallback \
+      --with-openssl && \
+    make -j$(nproc) curl_LDFLAGS="-all-static" && make install && \
+    cd .. && rm -rf curl-$CURL_VER
+
+# Build libpq
+RUN curl -sSL https://ftp.postgresql.org/pub/source/v$PQ_VER/postgresql-$PQ_VER.tar.gz | tar xz && \
+    cd postgresql-$PQ_VER && \
+    CC="musl-gcc -fPIE -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \
+    --without-readline \
+    --with-openssl \
+    --prefix=$PREFIX --host=x86_64-unknown-linux-musl && \
+    cd src/interfaces/libpq make -s -j$(nproc) all-static-lib && make -s install install-lib-static && \
+    cd ../../bin/pg_config && make -j $(nproc) && make install && \
+    cd .. && rm -rf postgresql-$PQ_VER

images/tests_runner.Dockerfile

@@ -0,0 +1,5 @@
+FROM rust:1.72
+
+RUN rustup target add x86_64-unknown-linux-musl
+RUN mkdir -p /tests && chmod 777 /tests
+ENTRYPOINT ["sleep", "3600"]

images/u_agent.Dockerfile

@@ -0,0 +1,5 @@
+FROM ubuntu:xenial
+
+RUN apt update && apt upgrade -y
+#todo: without this, request to 1.1.1.1 fails due to invalid cert (?), research more
+RUN apt install curl -y

images/u_db.Dockerfile

@@ -0,0 +1,8 @@
+FROM postgres:14.5
+
+RUN apt update && apt upgrade -y
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US.UTF-8
+RUN apt install -y locales locales-all iproute2

images/u_server.Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine:3.17
+
+RUN apk add iproute2 bash file