Merge pull request 'Payload table' (#9) from 8-payload-table into master

Reviewed-on: #9

.cargo/config.toml

@@ -1,6 +1,15 @@
 [build]
 rustflags = [
-    "-L", "/home/ortem/src/rust/unki/static/lib",
+    "-L/usr/lib/musl/lib",
+    "-L/home/ortem/src/rust/unki/static/lib",
     "--remap-path-prefix=/home/ortem/src/rust/unki=src",
     "--remap-path-prefix=/home/ortem/.cargo=cargo"
 ]
+target = "x86_64-unknown-linux-musl"
+
+[env]
+STATIC_PREFIX = "static"
+PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true"
+PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = { value = "static/bin/pg_config", relative = true }
+OPENSSL_STATIC = "true"
+OPENSSL_DIR = { value = "static", relative = true }

Cargo.toml

@@ -6,13 +6,15 @@ members = [
     "bin/u_run",
     "bin/u_server",
     "lib/u_lib",
-    "integration",
+    "integration-tests",
 ]
+resolver = "2"
 
 [workspace.dependencies]
 anyhow = "=1.0.63"
 deadpool-diesel = "0.4.0"
 diesel = { version = "2", features = ["postgres", "uuid"] }
+mime_guess = "2.0"
 openssl = "0.10"
 reqwest = { version = "0.11", features = ["json"] }
 serde = { version = "1.0", features = ["derive"] }

Makefile.toml

@@ -22,13 +22,6 @@ default_to_workspace = false
 [env]
 TARGET = "x86_64-unknown-linux-musl"
 CARGO = "cargo"
-ROOTDIR = "${CARGO_MAKE_WORKING_DIRECTORY}"
-STATIC_PREFIX = "${ROOTDIR}/static"
-PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true"
-PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = "${STATIC_PREFIX}/bin/pg_config"
-OPENSSL_STATIC = "true"
-OPENSSL_DIR = "${STATIC_PREFIX}"
-
 
 [tasks.build_static_libs]
 script = "./scripts/build_musl_libs.sh"
@@ -63,29 +56,41 @@ upx -9 $BINS
 '''
 
 [tasks.build]
-dependencies = ["cargo_update", "cargo_build", "release_tasks"]
+dependencies = ["cargo_build", "release_tasks"]
 clear = true
 
 [tasks.run]
 disabled = true
 
-[tasks.unit]
+[tasks.run_front]
+script = '''
+cd ./bin/u_panel/src/gui/fe
+ng serve
+'''
+
+[tasks.unit-tests]
 command = "${CARGO}"
 args = ["test", "--target", "${TARGET}", "--lib", "--", "${@}"]
 
-[tasks.integration]
+[tasks.ut]
+alias = "unit-tests"
+
+[tasks.integration-tests]
 dependencies = ["cargo_update"]
 script = '''
 [[ ! -d "./target/${TARGET}/${PROFILE_OVERRIDE}" ]] && echo 'No target folder. Build project first' && exit 1
-cd ./integration
+cd ./integration-tests
 bash integration_tests.sh ${@}
 '''
 
+[tasks.it]
+alias = "integration-tests"
+
 [tasks.test]
-dependencies = ["unit", "integration"]
+dependencies = ["unit", "integration-tests"]
 
 [tasks.gen_schema]
 script = './scripts/gen_schema.sh'
 
 [tasks.deploy]
-script = './scripts/deploy.sh'
+script = './scripts/deploy.sh'

bin/u_agent/Cargo.toml

@@ -12,5 +12,5 @@ reqwest = { workspace = true }
 sysinfo = "0.10.5"
 tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "time"] }
 uuid = { workspace = true }
-u_lib = { path = "../../lib/u_lib" }
+u_lib = { path = "../../lib/u_lib", features = ["agent"] }
 

bin/u_agent/src/lib.rs

@@ -5,24 +5,25 @@ use std::process::exit;
 use tokio::runtime::Builder;
 use tokio::time::{sleep, Duration};
 use u_lib::{
-    api::ClientHandler,
+    api::HttpClient,
     cache::JobCache,
     config::{get_self_id, EndpointsEnv, AGENT_ITERATION_INTERVAL},
     error::ErrChan,
-    executor::pop_completed,
-    jobs::{fat_meta_to_thin, AnonymousJobBatch},
+    executor,
+    jobs::AnonymousJobBatch,
     logging::init_logger,
     messaging::Reportable,
     models::AssignedJobById,
 };
 
-pub async fn process_request(jobs: Vec<AssignedJobById>, client: &ClientHandler) {
+async fn process_request(jobs: Vec<AssignedJobById>, client: &HttpClient) {
     if !jobs.is_empty() {
         for jr in &jobs {
             if !JobCache::contains(jr.job_id) {
                 info!("Fetching job: {}", &jr.job_id);
-                let fetched_job = loop {
-                    match client.get_job(jr.job_id).await {
+                let mut fetched_job = loop {
+                    //todo: use payload cache
+                    match client.get_full_job(jr.job_id).await {
                         Ok(result) => break result,
                         Err(err) => {
                             debug!("{:?} \nretrying...", err);
@@ -30,10 +31,12 @@ pub async fn process_request(jobs: Vec<AssignedJobById>, client: &ClientHandler)
                         }
                     }
                 };
-                match fat_meta_to_thin(fetched_job) {
-                    Ok(thin_meta) => JobCache::insert(thin_meta),
-                    Err(e) => ErrChan::send(e, "pld").await,
+                if let Some(payload) = &mut fetched_job.payload {
+                    if let Err(e) = payload.maybe_split_payload() {
+                        ErrChan::send(e, "pld").await;
+                    }
                 }
+                JobCache::insert(fetched_job);
             }
         }
         info!(
@@ -58,12 +61,12 @@ pub async fn process_request(jobs: Vec<AssignedJobById>, client: &ClientHandler)
     }
 }
 
-async fn error_reporting(client: ClientHandler) -> ! {
+async fn error_reporting(client: HttpClient) -> ! {
     loop {
         match ErrChan::recv().await {
             Some(err) => {
                 'retry: for _ in 0..3 {
-                    match client.report(Reportable::Error(err.clone())).await {
+                    match client.report([Reportable::Error(err.clone())]).await {
                         Ok(_) => break 'retry,
                         Err(e) => {
                             debug!("Reporting error: {:?}", e);
@@ -77,7 +80,7 @@ async fn error_reporting(client: ClientHandler) -> ! {
     }
 }
 
-async fn agent_loop(client: ClientHandler) -> ! {
+async fn agent_loop(client: HttpClient) -> ! {
     let self_id = get_self_id();
     loop {
         match client.get_personal_jobs(self_id).await {
@@ -87,7 +90,7 @@ async fn agent_loop(client: ClientHandler) -> ! {
             Err(err) => ErrChan::send(err, "processing").await,
         }
 
-        let result: Vec<Reportable> = pop_completed()
+        let result: Vec<Reportable> = executor::pop_completed()
             .await
             .into_iter()
             .map(|result| match result {
@@ -109,7 +112,7 @@ pub fn run_forever() -> ! {
     let env = EndpointsEnv::load();
 
     if cfg!(debug_assertions) {
-        init_logger(Some(format!(
+        let logfile_uid = format!(
             "u_agent-{}",
             get_self_id()
                 .hyphenated()
@@ -117,7 +120,8 @@ pub fn run_forever() -> ! {
                 .split("-")
                 .next()
                 .unwrap()
-        )));
+        );
+        init_logger(Some(&logfile_uid));
     } else {
         #[cfg(unix)]
         u_lib::unix::daemonize()
@@ -130,14 +134,14 @@ pub fn run_forever() -> ! {
         .build()
         .unwrap()
         .block_on(async {
-            match ClientHandler::new(&env.u_server, None).await {
+            match HttpClient::new(&env.u_server, None).await {
                 Ok(client) => {
                     tokio::spawn(error_reporting(client.clone()));
                     agent_loop(client).await
                 }
                 Err(e) => {
                     error!("client init failed: {}", e);
-                    exit(7)
+                    exit(7) // todo: wtf?
                 }
             }
         })

bin/u_panel/Cargo.toml

@@ -11,7 +11,7 @@ actix-cors = "0.6.1"
 actix-web = "4.1"
 anyhow = { workspace = true }
 futures-util = "0.3.21"
-mime_guess = "2.0.4"
+mime_guess = { workspace = true }
 once_cell = "1.8.0"
 rust-embed = { version = "6.3.0", features = ["debug-embed", "compression"] }
 serde = { workspace = true }

bin/u_panel/src/argparse.rs

@@ -1,124 +1,162 @@
 use serde_json::{from_str, to_value, Value};
 use structopt::StructOpt;
 use u_lib::{
-    api::ClientHandler,
-    messaging::AsMsg,
-    models::{Agent, AssignedJob, FatJobMeta},
-    types::Id,
-    types::PanelResult,
-    UError, UResult,
+    api::HttpClient, messaging::AsMsg, models::*, types::Id, types::PanelResult, UError, UResult,
 };
 
 #[derive(StructOpt, Debug)]
 pub struct Args {
     #[structopt(subcommand)]
     cmd: Cmd,
+    #[structopt(short, long, default_value)]
+    brief: Brief,
 }
 
 #[derive(StructOpt, Debug)]
 enum Cmd {
     Agents(RUD),
-    Jobs(JobCRUD),
-    Map(JobMapCRUD),
+    Jobs(CRUD),
+    Map(AssignedCRUD),
+    Payloads(PayloadCRUD),
     Ping,
     Serve,
 }
 
 #[derive(StructOpt, Debug)]
-enum JobCRUD {
+enum CRUD {
     Create {
-        job: String,
+        item: String,
     },
     #[structopt(flatten)]
     RUD(RUD),
 }
 
 #[derive(StructOpt, Debug)]
-enum JobCmd {
-    #[structopt(external_subcommand)]
-    Cmd(Vec<String>),
+enum AssignedCRUD {
+    Create {
+        item: String,
+    },
+    #[structopt(flatten)]
+    RUD(RUD),
 }
 
 #[derive(StructOpt, Debug)]
-enum JobMapCRUD {
+enum PayloadCRUD {
     Create {
-        #[structopt(parse(try_from_str = parse_uuid))]
-        agent_id: Id,
-
-        job_idents: Vec<String>,
+        item: String,
+    },
+    Read {
+        id: Option<String>,
+    },
+    Update {
+        item: String,
+    },
+    Delete {
+        #[structopt(parse(try_from_str = parse::uuid))]
+        id: Id,
     },
-    #[structopt(flatten)]
-    RUD(RUD),
 }
 
 #[derive(StructOpt, Debug)]
 enum RUD {
     Read {
-        #[structopt(parse(try_from_str = parse_uuid))]
+        #[structopt(parse(try_from_str = parse::uuid))]
         id: Option<Id>,
     },
     Update {
         item: String,
     },
     Delete {
-        #[structopt(parse(try_from_str = parse_uuid))]
+        #[structopt(parse(try_from_str = parse::uuid))]
         id: Id,
     },
 }
 
-fn parse_uuid(src: &str) -> Result<Id, String> {
-    Id::parse_str(src).map_err(|e| e.to_string())
+mod parse {
+    use super::*;
+
+    pub fn uuid(src: &str) -> Result<Id, String> {
+        Id::parse_str(src).map_err(|e| e.to_string())
+    }
 }
 
 pub fn into_value<M: AsMsg>(data: M) -> Value {
     to_value(data).unwrap()
 }
 
-pub async fn process_cmd(client: ClientHandler, args: Args) -> PanelResult<Value> {
+pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult<Value> {
     let catcher: UResult<Value> = (|| async {
         Ok(match args.cmd {
             Cmd::Agents(action) => match action {
                 RUD::Read { id } => into_value(client.get_agents(id).await?),
                 RUD::Update { item } => {
-                    let agent = from_str::<Agent>(&item)?;
-                    into_value(client.update_agent(agent).await?)
+                    let agent = from_str::<Agent>(&item)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), item))?;
+                    into_value(client.update_agent(&agent).await?)
                 }
                 RUD::Delete { id } => into_value(client.del(id).await?),
             },
             Cmd::Jobs(action) => match action {
-                JobCRUD::Create { job } => {
-                    let raw_job = from_str::<FatJobMeta>(&job)?;
-                    let mut job = raw_job.validated()?;
+                CRUD::Create { item: job } => {
+                    let raw_job = from_str::<RawJob>(&job)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), job))?;
+                    let mut job = raw_job.try_into_job()?;
 
                     if let Some(payload) = &mut job.payload {
-                        payload.read_into_self()?;
+                        payload.join_payload()?;
                     }
 
-                    into_value(client.upload_jobs(job).await?)
+                    into_value(client.upload_jobs([&job]).await?)
                 }
-                JobCRUD::RUD(RUD::Read { id }) => match id {
-                    //todo: use vec not to break frontend api, possibly refactor later
-                    Some(id) => into_value(vec![client.get_job(id).await?]),
+                CRUD::RUD(RUD::Read { id }) => match id {
+                    Some(id) => into_value(vec![client.get_job(id, args.brief).await?]),
                     None => into_value(client.get_jobs().await?),
                 },
-                JobCRUD::RUD(RUD::Update { item }) => {
-                    let raw_job = from_str::<FatJobMeta>(&item)?;
-                    let job = raw_job.validated()?;
-                    into_value(client.update_job(job).await?)
+                CRUD::RUD(RUD::Update { item }) => {
+                    let raw_job = from_str::<JobMeta>(&item)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), item))?;
+                    let job = raw_job.validate()?;
+
+                    // if let Some(payload) = &mut job.payload {
+                    //     payload.join_payload()?;
+                    // }
+
+                    into_value(client.update_job(&job).await?)
                 }
-                JobCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?),
+                CRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?),
             },
             Cmd::Map(action) => match action {
-                JobMapCRUD::Create {
-                    agent_id,
-                    job_idents,
-                } => into_value(client.set_jobs(agent_id, job_idents).await?),
-                JobMapCRUD::RUD(RUD::Read { id }) => into_value(client.get_agent_jobs(id).await?),
-                JobMapCRUD::RUD(RUD::Update { item }) => {
-                    let assigned = from_str::<AssignedJob>(&item)?;
-                    into_value(client.update_result(assigned).await?)
+                AssignedCRUD::Create { item } => {
+                    let payload = serde_json::from_str::<Vec<AssignedJobById>>(&item)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), item))?;
+                    into_value(client.assign_jobs(&payload).await?)
+                }
+                AssignedCRUD::RUD(RUD::Read { id }) => {
+                    into_value(client.get_assigned_jobs(id).await?)
+                }
+                AssignedCRUD::RUD(RUD::Update { item }) => {
+                    let assigned = from_str::<AssignedJob>(&item)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), item))?;
+                    into_value(client.update_result(&assigned).await?)
+                }
+                AssignedCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?),
+            },
+            Cmd::Payloads(action) => match action {
+                PayloadCRUD::Create { item } => {
+                    let payload = from_str::<RawPayload>(&item)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), item))?;
+                    into_value(client.upload_payload(&payload).await?)
+                }
+                PayloadCRUD::Read { id } => match id {
+                    None => into_value(client.get_payloads().await?),
+                    Some(id) => into_value(vec![client.get_payload(id, args.brief).await?]),
+                },
+                PayloadCRUD::Update { item } => {
+                    let payload = from_str::<Payload>(&item)
+                        .map_err(|e| UError::DeserializeError(e.to_string(), item))?;
+                    into_value(client.update_payload(&payload).await?)
                 }
-                JobMapCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?),
+                PayloadCRUD::Delete { id } => into_value(client.del(id).await?),
             },
             Cmd::Ping => into_value(client.ping().await?),
             Cmd::Serve => {

bin/u_panel/src/gui/fe/src/app/app-routing.module.ts

@@ -1,14 +1,13 @@
 import { NgModule } from '@angular/core';
 import { RouterModule, Routes } from '@angular/router';
-import { AgentComponent } from './core/tables/agent.component';
-import { JobComponent } from './core/tables/job.component';
-import { ResultComponent } from './core/tables/result.component';
-import { AgentInfoDialogComponent } from './core/tables/dialogs/agent_info.component';
+import { JobComponent, ResultComponent, AgentComponent, PayloadComponent } from './components/tables';
+//import { AgentInfoDialogComponent } from './core/tables/dialogs/agent-info-dialog.component';
 
 const routes: Routes = [
   { path: '', redirectTo: 'agents', pathMatch: 'full' },
   { path: 'agents', component: AgentComponent },
   { path: 'jobs', component: JobComponent },
+  { path: 'payloads', component: PayloadComponent },
   { path: 'results', component: ResultComponent },
 ];
 

bin/u_panel/src/gui/fe/src/app/app.component.html

@@ -2,4 +2,5 @@
   <a mat-tab-link *ngFor="let tab of tabs" routerLink={{tab.link}} routerLinkActive #rla="routerLinkActive"
     [active]="rla.isActive" [routerLinkActiveOptions]="{ exact: true }">{{tab.name}}</a>
 </nav>
-<router-outlet></router-outlet>
+<router-outlet></router-outlet>
+<global-error></global-error>

bin/u_panel/src/gui/fe/src/app/app.component.ts

@@ -1,4 +1,4 @@
-import { Component, ViewChild, AfterViewInit } from '@angular/core';
+import { Component } from '@angular/core';
 
 @Component({
   selector: 'app-root',
@@ -9,6 +9,7 @@ export class AppComponent {
   tabs = [
     { name: 'Agents', link: '/agents' },
     { name: 'Jobs', link: '/jobs' },
-    { name: 'Results', link: '/results' }
+    { name: 'Results', link: '/results' },
+    { name: 'Payloads', link: '/payloads' }
   ];
 }

bin/u_panel/src/gui/fe/src/app/app.module.ts

@@ -8,22 +8,27 @@ import { MatTableModule } from '@angular/material/table';
 import { MatFormFieldModule } from '@angular/material/form-field';
 import { MatButtonModule } from '@angular/material/button'
 import { MatInputModule } from '@angular/material/input';
+import { MatSelectModule } from '@angular/material/select';
 import { MatProgressSpinnerModule } from '@angular/material/progress-spinner';
 import { HttpClientModule } from '@angular/common/http';
 import { MatDialogModule } from '@angular/material/dialog';
 import { MatIconModule } from '@angular/material/icon';
 import { FormsModule } from '@angular/forms';
-import { AgentComponent, JobComponent, ResultComponent } from './core/tables';
+import { AgentComponent, JobComponent, ResultComponent, PayloadComponent } from './components/tables';
 import {
   AgentInfoDialogComponent,
   AssignJobDialogComponent,
   JobInfoDialogComponent,
-  ResultInfoDialogComponent
-} from './core/tables/dialogs';
+  ResultInfoDialogComponent,
+  PayloadInfoDialogComponent
+} from './components/dialogs';
 import { APP_BASE_HREF } from '@angular/common';
 import { MatTooltipModule } from '@angular/material/tooltip';
 import { MatSnackBarModule } from '@angular/material/snack-bar';
 import { MatListModule } from '@angular/material/list';
+import { GlobalErrorComponent } from './components/global-error/global-error.component';
+import { PayloadOverviewComponent } from './components/payload-overview/payload-overview.component';
+import { NewPayloadDialogComponent } from './components/dialogs/new-payload-dialog/new-payload-dialog.component';
 
 @NgModule({
   declarations: [
@@ -34,7 +39,12 @@ import { MatListModule } from '@angular/material/list';
     AgentInfoDialogComponent,
     JobInfoDialogComponent,
     ResultInfoDialogComponent,
-    AssignJobDialogComponent
+    AssignJobDialogComponent,
+    PayloadComponent,
+    PayloadInfoDialogComponent,
+    GlobalErrorComponent,
+    PayloadOverviewComponent,
+    NewPayloadDialogComponent
   ],
   imports: [
     BrowserModule,
@@ -50,6 +60,7 @@ import { MatListModule } from '@angular/material/list';
     MatIconModule,
     MatTooltipModule,
     MatSnackBarModule,
+    MatSelectModule,
     MatListModule,
     FormsModule,
     BrowserAnimationsModule

bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.ts

@@ -1,12 +1,12 @@
 import { Component, Inject } from '@angular/core';
 import { MAT_DIALOG_DATA } from '@angular/material/dialog';
-import { AgentModel } from '../../models/agent.model';
+import { AgentModel } from '../../../models/agent.model';
 import { EventEmitter } from '@angular/core';
 
 @Component({
     selector: 'agent-info-dialog',
-    templateUrl: 'agent-info-dialog.html',
-    styleUrls: ['info-dialog.component.less']
+    templateUrl: 'agent-info-dialog.component.html',
+    styleUrls: ['../base-info-dialog.component.less']
 })
 export class AgentInfoDialogComponent {
     is_preview = true;

bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts

@@ -0,0 +1,36 @@
+import { Component, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { AssignedJobByIdModel } from 'src/app/models';
+import { ApiTableService } from '../../../services';
+
+@Component({
+    selector: 'assign-job-dialog',
+    templateUrl: 'assign-job-dialog.component.html',
+    styleUrls: []
+})
+export class AssignJobDialogComponent {
+    rows: string[] = [];
+    selected_rows: string[] = [];
+
+    constructor(
+        @Inject(MAT_DIALOG_DATA) public agent_id: string,
+        private dataSource: ApiTableService,
+    ) {
+        dataSource.getJobs().subscribe(resp => {
+            this.rows = resp.map(j => `${j.id} ${j.alias}`)
+        })
+    }
+
+    assignSelectedJobs() {
+        const assigned_jobs: AssignedJobByIdModel[] = this.selected_rows.map(row => {
+            const job_id = row.split(' ', 1)[0];
+            return {
+                job_id: job_id,
+                agent_id: this.agent_id
+            }
+        });
+        this.dataSource.createResult(assigned_jobs).subscribe(_ => {
+            alert("Created")
+        });
+    }
+}

bin/u_panel/src/gui/fe/src/app/components/dialogs/index.ts

@@ -0,0 +1,5 @@
+export * from './agent-info-dialog/agent-info-dialog.component';
+export * from './result-info-dialog/result-info-dialog.component';
+export * from './job-info-dialog/job-info-dialog.component';
+export * from './assign-job-dialog/assign-job-dialog.component';
+export * from './payload-info-dialog/payload-info-dialog.component';

bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html

@@ -0,0 +1,48 @@
+<h2 mat-dialog-title *ngIf="isPreview">Job info</h2>
+<h2 mat-dialog-title *ngIf="!isPreview">Editing job info</h2>
+<mat-dialog-content>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field">
+            <mat-label>ID</mat-label>
+            <input matInput disabled value="{{data.meta.id}}">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Alias</mat-label>
+            <input matInput [readonly]="isPreview" [(ngModel)]="data.meta.alias">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Args</mat-label>
+            <input matInput [readonly]="isPreview" [(ngModel)]="data.meta.argv">
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Type</mat-label>
+            <input matInput [readonly]="isPreview" [(ngModel)]="data.meta.exec_type">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Platform</mat-label>
+            <input matInput [readonly]="isPreview" [(ngModel)]="data.meta.target_platforms">
+        </mat-form-field>
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Schedule</mat-label>
+            <input matInput [readonly]="isPreview" [(ngModel)]="data.meta.schedule">
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box">
+        <mat-form-field class="info-dlg-field">
+            <mat-label>Payload</mat-label>
+            <mat-select [disabled]="isPreview" [(value)]="data.meta.payload_id">
+                <mat-option *ngFor="let pld of allPayloads" [value]="pld[0]">{{ pld[1] }}</mat-option>
+            </mat-select>
+        </mat-form-field>
+    </div>
+    <div class="info-dialog-forms-box">
+        <payload-overview *ngIf="data.payload" [preview]="true" [payload]="data.payload.data"></payload-overview>
+    </div>
+</mat-dialog-content>
+<mat-dialog-actions align="end">
+    <button mat-raised-button *ngIf="isPreview" (click)="isPreview = false">Edit</button>
+    <button mat-raised-button *ngIf="!isPreview" (click)="updateJob()">Save</button>
+    <button mat-button mat-dialog-close>Cancel</button>
+</mat-dialog-actions>

bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts

@@ -0,0 +1,28 @@
+import { Component, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { EventEmitter } from '@angular/core';
+import { Job, JobModel } from '../../../models/job.model';
+import { ApiTableService } from 'src/app/services';
+
+@Component({
+    selector: 'job-info-dialog',
+    templateUrl: 'job-info-dialog.component.html',
+    styleUrls: ['../base-info-dialog.component.less']
+})
+export class JobInfoDialogComponent {
+    //[id, name]
+    isPreview = true;
+    allPayloads: [string | null, string][] = [[null, "none"]];
+
+    onSave = new EventEmitter<JobModel>();
+
+    constructor(@Inject(MAT_DIALOG_DATA) public data: Job, dataSource: ApiTableService) {
+        dataSource.getPayloads().subscribe(resp => {
+            this.allPayloads = this.allPayloads.concat(resp.map(r => [r.id, r.name]))
+        })
+    }
+
+    updateJob() {
+        this.onSave.emit(this.data.meta);
+    }
+}

bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.html

@@ -0,0 +1,20 @@
+<h2 mat-dialog-title>New payload</h2>
+<mat-dialog-content>
+    <div class="info-dialog-forms-box-smol">
+        <mat-form-field class="info-dlg-field" cdkFocusInitial>
+            <mat-label>Name</mat-label>
+            <input matInput [(ngModel)]="payload.name">
+        </mat-form-field>
+        <input type="file" class="file-input" (change)="onFileSelected($event)" #fileUpload>
+    </div>
+    <div class="info-dialog-forms-box">
+        <mat-form-field class="info-dlg-field" *ngIf="!uploadMode">
+            <mat-label>Data</mat-label>
+            <textarea matInput [(ngModel)]="decodedPayload"></textarea>
+        </mat-form-field>
+    </div>
+</mat-dialog-content>
+<mat-dialog-actions align="end">
+    <button mat-raised-button (click)="save()">Save</button>
+    <button mat-button mat-dialog-close>Close</button>
+</mat-dialog-actions>

bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.ts

@@ -0,0 +1,43 @@
+import { Component, EventEmitter, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { NewPayloadModel } from 'src/app/models/payload.model';
+
+@Component({
+  selector: 'new-payload-dialog',
+  templateUrl: 'new-payload-dialog.component.html',
+  styleUrls: ['../base-info-dialog.component.less']
+})
+export class NewPayloadDialogComponent {
+  decodedPayload = "";
+  uploadMode = false;
+  onSave = new EventEmitter<NewPayloadModel>();
+
+  constructor(@Inject(MAT_DIALOG_DATA) public payload: NewPayloadModel) { }
+
+  save() {
+    if (this.payload.data.length == 0) {
+      this.payload.data = Array.from(new TextEncoder().encode(this.decodedPayload));
+    }
+    this.onSave.emit(this.payload);
+  }
+
+  onFileSelected(event: any) {
+    const file: File = event.target.files[0];
+    if (file) {
+      this.uploadMode = true
+      const reader = new FileReader();
+      reader.onload = e => {
+        this.payload.name = file.name;
+        const result = e.target?.result;
+        if (result instanceof ArrayBuffer) {
+          const d = Array.from(new Uint8Array(result));
+          this.payload.data = d;
+          console.log(this.payload.data)
+        } else {
+          alert!("no file")
+        }
+      }
+      reader.readAsArrayBuffer(file)
+    }
+  }
+}

bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html

@@ -0,0 +1,34 @@
+<h2 mat-dialog-title *ngIf="isPreview">Payload</h2>
+<h2 mat-dialog-title *ngIf="!isPreview">Editing payload</h2>
+<mat-dialog-content>
+    <div class="info-dialog-forms-box">
+        <div class="info-dialog-forms-box-smol">
+            <mat-form-field class="info-dlg-field" cdkFocusInitial>
+                <mat-label>ID</mat-label>
+                <input matInput disabled value="{{payload.id}}">
+            </mat-form-field>
+            <mat-form-field class="info-dlg-field">
+                <mat-label>Name</mat-label>
+                <input matInput [readonly]="isPreview" [(ngModel)]="payload.name">
+            </mat-form-field>
+        </div>
+        <div class="info-dialog-forms-box-smol">
+            <mat-form-field class="info-dlg-field">
+                <mat-label>MIME-type</mat-label>
+                <input matInput disabled value="{{payload.mime_type}}">
+            </mat-form-field>
+            <mat-form-field class="info-dlg-field">
+                <mat-label>Size</mat-label>
+                <input matInput disabled value="{{payload.size}}">
+            </mat-form-field>
+        </div>
+    </div>
+    <div class="info-dialog-forms-box">
+        <payload-overview [preview]="isPreview" [payload]="payload.data"></payload-overview>
+    </div>
+</mat-dialog-content>
+<mat-dialog-actions align="end">
+    <button mat-raised-button *ngIf="isPreview" (click)="isPreview = false">Edit</button>
+    <button mat-raised-button *ngIf="!isPreview" (click)="updatePayload()">Save</button>
+    <button mat-button mat-dialog-close>Close</button>
+</mat-dialog-actions>

bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts

@@ -0,0 +1,20 @@
+import { Component, EventEmitter, Inject } from '@angular/core';
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { PayloadModel } from 'src/app/models/payload.model';
+
+@Component({
+  selector: 'payload-info-dialog',
+  templateUrl: 'payload-info-dialog.component.html',
+  styleUrls: ['../base-info-dialog.component.less']
+})
+export class PayloadInfoDialogComponent {
+  isPreview = true;
+
+  onSave = new EventEmitter<PayloadModel>();
+
+  constructor(@Inject(MAT_DIALOG_DATA) public payload: PayloadModel) { }
+
+  updatePayload() {
+    this.onSave.emit(this.payload);
+  }
+}

bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.html

@@ -39,13 +39,7 @@
         </mat-form-field>
     </div>
     <div class="info-dialog-forms-box">
-        <p>
-            <mat-form-field class="info-dlg-field">
-                <mat-label>Result</mat-label>
-                <textarea matInput cdkTextareaAutosize readonly value="{{decodedResult}}">
-                </textarea>
-            </mat-form-field>
-        </p>
+        <payload-overview [preview]="true" [payload]="data.result"></payload-overview>
     </div>
 </mat-dialog-content>
 <mat-dialog-actions align="end">

bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.ts

@@ -1,11 +1,11 @@
 import { Component, Inject } from '@angular/core';
 import { MAT_DIALOG_DATA } from '@angular/material/dialog';
-import { ResultModel } from '../../models/result.model';
+import { ResultModel } from '../../../models/result.model';
 
 @Component({
     selector: 'result-info-dialog',
-    templateUrl: 'result-info-dialog.html',
-    styleUrls: ['info-dialog.component.less']
+    templateUrl: 'result-info-dialog.component.html',
+    styleUrls: ['../base-info-dialog.component.less']
 })
 export class ResultInfoDialogComponent {
     decodedResult: string;

bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.ts

@@ -0,0 +1,34 @@
+import { Component, OnInit } from '@angular/core';
+import { MatSnackBar, MatSnackBarConfig } from '@angular/material/snack-bar';
+import { ErrorService } from 'src/app/services/error.service';
+
+@Component({
+  selector: 'global-error',
+  templateUrl: './global-error.component.html',
+  styleUrls: ['./global-error.component.less']
+})
+export class GlobalErrorComponent implements OnInit {
+
+  constructor(
+    private snackBar: MatSnackBar,
+    private errorService: ErrorService
+  ) { }
+
+  ngOnInit() {
+    this.errorService.error$.subscribe(err => {
+      const _config = (duration: number): MatSnackBarConfig => {
+        return {
+          horizontalPosition: 'right',
+          verticalPosition: 'bottom',
+          duration
+        }
+      }
+      const error = true;
+      const cfg = error ? _config(0) : _config(2000)
+
+      if (err != '') {
+        this.snackBar.open(err, 'Ok', cfg)
+      }
+    })
+  }
+}

bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html

@@ -0,0 +1,6 @@
+<mat-form-field class="info-dlg-field" floatLabel="always">
+    <mat-label>Payload data</mat-label>
+    <textarea matInput cdkTextareaAutosize *ngIf="!isTooBigPayload" [readonly]="isPreview" [(ngModel)]="decodedPayload">
+    </textarea>
+    <input matInput *ngIf="isTooBigPayload" disabled placeholder="Payload is too big to display">
+</mat-form-field>

bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts

@@ -0,0 +1,21 @@
+import { Component, Input, OnInit } from '@angular/core';
+
+@Component({
+  selector: 'payload-overview',
+  templateUrl: './payload-overview.component.html',
+  styleUrls: ['./payload-overview.component.less']
+})
+export class PayloadOverviewComponent implements OnInit {
+  @Input() payload: number[] | null = null;
+  @Input("preview") isPreview = true;
+  isTooBigPayload = false;
+  decodedPayload = "";
+
+  ngOnInit() {
+    if (this.payload !== null) {
+      this.decodedPayload = new TextDecoder().decode(new Uint8Array(this.payload))
+    } else {
+      this.isTooBigPayload = true
+    }
+  }
+}

bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.html

@@ -6,7 +6,7 @@
         </div>
         <mat-form-field appearance="standard">
             <mat-label>Filter</mat-label>
-            <input matInput (keyup)="apply_filter($event)" #input>
+            <input matInput (keyup)="applyFilter($event)" #input>
         </mat-form-field>
         <button id="refresh_btn" mat-raised-button color="primary" (click)="loadTableData()">Refresh</button>
 

bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts

@@ -0,0 +1,42 @@
+import { Component, OnInit } from '@angular/core';
+import { TableComponent } from '../base-table/base-table.component';
+import { AgentModel, Area } from '../../../models';
+import { AssignJobDialogComponent, AgentInfoDialogComponent } from '../../dialogs';
+
+@Component({
+  selector: 'agent-table',
+  templateUrl: './agent-table.component.html',
+  styleUrls: ['../base-table/base-table.component.less'],
+})
+export class AgentComponent extends TableComponent<AgentModel> implements OnInit {
+  area = 'agents' as Area
+  displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions']
+
+  showItemDialog(id: string) {
+    this.dataSource.getAgent(id).subscribe(resp => {
+      const dialog = this.infoDialog.open(AgentInfoDialogComponent, {
+        data: resp,
+        width: '1000px',
+      });
+
+      const saveSub = dialog.componentInstance.onSave.subscribe(result => {
+        this.dataSource.updateAgent(result).subscribe(_ => {
+          alert('Saved')
+          this.loadTableData()
+        })
+      })
+
+      dialog.afterClosed().subscribe(result => {
+        saveSub.unsubscribe()
+        this.router.navigate(['.'], { relativeTo: this.route })
+      })
+    })
+  }
+
+  assignJobs(id: string) {
+    const dialog = this.infoDialog.open(AssignJobDialogComponent, {
+      data: id,
+      width: '1000px',
+    });
+  }
+}

bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts

@@ -0,0 +1,59 @@
+import { OnInit, Directive, Component } from '@angular/core';
+import { ApiTableService } from '../../..';
+import { MatTableDataSource } from '@angular/material/table';
+import { MatDialog } from '@angular/material/dialog';
+import { ApiModel, Area } from '../../../models';
+import { ActivatedRoute, Router } from '@angular/router';
+
+@Directive()
+export abstract class TableComponent<T extends ApiModel> implements OnInit {
+  abstract area: Area;
+  table_data: MatTableDataSource<T> = new MatTableDataSource;
+  isLoadingResults = true;
+
+  constructor(
+    public dataSource: ApiTableService,
+    public infoDialog: MatDialog,
+    public route: ActivatedRoute,
+    public router: Router,
+  ) { }
+
+  ngOnInit() {
+    this.loadTableData();
+    this.route.queryParams.subscribe(params => {
+      const id = params['id']
+      const new_item = params['new']
+      if (id) {
+        this.showItemDialog(id);
+      }
+      if (new_item) {
+        this.showItemDialog(null);
+      }
+    })
+    //interval(10000).subscribe(_ => this.loadTableData());
+  }
+
+  loadTableData() {
+    this.isLoadingResults = true;
+
+    this.dataSource.getMany(this.area).subscribe(resp => {
+      this.isLoadingResults = false;
+      this.table_data.data = resp;
+    })
+  }
+
+  applyFilter(event: Event) {
+    const filterValue = (event.target as HTMLInputElement).value;
+    this.table_data.filter = filterValue.trim().toLowerCase();
+  }
+
+  deleteItem(id: string) {
+    if (confirm(`Delete ${id}?`)) {
+      this.dataSource.delete(id, this.area).subscribe(_ => { })
+      this.loadTableData()
+    }
+  }
+
+  abstract displayedColumns: string[];
+  abstract showItemDialog(id: string | null): void;
+}

bin/u_panel/src/gui/fe/src/app/components/tables/index.ts

@@ -0,0 +1,5 @@
+export * from './agent-table/agent-table.component';
+export * from './base-table/base-table.component';
+export * from './job-table/job-table.component';
+export * from './payload-table/payload-table.component';
+export * from './result-table/result-table.component';

bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.html

@@ -6,7 +6,7 @@
         </div>
         <mat-form-field appearance="standard">
             <mat-label>Filter</mat-label>
-            <input matInput (keyup)="apply_filter($event)" #input>
+            <input matInput (keyup)="applyFilter($event)" #input>
         </mat-form-field>
         <button id="refresh_btn" mat-raised-button color="basic" (click)="loadTableData()">Refresh</button>
         <button id="new_btn" mat-raised-button color="primary" routerLink='.' [queryParams]="{new: true}">Add
@@ -39,7 +39,7 @@
             <ng-container matColumnDef="platform">
                 <th mat-header-cell *matHeaderCellDef>Platform</th>
                 <td mat-cell *matCellDef="let row">
-                    {{row.platform}}
+                    {{row.target_platforms}}
                 </td>
             </ng-container>
 

bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.ts

@@ -0,0 +1,72 @@
+import { Component, OnInit } from '@angular/core';
+import { TableComponent } from '../base-table/base-table.component';
+import { Area, JobModel, Job } from '../../../models';
+import { JobInfoDialogComponent } from '../../dialogs';
+import { Observable } from 'rxjs';
+
+@Component({
+  selector: 'job-table',
+  templateUrl: './job-table.component.html',
+  styleUrls: ['../base-table/base-table.component.less'],
+  providers: [{ provide: 'area', useValue: 'jobs' }]
+})
+export class JobComponent extends TableComponent<JobModel> {
+  area = 'jobs' as Area;
+  displayedColumns = ['id', 'alias', 'platform', 'schedule', 'exec_type', 'actions']
+
+  showItemDialog(id: string | null) {
+    const is_new_job = id === null;
+
+    var dialogData$: Observable<Job>;
+
+    if (is_new_job) {
+      dialogData$ = new Observable(subscriber => {
+        var defaultJob: Job = {
+          meta: {
+            alias: null,
+            argv: '',
+            exec_type: 'shell',
+            target_platforms: '*',
+            payload_id: null,
+            schedule: null
+          },
+          payload: null
+        };
+        subscriber.next(defaultJob)
+      })
+    } else {
+      dialogData$ = this.dataSource.getJob(id)
+    }
+
+    dialogData$.subscribe(dialogData => {
+      const dialog = this.infoDialog.open(JobInfoDialogComponent, {
+        data: dialogData,
+        width: '1000px',
+      });
+
+      dialog.componentInstance.isPreview = !is_new_job;
+
+      const saveSub = dialog.componentInstance.onSave.subscribe(result => {
+        if (is_new_job) {
+          this.dataSource.create(dialogData.meta, this.area)
+            .subscribe(_ => {
+              alert("Created")
+              this.loadTableData()
+            })
+        } else {
+          this.dataSource.updateJob(result)
+            .subscribe(_ => {
+              alert("Updated")
+              this.loadTableData()
+            })
+        }
+        dialog.close()
+      })
+
+      dialog.afterClosed().subscribe(result => {
+        saveSub.unsubscribe()
+        this.router.navigate(['.'], { relativeTo: this.route })
+      })
+    })
+  }
+}

bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.html

@@ -0,0 +1,63 @@
+<div class="mat-elevation-z8">
+
+    <div class="table-container">
+        <div class="loading-shade" *ngIf="isLoadingResults">
+            <mat-spinner *ngIf="isLoadingResults"></mat-spinner>
+        </div>
+        <mat-form-field appearance="standard">
+            <mat-label>Filter</mat-label>
+            <input matInput (keyup)="applyFilter($event)" #input>
+        </mat-form-field>
+        <button id="refresh_btn" mat-raised-button color="primary" (click)="loadTableData()">Refresh</button>
+        <button id="new_btn" mat-raised-button color="primary" routerLink='.' [queryParams]="{new: true}">Add
+            payload</button>
+
+
+        <table mat-table fixedLayout="true" [dataSource]="table_data" class="data-table" matSort matSortActive="id"
+            matSortDisableClear matSortDirection="desc">
+
+            <ng-container matColumnDef="name">
+                <th mat-header-cell *matHeaderCellDef>Name</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.name}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="mime_type">
+                <th mat-header-cell *matHeaderCellDef>MIME-type</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.mime_type}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="size">
+                <th mat-header-cell *matHeaderCellDef>Size</th>
+                <td mat-cell *matCellDef="let row">
+                    {{row.size}}
+                </td>
+            </ng-container>
+
+            <ng-container matColumnDef="actions">
+                <th mat-header-cell *matHeaderCellDef></th>
+                <td mat-cell *matCellDef="let row">
+                    <button mat-icon-button routerLink='.' [queryParams]="{id: row.id}">
+                        <mat-icon>more_horiz</mat-icon>
+                    </button>
+                    |
+                    <button mat-icon-button (click)="deleteItem(row.id)">
+                        <mat-icon>delete</mat-icon>
+                    </button>
+                </td>
+            </ng-container>
+
+            <tr mat-header-row *matHeaderRowDef="displayedColumns"></tr>
+            <tr mat-row class="data-table-row" *matRowDef="let row; columns: displayedColumns;"></tr>
+            <tr class="mat-row" *matNoDataRow>
+                <td class="mat-cell">No data</td>
+            </tr>
+        </table>
+    </div>
+
+    <!-- <mat-paginator [length]="resultsLength" [pageSize]="30" aria-label="Select page of GitHub search results">
+    </mat-paginator> -->
+</div>

bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts

@@ -0,0 +1,67 @@
+import { Component } from '@angular/core';
+import { Area } from 'src/app/models';
+import { NewPayloadModel, PayloadModel } from 'src/app/models/payload.model';
+import { PayloadInfoDialogComponent } from '../../dialogs';
+import { NewPayloadDialogComponent } from '../../dialogs/new-payload-dialog/new-payload-dialog.component';
+import { TableComponent } from '../base-table/base-table.component';
+
+@Component({
+  selector: 'payload-table',
+  templateUrl: './payload-table.component.html',
+  styleUrls: ['../base-table/base-table.component.less'],
+  providers: [{ provide: 'area', useValue: 'payloads' }]
+})
+export class PayloadComponent extends TableComponent<PayloadModel> {
+  area = 'payloads' as Area
+  displayedColumns = ["name", "mime_type", "size", 'actions'];
+
+  showItemDialog(id: string | null) {
+    if (id === null) {
+      const payload: NewPayloadModel = {
+        name: "",
+        data: []
+      }
+      const dialog = this.infoDialog.open(NewPayloadDialogComponent, {
+        data: payload,
+        width: '1000px',
+      });
+
+      dialog.componentInstance.onSave.subscribe(result => {
+        this.dataSource.createPayload(result)
+          .subscribe(_ => {
+            alert("Created")
+            this.loadTableData()
+          })
+
+        dialog.close()
+      })
+
+      dialog.afterClosed().subscribe(_ => {
+        this.router.navigate(['.'], { relativeTo: this.route })
+      })
+
+    } else {
+      this.dataSource.getPayload(id as string).subscribe(resp => {
+        const dialog = this.infoDialog.open(PayloadInfoDialogComponent, {
+          data: resp,
+          width: '1000px',
+        });
+
+        const saveSub = dialog.componentInstance.onSave.subscribe(result => {
+          this.dataSource.updatePayload(result)
+            .subscribe(_ => {
+              alert("Updated")
+              this.loadTableData()
+            })
+          dialog.close()
+        })
+
+        dialog.afterClosed().subscribe(_ => {
+          saveSub.unsubscribe()
+          this.router.navigate(['.'], { relativeTo: this.route })
+        })
+      })
+    }
+  }
+
+}

bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.html

@@ -6,7 +6,7 @@
         </div>
         <mat-form-field appearance="standard">
             <mat-label>Filter</mat-label>
-            <input matInput (keyup)="apply_filter($event)" #input>
+            <input matInput (keyup)="applyFilter($event)" #input>
         </mat-form-field>
         <button id="refresh_btn" mat-raised-button color="primary" (click)="loadTableData()">Refresh</button>
 
@@ -49,7 +49,7 @@
             </ng-container>
 
             <ng-container matColumnDef="last_updated">
-                <th mat-header-cell *matHeaderCellDef>ID</th>
+                <th mat-header-cell *matHeaderCellDef>Last updated</th>
                 <td mat-cell *matCellDef="let row">
                     {{row.updated.secs_since_epoch * 1000| date:'long'}}
                 </td>

bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts

@@ -0,0 +1,36 @@
+import { Component, OnInit } from '@angular/core';
+import { TableComponent } from '../base-table/base-table.component';
+import { Area, ResultModel } from '../../../models';
+import { ResultInfoDialogComponent } from '../../dialogs';
+
+@Component({
+  selector: 'results-table',
+  templateUrl: './result-table.component.html',
+  styleUrls: ['../base-table/base-table.component.less'],
+  providers: [{ provide: 'area', useValue: 'map' }]
+})
+export class ResultComponent extends TableComponent<ResultModel> {
+  area = 'map' as Area
+  displayedColumns = [
+    'id',
+    'alias',
+    'agent_id',
+    'job_id',
+    'state',
+    'last_updated',
+    'actions'
+  ];
+
+  showItemDialog(id: string) {
+    this.dataSource.getResult(id).subscribe(resp => {
+      const dialog = this.infoDialog.open(ResultInfoDialogComponent, {
+        data: resp,
+        width: '1000px',
+      });
+
+      dialog.afterClosed().subscribe(_ => {
+        this.router.navigate(['.'], { relativeTo: this.route })
+      })
+    })
+  }
+}

bin/u_panel/src/gui/fe/src/app/core/models/index.ts

@@ -1,14 +0,0 @@
-export * from './agent.model';
-export * from './result.model';
-export * from './job.model';
-
-export interface UTCDate {
-    secs_since_epoch: number,
-    nanos_since_epoch: number
-}
-
-export abstract class ApiModel { }
-
-export interface Empty extends ApiModel { }
-
-export type Area = "agents" | "jobs" | "map";

bin/u_panel/src/gui/fe/src/app/core/models/job.model.ts

@@ -1,11 +0,0 @@
-import { ApiModel } from ".";
-
-export interface JobModel extends ApiModel {
-    alias: string,
-    argv: string,
-    id: string,
-    exec_type: string,
-    platform: string,
-    payload: number[] | null,
-    schedule: string | null,
-}

bin/u_panel/src/gui/fe/src/app/core/services/api.service.ts

@@ -1,53 +0,0 @@
-import { environment } from 'src/environments/environment';
-import { HttpClient } from '@angular/common/http';
-import { firstValueFrom } from 'rxjs';
-import { ApiModel, Empty, Area } from '../models';
-
-interface ServerResponse<T extends ApiModel> {
-  status: "ok" | "err",
-  data: T | string
-}
-
-export class ApiTableService<T extends ApiModel> {
-  area: Area;
-
-  constructor(private http: HttpClient, area: Area) {
-    this.area = area;
-  }
-
-  requestUrl = `${environment.server}/cmd/`;
-
-  async req<R extends ApiModel>(cmd: string): Promise<ServerResponse<R>> {
-    return await firstValueFrom(this.http.post<ServerResponse<R>>(this.requestUrl, cmd))
-  }
-
-  async getOne(id: string, area: string = this.area): Promise<ServerResponse<T>> {
-    const resp = await this.req<T[]>(`${area} read ${id}`)
-    if (resp.data.length === 0) {
-      return {
-        status: 'err',
-        data: `${id} not found in ${area}`
-      }
-    }
-    return {
-      status: resp.status,
-      data: resp.data[0]
-    }
-  }
-
-  async getMany(): Promise<ServerResponse<T[]>> {
-    return await this.req(`${this.area} read`)
-  }
-
-  async update(item: T): Promise<ServerResponse<Empty>> {
-    return await this.req(`${this.area} update '${JSON.stringify(item)}'`)
-  }
-
-  async delete(id: string): Promise<ServerResponse<Empty>> {
-    return await this.req(`${this.area} delete ${id}`)
-  }
-
-  async create(item: string): Promise<ServerResponse<string[]>> {
-    return await this.req(`${this.area} create ${item}`)
-  }
-}

bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.ts

@@ -1,52 +0,0 @@
-import { Component, OnDestroy, OnInit } from '@angular/core';
-import { TablesComponent } from './table.component';
-import { AgentModel } from '../models';
-import { AgentInfoDialogComponent } from './dialogs/agent_info.component';
-import { HttpErrorResponse } from '@angular/common/http';
-import { AssignJobDialogComponent } from './dialogs';
-
-@Component({
-  selector: 'agent-table',
-  templateUrl: './agent.component.html',
-  styleUrls: ['./table.component.less']
-})
-export class AgentComponent extends TablesComponent<AgentModel> implements OnInit {
-
-  //dialogSubscr!: Subscription;
-  area = 'agents' as const;
-
-  displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions']
-
-  show_item_dialog(id: string) {
-    this.data_source!.getOne(id).then(resp => {
-      if (resp.status === 'ok') {
-        const dialog = this.infoDialog.open(AgentInfoDialogComponent, {
-          data: resp.data as AgentModel,
-          width: '1000px',
-        });
-
-        const saveSub = dialog.componentInstance.onSave.subscribe(result => {
-          this.data_source!.update(result).then(_ => {
-            this.openSnackBar('Saved', false)
-            this.loadTableData()
-          })
-            .catch((err: HttpErrorResponse) => this.openSnackBar(err.error))
-        })
-
-        dialog.afterClosed().subscribe(result => {
-          saveSub.unsubscribe()
-          this.router.navigate(['.'], { relativeTo: this.route })
-        })
-      } else {
-        this.openSnackBar(resp.data)
-      }
-    }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error))
-  }
-
-  assignJobs(id: string) {
-    const dialog = this.infoDialog.open(AssignJobDialogComponent, {
-      data: id,
-      width: '1000px',
-    });
-  }
-}

bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/assign_job.component.ts

@@ -1,33 +0,0 @@
-import { Component, Inject } from '@angular/core';
-import { MAT_DIALOG_DATA } from '@angular/material/dialog';
-import { HttpClient } from '@angular/common/http';
-import { ApiTableService } from '../../services';
-import { JobModel } from '../../models';
-import { MatListOption } from '@angular/material/list';
-
-@Component({
-    selector: 'assign-job-dialog',
-    templateUrl: 'assign-job-dialog.html',
-    styleUrls: []
-})
-export class AssignJobDialogComponent {
-    rows: string[] = [];
-    selected_rows: string[] = [];
-
-    constructor(@Inject(MAT_DIALOG_DATA) public agent_id: string, private http: HttpClient) {
-        new ApiTableService(http, "jobs").getMany().then(result => {
-            if (result.status == "ok") {
-                const jobs = result.data as JobModel[]
-                this.rows = jobs.map(j => `${j.id} ${j.alias}`)
-            } else {
-                alert(result.data as string)
-            }
-        }).catch(err => alert(err))
-    }
-
-    assignSelectedJobs() {
-        const job_ids = this.selected_rows.map(row => row.split(' ', 1)[0]).join(' ');
-        const request = `${this.agent_id} ${job_ids}`
-        new ApiTableService(this.http, "map").create(request).catch(err => alert(err))
-    }
-}

bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/index.ts

@@ -1,4 +0,0 @@
-export * from './agent_info.component';
-export * from './result_info.component';
-export * from './job_info.component';
-export * from './assign_job.component';

bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job-info-dialog.html

@@ -1,44 +0,0 @@
-<h2 mat-dialog-title *ngIf="is_preview">Job info</h2>
-<h2 mat-dialog-title *ngIf="!is_preview">Editing job info</h2>
-<mat-dialog-content>
-    <div class="info-dialog-forms-box-smol">
-        <mat-form-field class="info-dlg-field" cdkFocusInitial>
-            <mat-label>ID</mat-label>
-            <input matInput disabled value="{{data.id}}">
-        </mat-form-field>
-        <mat-form-field class="info-dlg-field">
-            <mat-label>Alias</mat-label>
-            <input matInput [readonly]="is_preview" [(ngModel)]="data.alias">
-        </mat-form-field>
-        <mat-form-field class="info-dlg-field">
-            <mat-label>Args</mat-label>
-            <input matInput [readonly]="is_preview" [(ngModel)]="data.argv">
-        </mat-form-field>
-    </div>
-    <div class="info-dialog-forms-box-smol">
-        <mat-form-field class="info-dlg-field">
-            <mat-label>Type</mat-label>
-            <input matInput [readonly]="is_preview" [(ngModel)]="data.exec_type">
-        </mat-form-field>
-        <mat-form-field class="info-dlg-field">
-            <mat-label>Platform</mat-label>
-            <input matInput [readonly]="is_preview" [(ngModel)]="data.platform">
-        </mat-form-field>
-        <mat-form-field class="info-dlg-field">
-            <mat-label>Schedule</mat-label>
-            <input matInput [readonly]="is_preview" [(ngModel)]="data.schedule">
-        </mat-form-field>
-    </div>
-    <div class="info-dialog-forms-box">
-        <mat-form-field class="info-dlg-field">
-            <mat-label>Payload</mat-label>
-            <textarea matInput cdkTextareaAutosize [readonly]="is_preview" [(ngModel)]="decodedPayload">
-                </textarea>
-        </mat-form-field>
-    </div>
-</mat-dialog-content>
-<mat-dialog-actions align="end">
-    <button mat-raised-button *ngIf="is_preview" (click)="is_preview = false">Edit</button>
-    <button mat-raised-button *ngIf="!is_preview" (click)="updateJob()">Save</button>
-    <button mat-button mat-dialog-close>Cancel</button>
-</mat-dialog-actions>

bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job_info.component.ts

@@ -1,30 +0,0 @@
-import { Component, Inject } from '@angular/core';
-import { MAT_DIALOG_DATA } from '@angular/material/dialog';
-import { EventEmitter } from '@angular/core';
-import { JobModel } from '../../models/job.model';
-
-@Component({
-    selector: 'job-info-dialog',
-    templateUrl: 'job-info-dialog.html',
-    styleUrls: ['info-dialog.component.less']
-})
-export class JobInfoDialogComponent {
-    is_preview = true;
-    decodedPayload: string;
-    onSave = new EventEmitter();
-
-    constructor(@Inject(MAT_DIALOG_DATA) public data: JobModel) {
-        if (data.payload !== null) {
-            this.decodedPayload = new TextDecoder().decode(new Uint8Array(data.payload))
-        } else {
-            this.decodedPayload = ""
-        }
-    }
-
-    updateJob() {
-        if (this.decodedPayload.length > 0) {
-            this.data.payload = Array.from(new TextEncoder().encode(this.decodedPayload))
-        }
-        this.onSave.emit(this.data);
-    }
-}

bin/u_panel/src/gui/fe/src/app/core/tables/index.ts

@@ -1,3 +0,0 @@
-export * from './agent.component';
-export * from './job.component';
-export * from './result.component';

bin/u_panel/src/gui/fe/src/app/core/tables/job.component.ts

@@ -1,59 +0,0 @@
-import { Component, OnInit } from '@angular/core';
-import { TablesComponent } from './table.component';
-import { JobModel } from '../models';
-import { JobInfoDialogComponent } from './dialogs';
-import { HttpErrorResponse } from '@angular/common/http';
-
-@Component({
-  selector: 'job-table',
-  templateUrl: './job.component.html',
-  styleUrls: ['./table.component.less']
-})
-export class JobComponent extends TablesComponent<JobModel> {
-  area = 'jobs' as const;
-  displayedColumns = ['id', 'alias', 'platform', 'schedule', 'exec_type', 'actions']
-
-  show_item_dialog(id: string | null) {
-    const show_dlg = (id: string, edit: boolean) => {
-      this.data_source!.getOne(id).then(resp => {
-        if (resp.status === 'ok') {
-          var dialog = this.infoDialog.open(JobInfoDialogComponent, {
-            data: resp.data as JobModel,
-            width: '1000px',
-          });
-          if (edit) {
-            dialog.componentInstance.is_preview = false
-          }
-
-          const saveSub = dialog.componentInstance.onSave.subscribe(result => {
-            this.data_source!.update(result)
-              .then(_ => {
-                this.openSnackBar("Saved", false)
-                this.loadTableData()
-              })
-              .catch((err: HttpErrorResponse) => this.openSnackBar(err.error))
-          })
-
-          dialog.afterClosed().subscribe(result => {
-            saveSub.unsubscribe()
-            this.router.navigate(['.'], { relativeTo: this.route })
-          })
-        } else {
-          this.openSnackBar(resp.data)
-        }
-      }).catch((err: any) => this.openSnackBar(err))
-    }
-
-    if (id) {
-      show_dlg(id, false)
-    } else {
-      this.data_source!.create('"{}"').then(resp => {
-        if (resp.status === 'ok') {
-          show_dlg(resp.data[0], true)
-        } else {
-          this.openSnackBar(resp.data)
-        }
-      }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error))
-    }
-  }
-}

bin/u_panel/src/gui/fe/src/app/core/tables/result.component.ts

@@ -1,41 +0,0 @@
-import { Component, OnInit } from '@angular/core';
-import { TablesComponent } from './table.component';
-import { ResultModel } from '../models';
-import { ResultInfoDialogComponent } from './dialogs';
-import { HttpErrorResponse } from '@angular/common/http';
-
-@Component({
-  selector: 'results-table',
-  templateUrl: './result.component.html',
-  styleUrls: ['./table.component.less']
-})
-export class ResultComponent extends TablesComponent<ResultModel> {
-  area = 'map' as const;
-
-  displayedColumns = [
-    'id',
-    'alias',
-    'agent_id',
-    'job_id',
-    'state',
-    'last_updated',
-    'actions'
-  ];
-
-  show_item_dialog(id: string) {
-    this.data_source!.getOne(id).then(resp => {
-      if (resp.status === 'ok') {
-        const dialog = this.infoDialog.open(ResultInfoDialogComponent, {
-          data: resp.data as ResultModel,
-          width: '1000px',
-        });
-
-        dialog.afterClosed().subscribe(result => {
-          this.router.navigate(['.'], { relativeTo: this.route })
-        })
-      } else {
-        this.openSnackBar(resp.data)
-      }
-    }).catch((err: HttpErrorResponse) => this.openSnackBar(err.message))
-  }
-}

bin/u_panel/src/gui/fe/src/app/core/tables/table.component.ts

@@ -1,84 +0,0 @@
-import { OnInit, Directive } from '@angular/core';
-import { HttpClient } from '@angular/common/http';
-import { ApiTableService } from '../';
-import { MatTableDataSource } from '@angular/material/table';
-import { MatDialog } from '@angular/material/dialog';
-import { ApiModel, Area } from '../models';
-import { ActivatedRoute, Router } from '@angular/router';
-import { interval } from 'rxjs';
-import { MatSnackBar, MatSnackBarConfig } from '@angular/material/snack-bar';
-
-@Directive()
-export abstract class TablesComponent<T extends ApiModel> implements OnInit {
-  abstract area: Area;
-  data_source!: ApiTableService<T>;
-  table_data!: MatTableDataSource<T>;
-
-  isLoadingResults = true;
-
-  constructor(
-    public httpClient: HttpClient,
-    public infoDialog: MatDialog,
-    public route: ActivatedRoute,
-    public router: Router,
-    public snackBar: MatSnackBar
-  ) {
-    this.table_data = new MatTableDataSource;
-  }
-
-  ngOnInit() {
-    this.data_source = new ApiTableService(this.httpClient, this.area);
-    this.loadTableData();
-    this.route.queryParams.subscribe(params => {
-      const id = params['id']
-      const new_agent = params['new']
-      if (id) {
-        this.show_item_dialog(id);
-      }
-      if (new_agent) {
-        this.show_item_dialog(null);
-      }
-    })
-    //interval(10000).subscribe(_ => this.loadTableData());
-  }
-
-  async loadTableData() {
-    this.isLoadingResults = true;
-    //possibly needs try/catch
-    const data = await this.data_source!.getMany();
-    this.isLoadingResults = false;
-
-    if (typeof data.data !== 'string') {
-      this.table_data.data = data.data
-    } else {
-      alert(`Error: ${data}`)
-    };
-  }
-
-  apply_filter(event: Event) {
-    const filterValue = (event.target as HTMLInputElement).value;
-    this.table_data.filter = filterValue.trim().toLowerCase();
-  }
-
-  deleteItem(id: string) {
-    if (confirm(`Delete ${id}?`)) {
-      this.data_source!.delete(id).catch(this.openSnackBar)
-    }
-  }
-
-  openSnackBar(message: any, error: boolean = true) {
-    const msg = JSON.stringify(message)
-    const _config = (duration: number): MatSnackBarConfig => {
-      return {
-        horizontalPosition: 'right',
-        verticalPosition: 'bottom',
-        duration
-      }
-    }
-    const cfg = error ? _config(0) : _config(2000)
-    this.snackBar.open(msg, 'Ok', cfg);
-  }
-
-  abstract displayedColumns: string[];
-  abstract show_item_dialog(id: string | null): void;
-}

bin/u_panel/src/gui/fe/src/app/models/agent.model.ts

@@ -1,6 +1,6 @@
-import { UTCDate, ApiModel } from ".";
+import { UTCDate } from ".";
 
-export interface AgentModel extends ApiModel {
+export interface AgentModel {
     alias: string | null,
     hostname: string,
     host_info: string,

bin/u_panel/src/gui/fe/src/app/models/index.ts

@@ -0,0 +1,20 @@
+import { AgentModel } from './agent.model';
+import { JobModel } from './job.model';
+import { PayloadModel } from './payload.model';
+import { ResultModel } from './result.model';
+
+export * from './agent.model';
+export * from './result.model';
+export * from './job.model';
+export * from './payload.model';
+
+export interface UTCDate {
+    secs_since_epoch: number,
+    nanos_since_epoch: number
+}
+
+export type Area = "agents" | "jobs" | "map" | "payloads";
+
+export type ApiModel = AgentModel | JobModel | ResultModel | PayloadModel | Empty;
+
+export interface Empty { }

bin/u_panel/src/gui/fe/src/app/models/job.model.ts

@@ -0,0 +1,16 @@
+import { PayloadModel } from './'
+
+export interface JobModel {
+    alias: string | null,
+    argv: string,
+    id?: string,
+    exec_type: string,
+    target_platforms: string,
+    payload_id: string | null,
+    schedule: string | null,
+}
+
+export interface Job {
+    meta: JobModel,
+    payload: PayloadModel | null,
+}

bin/u_panel/src/gui/fe/src/app/models/payload.model.ts

@@ -0,0 +1,12 @@
+export interface PayloadModel {
+    id: string,
+    mime_type: string,
+    name: string,
+    size: number,
+    data: number[] | null
+}
+
+export interface NewPayloadModel {
+    name: string,
+    data: number[]
+}

bin/u_panel/src/gui/fe/src/app/models/result.model.ts

@@ -1,13 +1,18 @@
-import { UTCDate, ApiModel } from ".";
+import { UTCDate } from ".";
 
-export interface ResultModel extends ApiModel {
+export interface ResultModel {
     agent_id: string,
     alias: string,
     created: UTCDate,
     id: string,
     job_id: string,
-    result: number[],
+    result: number[] | null,
     state: "Queued" | "Running" | "Finished",
     retcode: number | null,
     updated: UTCDate,
+}
+
+export interface AssignedJobByIdModel {
+    job_id: string,
+    agent_id: string
 }

bin/u_panel/src/gui/fe/src/app/services/api.service.ts

@@ -0,0 +1,142 @@
+import { environment } from 'src/environments/environment';
+import { HttpClient, HttpErrorResponse } from '@angular/common/http';
+import { Observable, map, catchError, throwError } from 'rxjs';
+import { ApiModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, Job, NewPayloadModel, AssignedJobByIdModel } from '../models';
+import { Injectable, Inject } from '@angular/core';
+import { ErrorService } from './error.service';
+
+type Status = "ok" | "err";
+
+interface ServerResponse<T extends ApiModel> {
+  status: Status,
+  data: T | string
+}
+
+@Injectable({
+  providedIn: 'root'
+})
+export class ApiTableService {
+
+  constructor(
+    private http: HttpClient,
+    private errorService: ErrorService
+  ) {
+  }
+
+  requestUrl = `${environment.server}/cmd/`;
+
+  req<R extends ApiModel>(cmd: string): Observable<ServerResponse<R>> {
+    return this.http.post<ServerResponse<R>>(this.requestUrl, cmd)
+  }
+
+  getOne<T extends ApiModel>(id: string, area: Area, brief: 'yes' | 'no' | 'auto' | null = null): Observable<T> {
+    const request = `${area} read ${id}` + (brief !== null ? `-b=${brief}` : '')
+    const resp = this.req<T[]>(request).pipe(
+      map(resp => {
+        if (resp.data.length === 0) {
+          return {
+            status: 'err' as Status,
+            data: `${id} not found in ${area}`
+          }
+        }
+        return {
+          status: resp.status,
+          data: resp.data[0]
+        }
+      }));
+    return this.filterErrStatus(resp)
+  }
+
+  getAgent(id: string): Observable<AgentModel> {
+    return this.getOne(id, 'agents')
+  }
+
+  getJob(id: string): Observable<Job> {
+    return this.getOne(id, 'jobs')
+  }
+
+  getResult(id: string): Observable<ResultModel> {
+    return this.getOne(id, 'map')
+  }
+
+  getPayload(id: string): Observable<PayloadModel> {
+    return this.getOne(id, 'payloads')
+  }
+
+  getMany(area: Area): Observable<any[]> {
+    return this.filterErrStatus(this.req(`${area} read`))
+  }
+
+  getAgents(): Observable<AgentModel[]> {
+    return this.getMany('agents')
+  }
+
+  getJobs(): Observable<JobModel[]> {
+    return this.getMany('jobs')
+  }
+
+  getResults(): Observable<ResultModel[]> {
+    return this.getMany('map')
+  }
+
+  getPayloads(): Observable<PayloadModel[]> {
+    return this.getMany('payloads')
+  }
+
+  update<T extends ApiModel>(item: T, area: Area): Observable<Empty> {
+    return this.filterErrStatus(this.req(`${area} update '${JSON.stringify(item)}'`))
+  }
+
+  updateAgent(item: AgentModel): Observable<Empty> {
+    return this.update(item, 'agents')
+  }
+
+  updateJob(item: JobModel): Observable<Empty> {
+    return this.update(item, 'jobs')
+  }
+
+  updateResult(item: ResultModel): Observable<Empty> {
+    return this.update(item, 'map')
+  }
+
+  updatePayload(item: PayloadModel): Observable<Empty> {
+    return this.update(item, 'payloads')
+  }
+
+  delete(id: string, area: Area): Observable<Empty> {
+    return this.filterErrStatus(this.req(`${area} delete ${id}`))
+  }
+
+  create<T extends ApiModel>(item: T | null, area: Area): Observable<string[]> {
+    var serialized = '"{}"'
+    if (item) {
+      serialized = JSON.stringify(item);
+    }
+    return this.filterErrStatus(this.req(`${area} create '${serialized}'`))
+  }
+
+  createResult(item: AssignedJobByIdModel[]): Observable<string[]> {
+    return this.create(item, 'map')
+  }
+
+  createPayload(item: NewPayloadModel): Observable<string[]> {
+    return this.create(item, 'payloads')
+  }
+
+  filterErrStatus<R extends ApiModel>(obs: Observable<ServerResponse<R>>): Observable<R> {
+    return obs.pipe(
+      map(r => {
+        if (r.status == 'err') {
+          throw new Error(r.data as string)
+        }
+        return r.data as R
+      }),
+      catchError(this.errorHandler.bind(this)))
+  }
+
+  errorHandler(err: HttpErrorResponse, caught: any) {
+    var error = err.error.data !== undefined ? JSON.stringify(err.error.data) : err.message;
+    this.errorService.handle(error);
+    return throwError(() => new Error());
+  }
+}

bin/u_panel/src/gui/fe/src/app/services/error.service.ts

@@ -0,0 +1,17 @@
+import { Injectable } from '@angular/core';
+import { Subject } from 'rxjs';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class ErrorService {
+  error$ = new Subject<string>();
+
+  handle(msg: string) {
+    this.error$.next(msg)
+  }
+
+  clear() {
+    this.handle('')
+  }
+}

bin/u_panel/src/gui/mod.rs

@@ -8,7 +8,7 @@ use futures_util::StreamExt;
 use rust_embed::RustEmbed;
 use std::borrow::Cow;
 use structopt::StructOpt;
-use u_lib::{api::ClientHandler, unwrap_enum};
+use u_lib::{api::HttpClient, unwrap_enum};
 
 #[derive(RustEmbed)]
 #[folder = "./src/gui/fe/dist/fe/"]
@@ -42,7 +42,7 @@ async fn resources_adapter(path: web::Path<(String,)>) -> impl Responder {
 #[post("/cmd/")]
 async fn send_cmd(
     mut body: web::Payload,
-    client: web::Data<ClientHandler>,
+    client: web::Data<HttpClient>,
 ) -> Result<impl Responder, Error> {
     let mut bytes = web::BytesMut::new();
 
@@ -65,16 +65,14 @@ async fn send_cmd(
 
     let response = if result.is_ok() {
         HttpResponse::Ok().body(result_string)
-    } else if result.is_err() {
-        HttpResponse::BadRequest().body(result_string)
     } else {
-        unreachable!()
+        HttpResponse::BadRequest().body(result_string)
     };
 
     Ok(response)
 }
 
-pub async fn serve(client: ClientHandler) -> anyhow::Result<()> {
+pub async fn serve(client: HttpClient) -> anyhow::Result<()> {
     info!("Connecting to u_server...");
     client.ping().await?;
 

bin/u_panel/src/main.rs

@@ -7,19 +7,19 @@ extern crate tracing;
 use anyhow::Result as AnyResult;
 use argparse::{process_cmd, Args};
 use structopt::StructOpt;
-use u_lib::api::ClientHandler;
+use u_lib::api::HttpClient;
 use u_lib::config::AccessEnv;
 use u_lib::logging::init_logger;
 
 #[actix_web::main]
 async fn main() -> AnyResult<()> {
+    init_logger(None);
+
     let env = AccessEnv::load()?;
-    let client = ClientHandler::new(&env.u_server, Some(env.admin_auth_token)).await?;
+    let client = HttpClient::new(&env.u_server, Some(env.admin_auth_token)).await?;
     let args = Args::from_args();
-
-    init_logger(None::<&str>);
-
     let result = process_cmd(client, args).await.to_string();
+
     println!("{result}");
     Ok(())
 }

bin/u_server/Cargo.toml

@@ -9,6 +9,7 @@ anyhow = { workspace = true }
 diesel = { workspace = true }
 deadpool-diesel = { workspace = true }
 hyper = "0.14"
+mime_guess = { workspace = true }
 once_cell = "1.7.2"
 openssl = { workspace = true }
 serde = { workspace = true }
@@ -19,6 +20,7 @@ tokio = { workspace = true, features = ["macros"] }
 uuid = { workspace = true, features = ["serde", "v4"] }
 u_lib = { path = "../../lib/u_lib", features = ["server"] }
 warp = { version = "0.3.1", features = ["tls"] }
+serde_qs = { version = "0.12.0", features = ["warp"] }
 
 [dev-dependencies]
 rstest = "0.12"

bin/u_server/src/db.rs

@@ -1,9 +1,14 @@
 use crate::error::Error;
 use diesel::{pg::PgConnection, prelude::*, result::Error as DslError, Connection};
-use u_lib::db::PgAsyncPool;
-use u_lib::models::{schema, Agent, AssignedJob, JobState, ThinJobMeta};
-use u_lib::platform::Platform;
-use u_lib::types::Id;
+use std::collections::{HashMap, HashSet};
+use std::mem::drop;
+use u_lib::{
+    db::PgAsyncPool,
+    models::{schema, Agent, AssignedJob, AssignedJobById, Job, JobMeta, JobState, Payload},
+    platform::Platform,
+    types::Id,
+};
+use uuid::Uuid;
 
 type Result<T> = std::result::Result<T, Error>;
 
@@ -47,27 +52,40 @@ pub struct UDB<'c> {
 }
 
 impl UDB<'_> {
-    pub fn insert_jobs(&mut self, job_metas: &[ThinJobMeta]) -> Result<Vec<Id>> {
+    pub fn insert_jobs(&mut self, jobs: &[JobMeta]) -> Result<()> {
         use schema::jobs;
 
         diesel::insert_into(jobs::table)
-            .values(job_metas)
-            .get_results(self.conn)
-            .map(|rows| rows.iter().map(|job: &ThinJobMeta| job.id).collect())
+            .values(jobs)
+            .execute(self.conn)
+            .map(drop)
             .map_err(with_err_ctx("Can't insert jobs"))
     }
 
-    pub fn get_job(&mut self, id: Id) -> Result<Option<ThinJobMeta>> {
-        use schema::jobs;
+    pub fn insert_payload(&mut self, payload: &Payload) -> Result<()> {
+        use schema::payloads;
 
-        jobs::table
+        diesel::insert_into(payloads::table)
+            .values(payload)
+            .execute(self.conn)
+            .map(drop)
+            .map_err(with_err_ctx(format!("Can't insert payload {payload:?}")))
+    }
+
+    pub fn get_job(&mut self, id: Id) -> Result<Option<Job>> {
+        use schema::{jobs, payloads};
+
+        let maybe_job_with_payload = jobs::table
+            .left_join(payloads::table)
             .filter(jobs::id.eq(id))
-            .first(self.conn)
+            .first::<(JobMeta, Option<Payload>)>(self.conn)
             .optional()
-            .map_err(with_err_ctx(format!("Can't get job {id}")))
+            .map_err(with_err_ctx(format!("Can't get job {id}")))?;
+
+        Ok(maybe_job_with_payload.map(|(job, payload)| Job { meta: job, payload }))
     }
 
-    pub fn get_jobs(&mut self) -> Result<Vec<ThinJobMeta>> {
+    pub fn get_jobs(&mut self) -> Result<Vec<JobMeta>> {
         use schema::jobs;
 
         jobs::table
@@ -75,27 +93,59 @@ impl UDB<'_> {
             .map_err(with_err_ctx("Can't get jobs"))
     }
 
-    pub fn find_job_by_alias(&mut self, alias: &str) -> Result<Option<ThinJobMeta>> {
-        use schema::jobs;
+    pub fn get_payload(&mut self, id: Id) -> Result<Option<Payload>> {
+        use schema::payloads;
 
-        jobs::table
-            .filter(jobs::alias.eq(alias))
+        payloads::table
+            .filter(payloads::id.eq(id))
             .first(self.conn)
             .optional()
-            .map_err(with_err_ctx(format!("Can't find job by alias {alias}")))
+            .map_err(with_err_ctx(format!("Can't get payload {id}")))
     }
 
-    pub fn insert_agent(&mut self, agent: &Agent) -> Result<()> {
-        use schema::agents;
+    pub fn get_payload_by_name(&mut self, name: String) -> Result<Option<Payload>> {
+        use schema::payloads;
 
-        diesel::insert_into(agents::table)
-            .values(agent)
-            .on_conflict(agents::id)
-            .do_update()
-            .set(agent)
-            .execute(self.conn)
-            .map_err(with_err_ctx(format!("Can't insert agent {agent:?}")))?;
-        Ok(())
+        payloads::table
+            .filter(payloads::name.eq(&name))
+            .first(self.conn)
+            .optional()
+            .map_err(with_err_ctx(format!("Can't get payload by name {name}")))
+    }
+
+    pub fn get_payloads(&mut self) -> Result<Vec<Payload>> {
+        use schema::payloads;
+
+        payloads::table
+            .load(self.conn)
+            .map_err(with_err_ctx("Can't get payloads"))
+    }
+
+    pub fn payload_exists(&mut self, payload_id: Id) -> Result<bool> {
+        use schema::payloads;
+
+        payloads::table
+            .find(payload_id)
+            .first::<Payload>(self.conn)
+            .optional()
+            .map(|r| r.is_some())
+            .map_err(with_err_ctx("Can't check payload {payload_id}"))
+    }
+
+    pub fn get_job_by_alias(&mut self, alias: &str) -> Result<Option<Job>> {
+        use schema::{jobs, payloads};
+
+        let maybe_job_with_payload = jobs::table
+            .left_join(payloads::table)
+            .filter(jobs::alias.eq(alias))
+            .first::<(JobMeta, Option<Payload>)>(self.conn)
+            .optional()
+            .map_err(with_err_ctx(format!("Can't get job by alias {alias}")))?;
+
+        Ok(maybe_job_with_payload.map(|(job, payload_meta)| Job {
+            meta: job,
+            payload: payload_meta,
+        }))
     }
 
     pub fn insert_result(&mut self, result: &AssignedJob) -> Result<()> {
@@ -138,7 +188,11 @@ impl UDB<'_> {
     }
 
     //TODO: filters possibly could work in a wrong way, check
-    pub fn get_exact_jobs(&mut self, id: Option<Id>, personal: bool) -> Result<Vec<AssignedJob>> {
+    pub fn get_assigned_jobs(
+        &mut self,
+        id: Option<Id>,
+        personal: bool,
+    ) -> Result<Vec<AssignedJob>> {
         use schema::results;
 
         let mut q = results::table.into_boxed();
@@ -163,39 +217,72 @@ impl UDB<'_> {
         Ok(result)
     }
 
-    pub fn set_jobs_for_agent(&mut self, agent_id: Id, job_ids: &[Id]) -> Result<Vec<Id>> {
+    // todo: move to handlers
+    pub fn assign_jobs(&mut self, assigned_jobs: &[AssignedJobById]) -> Result<()> {
         use schema::{jobs, results};
 
-        let agent_platform = match self.get_agent(agent_id)? {
-            Some(agent) => Platform::new(&agent.platform),
-            None => {
-                return Err(Error::ProcessingError(format!(
-                    "Agent {agent_id} not found"
-                )))
-            }
-        };
+        struct JobBriefMeta {
+            alias: Option<String>,
+            target_platform: String,
+        }
+
+        let assigned_job_ids = HashSet::<Uuid>::from_iter(assigned_jobs.iter().map(|a| a.job_id));
+
+        let jobs_meta = HashMap::<Id, JobBriefMeta>::from_iter(
+            jobs::table
+                .select((jobs::id, jobs::alias, jobs::target_platforms))
+                .filter(jobs::id.eq_any(&assigned_job_ids))
+                .load::<(Id, Option<String>, String)>(self.conn)
+                .map_err(with_err_ctx(format!(
+                    "Can't find jobs {:?}",
+                    assigned_job_ids
+                )))?
+                .into_iter()
+                .map(|(id, alias, target_platform)| {
+                    (
+                        id,
+                        JobBriefMeta {
+                            alias,
+                            target_platform,
+                        },
+                    )
+                }),
+        );
 
-        let jobs_meta = jobs::table
-            .select((jobs::id, jobs::alias, jobs::platform))
-            .filter(jobs::id.eq_any(job_ids))
-            .load::<(Id, Option<String>, String)>(self.conn)
-            .map_err(with_err_ctx(format!("Can't find jobs {job_ids:?}")))?;
+        let existing_job_ids = HashSet::from_iter(jobs_meta.keys().copied());
 
-        for meta in &jobs_meta {
-            if !agent_platform.matches(&meta.2) {
+        if assigned_job_ids != existing_job_ids {
+            return Err(Error::ProcessingError(format!(
+                "Jobs not found: {:?}",
+                assigned_job_ids.difference(&existing_job_ids),
+            )));
+        }
+
+        for ajob in assigned_jobs {
+            let meta = &jobs_meta[&ajob.job_id];
+            let agent_platform = match self.get_agent(ajob.agent_id)? {
+                Some(agent) => Platform::new(&agent.platform),
+                None => {
+                    return Err(Error::ProcessingError(format!(
+                        "Agent {} not found",
+                        ajob.agent_id
+                    )))
+                }
+            };
+            if !agent_platform.matches(&meta.target_platform) {
                 return Err(Error::InsuitablePlatform(
                     agent_platform.into_string(),
-                    meta.2.clone(),
+                    meta.target_platform.clone(),
                 ));
             }
         }
 
-        let job_requests = jobs_meta
+        let job_requests = assigned_jobs
             .into_iter()
-            .map(|(job_id, alias, _)| AssignedJob {
-                job_id,
-                agent_id,
-                alias,
+            .map(|a| AssignedJob {
+                job_id: a.job_id,
+                agent_id: a.agent_id,
+                alias: jobs_meta[&a.job_id].alias.clone(),
                 ..Default::default()
             })
             .collect::<Vec<AssignedJob>>();
@@ -203,68 +290,76 @@ impl UDB<'_> {
         diesel::insert_into(results::table)
             .values(&job_requests)
             .execute(self.conn)
-            .map_err(with_err_ctx(format!(
-                "Can't setup jobs {job_ids:?} for agent {agent_id:?}"
-            )))?;
-
-        Ok(job_requests.iter().map(|aj| aj.id).collect())
+            .map(drop)
+            .map_err(with_err_ctx("Can't assign jobs"))
     }
 
-    pub fn del_jobs(&mut self, ids: &[Id]) -> Result<usize> {
+    pub fn del_jobs(&mut self, ids: &[Id]) -> Result<()> {
         use schema::jobs;
 
-        let mut affected = 0;
-        for id in ids {
-            let deleted = diesel::delete(jobs::table)
-                .filter(jobs::id.eq(id))
-                .execute(self.conn)
-                .map_err(with_err_ctx("Can't delete jobs"))?;
-            affected += deleted;
-        }
-        Ok(affected)
+        diesel::delete(jobs::table)
+            .filter(jobs::id.eq_any(ids))
+            .execute(self.conn)
+            .map(drop)
+            .map_err(with_err_ctx("Can't delete jobs"))
     }
 
-    pub fn del_results(&mut self, ids: &[Id]) -> Result<usize> {
+    pub fn del_results(&mut self, ids: &[Id]) -> Result<()> {
         use schema::results;
 
-        let mut affected = 0;
-        for id in ids {
-            let deleted = diesel::delete(results::table)
-                .filter(results::id.eq(id))
-                .execute(self.conn)
-                .map_err(with_err_ctx("Can't delete results"))?;
-            affected += deleted;
-        }
-        Ok(affected)
+        diesel::delete(results::table)
+            .filter(results::id.eq_any(ids))
+            .execute(self.conn)
+            .map(drop)
+            .map_err(with_err_ctx("Can't delete results"))
     }
 
-    pub fn del_agents(&mut self, ids: &[Id]) -> Result<usize> {
+    pub fn del_agents(&mut self, ids: &[Id]) -> Result<()> {
         use schema::agents;
 
-        let mut affected = 0;
-        for id in ids {
-            let deleted = diesel::delete(agents::table)
-                .filter(agents::id.eq(id))
-                .execute(self.conn)
-                .map_err(with_err_ctx("Can't delete agents"))?;
-            affected += deleted;
-        }
-        Ok(affected)
+        diesel::delete(agents::table)
+            .filter(agents::id.eq_any(ids))
+            .execute(self.conn)
+            .map(drop)
+            .map_err(with_err_ctx("Can't delete agents"))
     }
 
-    pub fn update_agent(&mut self, agent: &Agent) -> Result<()> {
-        agent
-            .save_changes::<Agent>(self.conn)
-            .map_err(with_err_ctx(format!("Can't update agent {agent:?}")))?;
+    pub fn del_payloads(&mut self, ids: &[Id]) -> Result<()> {
+        use schema::payloads;
+
+        diesel::delete(payloads::table)
+            .filter(payloads::id.eq_any(ids))
+            .execute(self.conn)
+            .map(drop)
+            .map_err(with_err_ctx("Can't delete payloads"))
+    }
+
+    pub fn upsert_agent(&mut self, agent: &Agent) -> Result<()> {
+        use schema::agents;
+
+        diesel::insert_into(agents::table)
+            .values(agent)
+            .on_conflict(agents::id)
+            .do_update()
+            .set(agent)
+            .execute(self.conn)
+            .map_err(with_err_ctx(format!("Can't insert agent {agent:?}")))?;
         Ok(())
     }
 
-    pub fn update_job(&mut self, job: &ThinJobMeta) -> Result<()> {
-        job.save_changes::<ThinJobMeta>(self.conn)
+    pub fn update_job(&mut self, job: &JobMeta) -> Result<()> {
+        job.save_changes::<JobMeta>(self.conn)
             .map_err(with_err_ctx(format!("Can't update job {job:?}")))?;
         Ok(())
     }
 
+    pub fn update_payload(&mut self, payload: &Payload) -> Result<()> {
+        payload
+            .save_changes::<Payload>(self.conn)
+            .map_err(with_err_ctx(format!("Can't update payload {payload:?}")))?;
+        Ok(())
+    }
+
     pub fn update_result(&mut self, result: &AssignedJob) -> Result<()> {
         debug!(
             "updating result: id = {}, job_id = {}, agent_id = {}",

bin/u_server/src/error.rs

@@ -1,6 +1,6 @@
 use diesel::result::Error as DslError;
 use thiserror::Error;
-use u_lib::ufs;
+use u_lib::{ufs, UError};
 use warp::{
     http::StatusCode,
     reject::Reject,
@@ -13,7 +13,7 @@ pub enum Error {
     #[error("Configs error: {0}")]
     ConfigError(#[from] u_lib::config::Error),
 
-    #[error("Error processing {0}")]
+    #[error("Processing error: {0}")]
     ProcessingError(String),
 
     #[error(transparent)]
@@ -30,6 +30,15 @@ pub enum Error {
 
     #[error("Job cannot be ran on this platform. Expected: {0}, got: {1}")]
     InsuitablePlatform(String, String),
+
+    #[error("{0}\nContext: {1}")]
+    Contexted(Box<Error>, String),
+
+    #[error(transparent)]
+    UError(#[from] UError),
+
+    #[error("Runtime error: {0}")]
+    Runtime(String),
 }
 
 impl Reject for Error {}
@@ -56,7 +65,7 @@ impl RejResponse {
 
     pub fn internal() -> Self {
         Self {
-            message: "INTERNAL_SERVER_ERROR".to_string(),
+            message: "INTERNAL SERVER ERROR".to_string(),
             status: StatusCode::INTERNAL_SERVER_ERROR,
         }
     }
@@ -67,3 +76,23 @@ impl Reply for RejResponse {
         with_status(self.message, self.status).into_response()
     }
 }
+
+impl From<anyhow::Error> for Error {
+    fn from(e: anyhow::Error) -> Self {
+        let ctx = e
+            .chain()
+            .rev()
+            .skip(1)
+            .map(|cause| format!("ctx: {}", cause))
+            .collect::<Vec<_>>()
+            .join("\n");
+
+        match e.downcast::<Error>() {
+            Ok(err) => Error::Contexted(Box::new(err), ctx),
+            Err(err) => match err.downcast::<ufs::Error>() {
+                Ok(err) => Error::Contexted(Box::new(Error::FSError(err)), ctx),
+                Err(err) => Error::Runtime(err.to_string()),
+            },
+        }
+    }
+}

bin/u_server/src/handlers.rs

@@ -2,23 +2,22 @@ use std::sync::Arc;
 
 use crate::db::{PgRepo, UDB};
 use crate::error::Error;
-use crate::ValidJobMeta;
-use u_lib::{
-    jobs::{fat_meta_to_thin, thin_meta_to_fat},
-    messaging::{AsMsg, Reportable},
-    misc::OneOrVec,
-    models::*,
-    types::Id
-};
+use serde::Deserialize;
+use u_lib::{api::api_types, messaging::Reportable, models::*, types::Id};
 use warp::reject::not_found;
 use warp::Rejection;
 
 type EndpResult<T> = Result<T, Rejection>;
 
+#[derive(Deserialize)]
+pub struct PayloadFlags {
+    brief: Brief,
+}
+
 pub struct Endpoints;
 
 impl Endpoints {
-    pub async fn get_agents(repo: Arc<PgRepo>, id: Option<Id>) -> EndpResult<Vec<Agent>> {
+    pub async fn get_agents(repo: Arc<PgRepo>, id: Option<Id>) -> EndpResult<api_types::GetAgents> {
         repo.interact(move |mut db| {
             Ok(match id {
                 Some(id) => {
@@ -35,58 +34,125 @@ impl Endpoints {
         .map_err(From::from)
     }
 
-    pub async fn get_job(repo: Arc<PgRepo>, id: Id) -> EndpResult<ValidJobMeta> {
-        let Some(job) = repo.interact(move |mut db| db.get_job(id)).await? else {
-            return Err(not_found())
+    pub async fn get_job(
+        repo: Arc<PgRepo>,
+        id: Id,
+        params: Option<PayloadFlags>,
+    ) -> EndpResult<api_types::GetJob> {
+        let Some(mut job) = repo.interact(move |mut db| db.get_job(id)).await? else {
+            return Err(not_found());
         };
 
-        let fat_meta = thin_meta_to_fat(job).map_err(Error::from)?;
-        Ok(fat_meta)
+        Ok(match params.map(|p| p.brief) {
+            Some(Brief::Yes) => job,
+            Some(Brief::Auto) | None => {
+                if let Some(payload) = &mut job.payload {
+                    payload.maybe_join_payload().map_err(Error::from)?;
+                }
+                job
+            }
+            Some(Brief::No) => {
+                if let Some(payload) = &mut job.payload {
+                    payload.join_payload().map_err(Error::from)?;
+                }
+                job
+            }
+        })
     }
 
-    pub async fn get_jobs(repo: Arc<PgRepo>) -> EndpResult<Vec<ThinJobMeta>> {
+    pub async fn get_jobs(repo: Arc<PgRepo>) -> EndpResult<api_types::GetJobs> {
         repo.interact(move |mut db| db.get_jobs())
             .await
             .map_err(From::from)
     }
 
-    pub async fn get_agent_jobs(
+    pub async fn get_assigned_jobs(
         repo: Arc<PgRepo>,
         id: Option<Id>,
-    ) -> EndpResult<Vec<AssignedJob>> {
-        repo.interact(move |mut db| db.get_exact_jobs(id, false))
+    ) -> EndpResult<api_types::GetAgentJobs> {
+        repo.interact(move |mut db| db.get_assigned_jobs(id, false))
             .await
             .map_err(From::from)
     }
 
-    pub async fn get_personal_jobs(repo: Arc<PgRepo>, id: Id) -> EndpResult<Vec<AssignedJob>> {
+    pub async fn get_payloads(repo: Arc<PgRepo>) -> EndpResult<api_types::GetPayloads> {
+        repo.interact(move |mut db| db.get_payloads())
+            .await
+            .map_err(From::from)
+    }
+
+    pub async fn get_payload(
+        repo: Arc<PgRepo>,
+        name_or_id: String,
+        params: Option<PayloadFlags>,
+    ) -> EndpResult<api_types::GetPayload> {
+        let mut payload = match repo
+            .interact(move |mut db| match Id::parse_str(&name_or_id) {
+                Ok(id) => db.get_payload(id),
+                Err(_) => db.get_payload_by_name(name_or_id),
+            })
+            .await?
+        {
+            Some(p) => p,
+            None => return Err(not_found()),
+        };
+
+        Ok(match params.map(|p| p.brief) {
+            Some(Brief::Yes) => {
+                payload.data = None;
+                payload
+            }
+            None | Some(Brief::Auto) => {
+                payload.maybe_join_payload().map_err(Error::from)?;
+                payload
+            }
+            _ => {
+                payload.join_payload().map_err(Error::from)?;
+                payload
+            }
+        })
+    }
+
+    pub async fn get_personal_jobs(
+        repo: Arc<PgRepo>,
+        agent_id: Id,
+    ) -> EndpResult<api_types::GetPersonalJobs> {
         repo.transaction(move |mut db| {
-            let agent = db.get_agent(id)?;
+            let agent = db.get_agent(agent_id)?;
             match agent {
                 Some(mut agent) => {
                     agent.touch();
-                    db.update_agent(&agent)?;
+                    db.upsert_agent(&agent)?;
                 }
                 None => {
-                    let new_agent = Agent::with_id(id);
+                    let mut new_agent = Agent::empty();
+                    new_agent.id = agent_id;
 
-                    db.insert_agent(&new_agent)?;
+                    db.upsert_agent(&new_agent)?;
 
                     let job = db
-                        .find_job_by_alias("agent_hello")?
+                        .get_job_by_alias("agent_hello")?
                         .expect("agent_hello job not found");
 
-                    db.set_jobs_for_agent(id, &[job.id])?;
+                    let assigned_job = AssignedJobById {
+                        agent_id,
+                        job_id: job.meta.id,
+                        ..Default::default()
+                    };
+                    db.assign_jobs(&[assigned_job])?;
                 }
             }
 
-            let assigned_jobs = db.get_exact_jobs(Some(id), true)?;
+            let assigned_jobs = db.get_assigned_jobs(Some(agent_id), true)?;
 
             for job in &assigned_jobs {
                 db.update_job_status(job.id, JobState::Running)?;
             }
 
-            Ok(assigned_jobs)
+            Ok(assigned_jobs
+                .into_iter()
+                .map(|j| AssignedJobById::from(&j))
+                .collect())
         })
         .await
         .map_err(From::from)
@@ -94,69 +160,91 @@ impl Endpoints {
 
     pub async fn upload_jobs(
         repo: Arc<PgRepo>,
-        msg: Vec<ValidJobMeta>,
-    ) -> EndpResult<Vec<Id>> {
-        let jobs = msg
+        jobs: Vec<Job>,
+    ) -> EndpResult<api_types::UploadJobs> {
+        let mut checked_jobs = vec![];
+        for mut job in jobs {
+            if let Some(payload) = &mut job.payload {
+                payload.maybe_split_payload().map_err(Error::from)?;
+            } else if let Some(pld_id) = job.meta.payload_id {
+                if !repo
+                    .interact(move |mut db| db.payload_exists(pld_id))
+                    .await?
+                {
+                    Err(Error::ProcessingError(format!(
+                        "Payload {pld_id} not found"
+                    )))?
+                }
+            }
+            checked_jobs.push(job)
+        }
+
+        let (jobs, payloads_opt): (Vec<_>, Vec<_>) = checked_jobs
+            .into_iter()
+            .map(|j| (j.meta, j.payload))
+            .unzip();
+
+        let payloads = payloads_opt
             .into_iter()
-            .map(|meta| Ok(fat_meta_to_thin(meta)?))
-            .collect::<Result<Vec<ThinJobMeta>, Error>>()?;
+            .filter_map(|p| p)
+            .collect::<Vec<_>>();
 
-        repo.interact(move |mut db| db.insert_jobs(&jobs))
+        repo.transaction(move |mut db| {
+            for payload in payloads {
+                db.insert_payload(&payload)?;
+            }
+            db.insert_jobs(&jobs)
+        })
+        .await
+        .map_err(From::from)
+    }
+
+    pub async fn upload_payload(
+        repo: Arc<PgRepo>,
+        raw_payload: RawPayload,
+    ) -> EndpResult<api_types::UploadPayloads> {
+        let payloads = raw_payload.into_payload().map_err(Error::from)?;
+
+        repo.interact(move |mut db| db.insert_payload(&payloads))
             .await
             .map_err(From::from)
     }
 
-    pub async fn del(repo: Arc<PgRepo>, id: Id) -> EndpResult<usize> {
+    pub async fn del(repo: Arc<PgRepo>, id: Id) -> EndpResult<()> {
         repo.transaction(move |mut db| {
-            let del_fns = &[UDB::del_agents, UDB::del_jobs, UDB::del_results];
-            for del_fn in del_fns {
-                let affected = del_fn(&mut db, &[id])?;
-                if affected > 0 {
-                    return Ok(affected);
-                }
-            }
-            Ok(0)
+            [
+                UDB::del_agents,
+                UDB::del_jobs,
+                UDB::del_results,
+                UDB::del_payloads,
+            ]
+            .iter()
+            .map(|f| f(&mut db, &[id]))
+            .collect::<Result<(), Error>>()
         })
         .await
         .map_err(From::from)
     }
 
-    pub async fn set_jobs(
+    pub async fn assign_jobs(
         repo: Arc<PgRepo>,
-        agent_id: Id,
-        job_idents: Vec<String>,
-    ) -> EndpResult<Vec<Id>> {
+        assigned_jobs: Vec<AssignedJobById>,
+    ) -> EndpResult<()> {
         repo.transaction(move |mut db| {
-            job_idents
-                .into_iter()
-                .map(|ident| {
-                    Id::parse_str(&ident).or_else(|_| {
-                        let job_from_db = db.find_job_by_alias(&ident);
-                        match job_from_db {
-                            Ok(job) => match job {
-                                Some(j) => Ok(j.id),
-                                None => {
-                                    Err(Error::ProcessingError(format!("unknown ident {ident}")))
-                                }
-                            },
-                            Err(e) => Err(e),
-                        }
-                    })
-                })
-                .collect::<Result<Vec<Id>, Error>>()
-                .and_then(|j| db.set_jobs_for_agent(agent_id, &j))
+            db.assign_jobs(&assigned_jobs)?;
+            Ok(())
         })
         .await
         .map_err(From::from)
     }
 
-    pub async fn report<Data: OneOrVec<Reportable> + AsMsg + Send + Sync + 'static>(
+    pub async fn report(
         repo: Arc<PgRepo>,
-        msg: Data,
-        agent_id: Id
-    ) -> EndpResult<()> {
+        msg: Vec<Reportable>,
+        agent_id: Id,
+    ) -> EndpResult<api_types::Report> {
         repo.transaction(move |mut db| {
-            for entry in msg.into_vec() {
+            for entry in msg {
                 match entry {
                     Reportable::Assigned(mut result) => {
                         let result_agent_id = &result.agent_id;
@@ -165,13 +253,13 @@ impl Endpoints {
                             continue;
                         }
                         result.touch();
-                        
+
                         info!("agent {agent_id} updated job {}", result.id);
 
                         match result.exec_type {
                             JobType::Init => {
                                 result.state = JobState::Finished;
-                                
+
                                 match &result.result {
                                 Some(rbytes) => {
                                     let mut agent: Agent = match serde_json::from_slice(&rbytes) {
@@ -182,7 +270,7 @@ impl Endpoints {
                                         }
                                     };
                                     agent.state = AgentState::Active;
-                                    db.insert_agent(&agent)?;
+                                    db.upsert_agent(&agent)?;
                                 }
                                 None => error!("Empty agent data"),
                             }},
@@ -209,32 +297,46 @@ impl Endpoints {
     pub async fn update_agent(
         repo: Arc<PgRepo>,
         agent: Agent,
-    ) -> EndpResult<()> {
-        repo.interact(move |mut db| db.update_agent(&agent))
-            .await?;
-        Ok(())
+    ) -> EndpResult<api_types::UpdateAgent> {
+        repo.interact(move |mut db| db.upsert_agent(&agent))
+            .await
+            .map_err(From::from)
     }
 
-    pub async fn update_job(
-        repo: Arc<PgRepo>,
-        job: ValidJobMeta,
-    ) -> EndpResult<()> {
-        let thin_meta = fat_meta_to_thin(job).map_err(Error::from)?;
-        repo.interact(move |mut db| db.update_job(&thin_meta))
-            .await?;
-        Ok(())
+    pub async fn update_job(repo: Arc<PgRepo>, job: JobMeta) -> EndpResult<api_types::UpdateJob> {
+        repo.interact(move |mut db| db.update_job(&job.validate()?))
+            .await
+            .map_err(From::from)
     }
 
     pub async fn update_assigned_job(
         repo: Arc<PgRepo>,
         assigned: AssignedJob,
-    ) -> EndpResult<()> {
+    ) -> EndpResult<api_types::UpdateResult> {
         repo.interact(move |mut db| db.update_result(&assigned))
-            .await?;
-        Ok(())
+            .await
+            .map_err(From::from)
     }
 
-    pub async fn download(_file_id: String) -> EndpResult<Vec<u8>> {
-        todo!()
+    pub async fn update_payload(
+        repo: Arc<PgRepo>,
+        payload: Payload,
+    ) -> EndpResult<api_types::UpdatePayload> {
+        debug!("update payload: {payload:?}");
+        match payload.data {
+            Some(data) => {
+                let mut well_formed_payload =
+                    Payload::from_data(data, Some(payload.name)).map_err(Error::from)?;
+                well_formed_payload.id = payload.id;
+
+                repo.interact(move |mut db| db.update_payload(&well_formed_payload))
+                    .await
+                    .map_err(From::from)
+            }
+            None => repo
+                .interact(move |mut db| db.update_payload(&payload))
+                .await
+                .map_err(From::from),
+        }
     }
 }

bin/u_server/src/u_server.rs

@@ -1,21 +1,17 @@
 #[macro_use]
 extern crate tracing;
 
-#[cfg(test)]
-#[macro_use]
-extern crate rstest;
-
 mod db;
 mod error;
 mod handlers;
 
+use crate::handlers::{Endpoints, PayloadFlags};
 use db::PgRepo;
 use error::{Error as ServerError, RejResponse};
 use std::{convert::Infallible, sync::Arc};
 use u_lib::{
     config,
     db::async_pool,
-    jobs::fat_meta_to_thin,
     messaging::{AsMsg, Reportable},
     models::*,
     types::Id,
@@ -23,13 +19,11 @@ use u_lib::{
 use warp::{
     body,
     log::{custom, Info},
-    reply::{json, reply, Json, Response},
+    reply::{json, Json, Response},
     Filter, Rejection, Reply,
 };
 
-use crate::handlers::Endpoints;
-
-type ValidJobMeta = FatJobMeta<true>;
+const DEFAULT_RESPONSE: &str = "null";
 
 fn into_message<M: AsMsg>(msg: M) -> Json {
     json(&msg)
@@ -39,8 +33,15 @@ pub fn init_endpoints(
     auth_token: &str,
     db: PgRepo,
 ) -> impl Filter<Extract = (impl Reply,), Error = Rejection> + Clone {
+    fn make_optional<T>(
+        f: impl Filter<Extract = (T,), Error = Rejection> + Clone,
+    ) -> impl Filter<Extract = (Option<T>,), Error = Infallible> + Clone {
+        f.map(Some)
+            .or_else(|_| async { Ok::<(Option<T>,), Infallible>((None,)) })
+    }
+
     let path = |p: &'static str| warp::post().and(warp::path(p));
-    let infallible_none = |_| async { Result::<(Option<Id>,), Infallible>::Ok((None,)) };
+    let create_qs_cfg = || serde_qs::Config::new(1, true);
 
     let with_db = {
         let adb = Arc::new(db);
@@ -49,19 +50,22 @@ pub fn init_endpoints(
 
     let get_agents = path("get_agents")
         .and(with_db.clone())
-        .and(warp::path::param::<Id>().map(Some).or_else(infallible_none))
+        .and(make_optional(warp::path::param::<Id>()))
         .and_then(Endpoints::get_agents)
         .map(into_message);
 
     let upload_jobs = path("upload_jobs")
         .and(with_db.clone())
-        .and(body::json::<Vec<ValidJobMeta>>())
+        .and(body::json::<Vec<Job>>())
         .and_then(Endpoints::upload_jobs)
         .map(into_message);
 
     let get_job = path("get_job")
         .and(with_db.clone())
         .and(warp::path::param::<Id>())
+        .and(make_optional(serde_qs::warp::query::<PayloadFlags>(
+            create_qs_cfg(),
+        )))
         .and_then(Endpoints::get_job)
         .map(into_message);
 
@@ -70,10 +74,10 @@ pub fn init_endpoints(
         .and_then(Endpoints::get_jobs)
         .map(into_message);
 
-    let get_agent_jobs = path("get_agent_jobs")
+    let get_assigned_jobs = path("get_assigned_jobs")
         .and(with_db.clone())
-        .and(warp::path::param::<Id>().map(Some).or_else(infallible_none))
-        .and_then(Endpoints::get_agent_jobs)
+        .and(make_optional(warp::path::param::<Id>()))
+        .and_then(Endpoints::get_assigned_jobs)
         .map(into_message);
 
     let get_personal_jobs = path("get_personal_jobs")
@@ -88,11 +92,10 @@ pub fn init_endpoints(
         .and_then(Endpoints::del)
         .map(ok);
 
-    let set_jobs = path("set_jobs")
+    let assign_jobs = path("assign_jobs")
         .and(with_db.clone())
-        .and(warp::path::param::<Id>())
-        .and(body::json::<Vec<String>>())
-        .and_then(Endpoints::set_jobs)
+        .and(body::json::<Vec<AssignedJobById>>())
+        .and_then(Endpoints::assign_jobs)
         .map(into_message);
 
     let report = path("report")
@@ -110,7 +113,7 @@ pub fn init_endpoints(
 
     let update_job = path("update_job")
         .and(with_db.clone())
-        .and(body::json::<ValidJobMeta>())
+        .and(body::json::<JobMeta>())
         .and_then(Endpoints::update_job)
         .map(ok);
 
@@ -120,12 +123,33 @@ pub fn init_endpoints(
         .and_then(Endpoints::update_assigned_job)
         .map(ok);
 
-    let download = path("download")
+    let get_payloads = path("get_payloads")
+        .and(with_db.clone())
+        .and_then(Endpoints::get_payloads)
+        .map(into_message);
+
+    let get_payload = path("get_payload")
+        .and(with_db.clone())
         .and(warp::path::param::<String>())
-        .and_then(Endpoints::download)
+        .and(make_optional(serde_qs::warp::query::<PayloadFlags>(
+            create_qs_cfg(),
+        )))
+        .and_then(Endpoints::get_payload)
+        .map(into_message);
+
+    let upload_payload = path("upload_payload")
+        .and(with_db.clone())
+        .and(body::json::<RawPayload>())
+        .and_then(Endpoints::upload_payload)
+        .map(ok);
+
+    let update_payload = path("update_payload")
+        .and(with_db.clone())
+        .and(body::json::<Payload>())
+        .and_then(Endpoints::update_payload)
         .map(ok);
 
-    let ping = path("ping").map(reply);
+    let ping = path("ping").map(|| DEFAULT_RESPONSE);
 
     let auth_token = format!("Bearer {auth_token}",).into_boxed_str();
     let auth_header = warp::header::exact("authorization", Box::leak(auth_token));
@@ -133,20 +157,21 @@ pub fn init_endpoints(
     let auth_zone = (get_agents
         .or(get_job.clone())
         .or(get_jobs.clone())
+        .or(get_payloads)
+        .or(get_payload)
         .or(upload_jobs)
+        .or(upload_payload)
         .or(del)
-        .or(set_jobs)
-        .or(get_agent_jobs)
-        .or(update_agent.or(update_job).or(update_assigned_job))
-        .or(download)
+        .or(assign_jobs)
+        .or(get_assigned_jobs)
+        .or(update_agent)
+        .or(update_job)
+        .or(update_assigned_job)
+        .or(update_payload)
         .or(ping))
     .and(auth_header);
 
-    let agent_zone = get_job
-        .or(get_jobs)
-        .or(get_personal_jobs)
-        .or(report)
-        .or(download);
+    let agent_zone = get_job.or(get_jobs).or(get_personal_jobs).or(report);
 
     auth_zone.or(agent_zone)
 }
@@ -154,16 +179,14 @@ pub fn init_endpoints(
 pub async fn preload_jobs(repo: &PgRepo) -> Result<(), ServerError> {
     repo.interact(|mut db| {
         let job_alias = "agent_hello";
-        let if_job_exists = db.find_job_by_alias(job_alias)?;
+        let if_job_exists = db.get_job_by_alias(job_alias)?;
         if if_job_exists.is_none() {
-            let agent_hello = fat_meta_to_thin(
-                FatJobMeta::builder()
-                    .with_type(JobType::Init)
-                    .with_alias(job_alias)
-                    .build()
-                    .unwrap(),
-            )?;
-            db.insert_jobs(&[agent_hello])?;
+            let agent_hello = RawJob::default()
+                .with_type(JobType::Init)
+                .with_alias(job_alias)
+                .try_into_job()
+                .unwrap();
+            db.insert_jobs(&[agent_hello.meta])?;
         }
         Ok(())
     })
@@ -219,13 +242,13 @@ fn logger(info: Info<'_>) {
                 .take(2)
                 .collect::<String>()
             )
-            .unwrap_or_else(|| "NO_AGENT".to_string()),
+            .unwrap_or_else(|| "NO_AGENT_UID".to_string()),
         status = info.status()
     );
 }
 
 fn ok<T>(_: T) -> impl Reply {
-    reply()
+    DEFAULT_RESPONSE
 }
 
 /*

deploy/podman-compose.yml

@@ -18,14 +18,14 @@ services:
       u_db:
         condition: service_healthy
     ports:
-      - 63714:63714
+      - 9990:9990
     env_file:
       - ./.env
       - ./.env.private
     environment:
       RUST_LOG: warp=info,u_server_lib=debug
     healthcheck:
-        test: ss -tlpn | grep 63714
+        test: ss -tlpn | grep 9990
         interval: 5s
         timeout: 2s
         retries: 2

images/tests_runner.Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.67
+FROM rust:1.72
 
 RUN rustup target add x86_64-unknown-linux-musl
 RUN mkdir -p /tests && chmod 777 /tests

images/u_server.Dockerfile

@@ -1,3 +1,3 @@
 FROM alpine:3.17
 
-RUN apk add iproute2 bash
+RUN apk add iproute2 bash file

integration-tests/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "integration"
+name = "integration-tests"
 version = "0.1.0"
 authors = ["plazmoid <kronos44@mail.ru>"]
 edition = "2021"
@@ -7,18 +7,19 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+ctor = "0.2.0"
 once_cell = "1.10.0"
 reqwest = { workspace = true }
-rstest = "0.12"
+rstest = "0.17"
 serde = { workspace = true }
 serde_json = { workspace = true }
 shlex = "1.0.0"
 tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "time"] }
 tracing = { workspace = true }
 uuid = { workspace = true, features = ["serde", "v4"] }
-u_lib = { path = "../lib/u_lib", features = ["panel"] }
+u_lib = { path = "../lib/u_lib", features = ["panel", "server"] }
 
 
 [[test]]
-name = "integration"
+name = "integration-tests"
 path = "tests/lib.rs"

integration-tests/docker-compose.yml

@@ -23,17 +23,17 @@ services:
       u_db:
         condition: service_healthy
     ports:
-      - 63714:63714
+      - 9990:9990
     env_file:
       - ../.env
       - ../.env.private
     environment:
-      RUST_LOG: warp=info,u_server_lib=debug
+      RUST_LOG: warp=info,u_server_lib=debug,u_lib=debug
     healthcheck:
-        test: ss -tlpn | grep 63714
-        interval: 5s
-        timeout: 2s
-        retries: 2
+      test: ss -tlpn | grep 9990
+      interval: 5s
+      timeout: 2s
+      retries: 2
 
   u_db:
     image: localhost/unki/u_db
@@ -51,11 +51,11 @@ services:
         target: /u_db_entrypoint.sh
     command: /u_db_entrypoint.sh
     healthcheck:
-        # test if db's port is open and db is created
-        test: ss -tlpn | grep 5432 && psql -lqt -U $${POSTGRES_USER} | grep -qw $${POSTGRES_DATABASE}
-        interval: 5s
-        timeout: 5s
-        retries: 3
+      # test if db's port is open and db is created
+      test: ss -tlpn | grep 5432 && psql -lqt -U $${POSTGRES_USER} | grep -qw $${POSTGRES_DATABASE}
+      interval: 5s
+      timeout: 5s
+      retries: 3
 
   u_agent:
     user: *user
@@ -83,13 +83,13 @@ services:
     volumes:
       - ${HOME}/.cargo/registry/:/usr/local/cargo/registry/
       - ../__Cargo_integration.toml:/tests/Cargo.toml
-      - ./:/tests/integration/
+      - ./:/tests/integration-tests/
       - ../certs:/tests/certs
       - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_panel:/u_panel
       - ../lib/u_lib:/tests/lib/u_lib
-      - ../logs:/tests/integration/logs:rw
+      - ../logs:/tests/integration-tests/logs:rw
     working_dir:
-      /tests/integration/
+      /tests/integration-tests/
     depends_on:
       u_agent:
         condition: service_started
@@ -100,4 +100,5 @@ services:
       - ../.env.private
     environment:
       RUST_BACKTRACE: 1
+      RUST_LOG: hyper=info,reqwest=info
       U_SERVER: u_server

integration-tests/docker.py

@@ -90,7 +90,7 @@ class Compose:
     ]
 
     def __init__(self):
-        self.container_tpl = 'integration-%s-%d'
+        self.container_tpl = 'integration-tests-%s-%d'
         self.cmd_container = self.container_tpl % ('tests_runner', 1)
         self.ALL_CONTAINERS = [self.container_tpl %
                                (c, 1) for c in self.ALL_IMAGES]

integration-tests/integration_tests.py

@@ -6,7 +6,7 @@ from docker import rebuild_images_if_needed, Compose
 from pathlib import Path
 from utils import *
 
-CARGO_INTEGRATION_TOML = Path('../__Cargo_integration.toml')
+CARGO_INTEGRATION_TESTS_TOML = Path('../__Cargo_integration.toml')
 CLUSTER = Compose()
 
 
@@ -22,12 +22,12 @@ def usage_exit():
     fail(usage)
 
 
-def create_integration_workspace():
-    if CARGO_INTEGRATION_TOML.exists():
-        CARGO_INTEGRATION_TOML.unlink()
+def create_integration_tests_workspace():
+    if CARGO_INTEGRATION_TESTS_TOML.exists():
+        CARGO_INTEGRATION_TESTS_TOML.unlink()
     workspace = toml.load('../Cargo.toml')
-    workspace['workspace']['members'] = ['integration']
-    with open(CARGO_INTEGRATION_TOML, 'w') as fo:
+    workspace['workspace']['members'] = ['integration-tests']
+    with open(CARGO_INTEGRATION_TESTS_TOML, 'w') as fo:
         toml.dump(workspace, fo)
 
 
@@ -44,7 +44,7 @@ def run_tests():
     def _cleanup():
         if not preserve_containers and not only_setup_cluster:
             CLUSTER.down()
-        CARGO_INTEGRATION_TOML.unlink(missing_ok=True)
+        CARGO_INTEGRATION_TESTS_TOML.unlink(missing_ok=True)
 
     def abort_handler(s, _):
         warn(f'Received signal: {s}, gracefully stopping...')
@@ -53,16 +53,16 @@ def run_tests():
     if down_cluster:
         _cleanup()
         return
-        
+
     for s in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP):
         signal.signal(s, abort_handler)
     rebuild_images_if_needed(force_rebuild)
-    create_integration_workspace()
+    create_integration_tests_workspace()
     try:
         CLUSTER.up()
         CLUSTER.is_alive()
         if not only_setup_cluster:
-            CLUSTER.run('cargo test --test integration')
+            CLUSTER.run('cargo test --test integration-tests')
     except Exception as e:
         #CLUSTER.print_containers_logs()
         fail(e)

integration-tests/integration_tests.sh

@@ -2,5 +2,7 @@
 set -e
 export DOCKER_UID=$(id -u)
 export DOCKER_GID=$(id -g)
+
+rm ../logs/u_agent* || true
 [[ "$@" =~ "--release" ]] && export PROFILE=release || export PROFILE=debug
 python integration_tests.py $@

integration-tests/tests/fixtures/agent.rs

@@ -0,0 +1,38 @@
+use super::connections::*;
+use super::run_async;
+use u_lib::{api::HttpClient, messaging::Reportable, models::*, types::Id};
+
+pub struct RegisteredAgent {
+    pub id: Id,
+}
+
+#[fixture]
+#[once]
+pub fn registered_agent(client: &HttpClient) -> RegisteredAgent {
+    run_async(async {
+        let agent = Agent::with_current_platform();
+        let agent_id = agent.id;
+        println!("registering agent {agent_id}");
+        debug!("registering agent1 {agent_id}");
+        let resp = client
+            .get_personal_jobs(agent_id)
+            .await
+            .unwrap()
+            .pop()
+            .unwrap();
+        let job_id = resp.job_id;
+        let job = client.get_job(job_id, Brief::No).await.unwrap();
+
+        assert_eq!(job.meta.alias, Some("agent_hello".to_string()));
+
+        let mut agent_data = AssignedJob::from((&job.meta, resp));
+        agent_data.set_result(&agent);
+
+        client
+            .report([Reportable::Assigned(agent_data)])
+            .await
+            .unwrap();
+
+        RegisteredAgent { id: agent_id }
+    })
+}

integration-tests/tests/fixtures/connections.rs

@@ -0,0 +1,28 @@
+use super::env::*;
+use super::run_async;
+
+pub use u_lib::api::HttpClient;
+use u_lib::db::unpooled;
+pub use u_lib::db::PgConnection;
+
+#[fixture]
+#[once]
+pub fn client(env_default: EndpointsEnv) -> HttpClient {
+    run_async(HttpClient::new(&env_default.u_server, None)).unwrap()
+}
+
+#[fixture]
+#[once]
+pub fn client_panel(env_access: AccessEnv) -> HttpClient {
+    run_async(HttpClient::new(
+        &env_access.u_server,
+        Some(env_access.admin_auth_token),
+    ))
+    .unwrap()
+}
+
+#[fixture]
+#[once]
+pub fn db(env_db: DBEnv) -> PgConnection {
+    unpooled(&env_db)
+}

integration-tests/tests/fixtures/env.rs

@@ -0,0 +1,16 @@
+pub use u_lib::config::{AccessEnv, DBEnv, EndpointsEnv};
+
+#[fixture]
+pub fn env_default() -> EndpointsEnv {
+    EndpointsEnv::load()
+}
+
+#[fixture]
+pub fn env_access() -> AccessEnv {
+    AccessEnv::load().unwrap()
+}
+
+#[fixture]
+pub fn env_db() -> DBEnv {
+    DBEnv::load().unwrap()
+}

integration-tests/tests/fixtures/mod.rs

@@ -0,0 +1,17 @@
+pub mod agent;
+pub mod connections;
+pub mod env;
+
+use std::future::Future;
+use std::thread;
+use tokio::runtime::Runtime;
+
+// tokio runtime cannot be created inside another runtime,
+// so i create a separate non-'static thread not to interfere
+fn run_async<R: Send>(fut: impl Future<Output = R> + Send) -> R {
+    thread::scope(|s| {
+        s.spawn(|| Runtime::new().unwrap().block_on(fut))
+            .join()
+            .expect("async task failed")
+    })
+}

integration-tests/tests/helpers/mod.rs

@@ -0,0 +1,4 @@
+pub mod jobs;
+pub mod panel;
+
+pub use panel::Panel;

integration-tests/tests/helpers/panel.rs

@@ -10,13 +10,22 @@ pub struct Panel;
 
 impl Panel {
     fn run(args: &[&str]) -> Output {
-        Command::new(PANEL_BINARY).args(args).output().unwrap()
+        Command::new(PANEL_BINARY)
+            .env("RUST_LOG", "u_lib=debug")
+            .args(args)
+            .output()
+            .unwrap()
     }
 
     pub fn output_argv<T: DeserializeOwned>(argv: &[&str]) -> PanelResult<T> {
         let result = Self::run(argv);
         let output = ProcOutput::from_output(&result);
 
+        let stderr = output.get_stderr();
+        if !stderr.is_empty() {
+            println!("\n{}\n", String::from_utf8_lossy(stderr));
+        }
+
         match from_slice(output.get_stdout()) {
             Ok(r) => r,
             Err(e) => {
@@ -40,8 +49,8 @@ impl Panel {
                 .as_ref(),
         );
         match &result {
-            PanelResult::Ok(r) => eprintln!("+<< {r:?}"),
-            PanelResult::Err(e) => eprintln!("!<< {e:?}"),
+            PanelResult::Ok(r) => eprintln!("+<< {r:#?}"),
+            PanelResult::Err(e) => eprintln!("!<< {e:#?}"),
         }
         result
     }

integration-tests/tests/integration_tests/behaviour.rs

@@ -2,19 +2,17 @@ use crate::fixtures::agent::*;
 use crate::helpers::{jobs::retry_with_interval, Panel};
 
 use rstest::rstest;
-use serde_json::{json, to_string};
+use serde_json::to_string;
 use u_lib::config::AGENT_ITERATION_INTERVAL;
 use u_lib::models::*;
-use uuid::Uuid;
 
 #[rstest]
 #[tokio::test]
-async fn registration(#[future] register_agent: RegisteredAgent) {
-    let agent = register_agent.await;
+async fn registration(registered_agent: &RegisteredAgent) {
     let agents: Vec<Agent> = Panel::check_output("agents read");
-    let found = agents.iter().find(|v| v.id == agent.id);
+    let found = agents.iter().find(|v| v.id == registered_agent.id);
     assert!(found.is_some());
-    Panel::check_status(format!("agents delete {}", agent.id));
+    Panel::check_status(format!("agents delete {}", registered_agent.id));
 }
 
 #[tokio::test]
@@ -22,19 +20,28 @@ async fn setup_tasks() {
     let agents: Vec<Agent> = Panel::check_output("agents read");
     let agent_id = agents[0].id;
     let job_alias = "passwd_contents";
-    let job = json!(
-        {"alias": job_alias, "payload": b"cat /etc/passwd", "argv": "/bin/bash {}" }
-    );
+    let job = RawJob::default()
+        .with_alias(job_alias)
+        .with_raw_payload("cat /etc/passwd")
+        .with_shell("/bin/bash {}")
+        .with_target_platforms("*linux*")
+        .try_into_job()
+        .unwrap();
+    let job_id = job.meta.id;
+
+    Panel::check_status(["jobs", "create", &to_string(&RawJob::from(job)).unwrap()]);
 
-    Panel::check_status(["jobs", "create", &to_string(&job).unwrap()]);
+    let assigned = AssignedJobById {
+        agent_id,
+        job_id,
+        ..Default::default()
+    };
 
-    let cmd = format!("map create {} {}", agent_id, job_alias);
-    let assigned_ids: Vec<Uuid> = Panel::check_output(cmd);
+    Panel::check_status(["map", "create", &to_string(&[assigned]).unwrap()]);
 
     retry_with_interval(5, AGENT_ITERATION_INTERVAL, || {
         let result =
-            Panel::check_output::<Vec<AssignedJob>>(format!("map read {}", assigned_ids[0]))
-                .remove(0);
+            Panel::check_output::<Vec<AssignedJob>>(format!("map read {}", job_id)).remove(0);
         if result.state == JobState::Finished {
             eprintln!("{}", result.to_str_result());
             assert!(result.to_str_result().contains("root:x:0:0"));
@@ -48,26 +55,30 @@ async fn setup_tasks() {
 
 #[tokio::test]
 async fn large_payload() {
-    let agent_id = Panel::check_output::<Vec<Agent>>("agents read")[0].id;
-
+    let agent = &Panel::check_output::<Vec<Agent>>("agents read")[0];
+    let agent_id = agent.id;
     let job_alias = "large_payload";
-
-    let job = FatJobMeta::builder()
+    let job = RawJob::default()
         .with_alias(job_alias)
         .with_payload_path("./tests/bin/echoer")
         .with_shell("{} type echo")
-        .build()
+        .with_target_platforms(&agent.platform)
+        .try_into_job()
         .unwrap();
+    let job_id = job.meta.id;
 
-    Panel::check_status(["jobs", "create", &to_string(&job).unwrap()]);
+    Panel::check_status(["jobs", "create", &to_string(&RawJob::from(job)).unwrap()]);
 
-    let cmd = format!("map create {agent_id} {job_alias}");
-    let assigned_ids: Vec<Uuid> = Panel::check_output(cmd);
+    let assigned = AssignedJobById {
+        agent_id,
+        job_id,
+        ..Default::default()
+    };
+    Panel::check_status(["map", "create", &to_string(&[assigned]).unwrap()]);
 
     retry_with_interval(5, AGENT_ITERATION_INTERVAL, || {
         let result =
-            Panel::check_output::<Vec<AssignedJob>>(format!("map read {}", assigned_ids[0]))
-                .remove(0);
+            Panel::check_output::<Vec<AssignedJob>>(format!("map read {}", job_id)).remove(0);
         if result.state == JobState::Finished {
             assert_eq!(result.to_str_result(), "type echo\n");
             Ok(())

integration-tests/tests/integration_tests/connection.rs

@@ -1,14 +1,15 @@
-use crate::helpers::ENV;
+use crate::fixtures::env::*;
 use u_lib::config::MASTER_PORT;
 
+#[rstest]
 #[tokio::test]
-async fn non_auth_connection_dropped() {
+async fn non_auth_connection_dropped(env_default: EndpointsEnv) {
     let client = reqwest::ClientBuilder::new()
         .danger_accept_invalid_certs(true)
         .build()
         .unwrap();
     match client
-        .get(format!("https://{}:{}", &ENV.u_server, MASTER_PORT))
+        .get(format!("https://{}:{}", &env_default.u_server, MASTER_PORT))
         .send()
         .await
     {

integration-tests/tests/integration_tests/endpoints.rs

@@ -0,0 +1,97 @@
+// get_personal_jobs(&self, url_param: Id)
+// report(&self, payload: impl OneOrVec<messaging::Reportable>)
+// dl(&self, file: String)
+// get_job(&self, job: Id)
+// get_jobs(&self)
+// get_agents(&self, agent: Option<Id>)
+// update_agent(&self, agent: Agent)
+// update_job(&self, job: FatJob)
+// update_result(&self, result: AssignedJob)
+// upload_jobs(&self, payload: impl OneOrVec<FatJob>)
+// del(&self, item: Id)
+// assign_jobs(&self, agent: Id, job_idents: impl OneOrVec<String>)
+// get_agent_jobs(&self, agent: Option<Id>)
+// ping(&self)
+
+use crate::fixtures::connections::*;
+use std::iter::repeat;
+use u_lib::models::{Brief, RawJob, RawPayload, MAX_READABLE_PAYLOAD_SIZE};
+
+#[rstest]
+#[tokio::test]
+async fn jobs_upload_update_get_del(client_panel: &HttpClient) {
+    let job_alias = "henlo";
+    let mut job = RawJob::default()
+        .with_shell("/bin/bash {}")
+        .with_raw_payload("echo henlo")
+        .with_alias(job_alias)
+        .try_into_job()
+        .unwrap();
+
+    let job_id = job.meta.id;
+
+    client_panel.upload_jobs([&job]).await.unwrap();
+
+    let fetched_job = client_panel.get_full_job(job_id).await.unwrap();
+    assert_eq!(job, fetched_job);
+
+    let new_alias = "henlo2".to_string();
+    job.meta.alias = Some(new_alias.clone());
+    client_panel.update_job(&job.meta).await.unwrap();
+
+    let fetched_job = client_panel.get_full_job(job_id).await.unwrap();
+    assert_eq!(
+        fetched_job.payload.as_ref().unwrap().data.as_ref().unwrap(),
+        b"echo henlo"
+    );
+    assert_eq!(fetched_job.meta.alias, Some(new_alias));
+
+    client_panel.del(job_id).await.unwrap();
+
+    let not_found_err = client_panel.get_brief_job(job_id).await.unwrap_err();
+    assert!(not_found_err.to_string().contains("404 Not Found"))
+}
+
+#[rstest]
+#[tokio::test]
+async fn payloads_upload_update_get_del(client_panel: &HttpClient) {
+    let name = "test1".to_string();
+    let data = b"qweasdzxc".to_vec();
+    let payload = RawPayload {
+        name: Some(name.clone()),
+        data: data.clone(),
+    };
+
+    client_panel.upload_payload(&payload).await.unwrap();
+
+    let mut fetched_payload = client_panel.get_payload(&name, Brief::No).await.unwrap();
+    let fetched_payload_auto = client_panel.get_payload(&name, Brief::Auto).await.unwrap();
+
+    assert_eq!(fetched_payload, fetched_payload_auto);
+    assert_eq!(fetched_payload.data.unwrap(), data);
+
+    let new_size = MAX_READABLE_PAYLOAD_SIZE + 1;
+    let big_data = repeat(1u8).take(new_size as usize).collect::<Vec<_>>();
+
+    fetched_payload.data = Some(big_data.clone());
+    client_panel.update_payload(&fetched_payload).await.unwrap();
+
+    let fetched_big_payload = client_panel.get_payload(&name, Brief::Yes).await.unwrap();
+    let fetched_big_payload_auto = client_panel.get_payload(&name, Brief::Auto).await.unwrap();
+
+    assert_eq!(fetched_big_payload, fetched_big_payload_auto);
+    assert_eq!(fetched_big_payload.size, new_size);
+    assert!(fetched_big_payload.data.is_none());
+
+    let fetched_big_payload_full = client_panel.get_payload(&name, Brief::No).await.unwrap();
+
+    assert_eq!(fetched_big_payload_full.data.unwrap(), big_data);
+
+    client_panel.del(fetched_big_payload_full.id).await.unwrap();
+
+    let not_found_err = client_panel
+        .get_payload(&name, Brief::Yes)
+        .await
+        .unwrap_err();
+    assert!(not_found_err.to_string().contains("404 Not Found"))
+}

integration-tests/tests/integration_tests/mod.rs

@@ -1,2 +1,3 @@
 mod behaviour;
 mod connection;
+mod endpoints;

integration-tests/tests/lib.rs

@@ -0,0 +1,14 @@
+mod fixtures;
+mod helpers;
+mod integration_tests;
+
+#[macro_use]
+extern crate rstest;
+
+#[macro_use]
+extern crate tracing;
+
+#[ctor::ctor]
+fn __init() {
+    u_lib::logging::init_logger(None);
+}

integration/tests/fixtures/agent.rs

@@ -1,36 +0,0 @@
-use crate::helpers::ENV;
-use u_lib::{
-    api::ClientHandler, config::get_self_id, jobs::fat_meta_to_thin, messaging::Reportable,
-    models::*, types::Id,
-};
-
-pub struct RegisteredAgent {
-    pub id: Id,
-}
-
-impl RegisteredAgent {
-    pub async fn unregister(self) {
-        let cli = ClientHandler::new(&ENV.u_server, None).await.unwrap();
-        cli.del(self.id).await.unwrap();
-    }
-}
-
-#[fixture]
-pub async fn register_agent() -> RegisteredAgent {
-    let cli = ClientHandler::new(&ENV.u_server, None).await.unwrap();
-    let agent_id = get_self_id();
-    println!("registering agent {agent_id}");
-    let resp = cli
-        .get_personal_jobs(agent_id)
-        .await
-        .unwrap()
-        .pop()
-        .unwrap();
-    let job_id = resp.job_id;
-    let job = cli.get_job(job_id).await.unwrap();
-    assert_eq!(job.alias, Some("agent_hello".to_string()));
-    let mut agent_data = AssignedJob::from((&fat_meta_to_thin(job).unwrap(), resp));
-    agent_data.set_result(&Agent::with_id(agent_id));
-    cli.report(Reportable::Assigned(agent_data)).await.unwrap();
-    RegisteredAgent { id: agent_id }
-}

integration/tests/fixtures/mod.rs

@@ -1 +0,0 @@
-pub mod agent;

integration/tests/helpers/mod.rs

@@ -1,9 +0,0 @@
-pub mod jobs;
-pub mod panel;
-
-pub use panel::Panel;
-
-use once_cell::sync::Lazy;
-use u_lib::config::EndpointsEnv;
-
-pub static ENV: Lazy<EndpointsEnv> = Lazy::new(|| EndpointsEnv::load());

integration/tests/lib.rs

@@ -1,6 +0,0 @@
-mod fixtures;
-mod helpers;
-mod integration;
-
-#[macro_use]
-extern crate rstest;

lib/u_lib/Cargo.toml

@@ -14,7 +14,6 @@ diesel-derive-enum = { version = "2.0.0", features = ["postgres"], optional = tr
 deadpool-diesel = { workspace = true, optional = true }
 dotenv = "0.15.0"
 envy = "0.4.2"
-futures = "0.3.5"
 guess_host_triple = "0.1.2"
 libc = "^0.2"
 lazy_static = "1.4.0"
@@ -32,12 +31,15 @@ tracing-appender = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
 uuid = { workspace = true, features = ["serde", "v4"] }
 parking_lot = "0.12.1"
+bincode = "1.3.3"
+sha3 = "0.10.7"
 
 [target.'cfg(unix)'.dependencies]
-daemonize = "0.4.1"
+daemonize = "0.5"
 nix = "0.17"
 
 [features]
+agent = []
 panel = []
 server = ["dep:diesel", "dep:diesel-derive-enum", "dep:deadpool-diesel"]