some progress

- add payloads page
- refactor frontend to more nice arch
- rename integration to integration-tests
pull/9/head
plazmoid 1 year ago
parent f5d2190fc4
commit a38e3a1561
  1. 84
      Cargo.lock
  2. 2
      Cargo.toml
  3. 6
      Makefile.toml
  4. 12
      bin/u_agent/src/lib.rs
  5. 46
      bin/u_panel/src/argparse.rs
  6. 7
      bin/u_panel/src/gui/fe/src/app/app-routing.module.ts
  7. 1
      bin/u_panel/src/gui/fe/src/app/app.component.html
  8. 5
      bin/u_panel/src/gui/fe/src/app/app.component.ts
  9. 15
      bin/u_panel/src/gui/fe/src/app/app.module.ts
  10. 0
      bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.html
  11. 6
      bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.ts
  12. 0
      bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.html
  13. 28
      bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts
  14. 0
      bin/u_panel/src/gui/fe/src/app/components/dialogs/base-info-dialog.component.less
  15. 5
      bin/u_panel/src/gui/fe/src/app/components/dialogs/index.ts
  16. 52
      bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html
  17. 46
      bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts
  18. 24
      bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html
  19. 14
      bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts
  20. 0
      bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.html
  21. 6
      bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.ts
  22. 0
      bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.html
  23. 0
      bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.less
  24. 34
      bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.ts
  25. 2
      bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.html
  26. 42
      bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts
  27. 0
      bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.less
  28. 58
      bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts
  29. 5
      bin/u_panel/src/gui/fe/src/app/components/tables/index.ts
  30. 4
      bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.html
  31. 50
      bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.ts
  32. 60
      bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.html
  33. 30
      bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts
  34. 4
      bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.html
  35. 36
      bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts
  36. 14
      bin/u_panel/src/gui/fe/src/app/core/models/index.ts
  37. 11
      bin/u_panel/src/gui/fe/src/app/core/models/job.model.ts
  38. 53
      bin/u_panel/src/gui/fe/src/app/core/services/api.service.ts
  39. 52
      bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.ts
  40. 33
      bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/assign_job.component.ts
  41. 4
      bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/index.ts
  42. 44
      bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job-info-dialog.html
  43. 30
      bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job_info.component.ts
  44. 3
      bin/u_panel/src/gui/fe/src/app/core/tables/index.ts
  45. 59
      bin/u_panel/src/gui/fe/src/app/core/tables/job.component.ts
  46. 41
      bin/u_panel/src/gui/fe/src/app/core/tables/result.component.ts
  47. 84
      bin/u_panel/src/gui/fe/src/app/core/tables/table.component.ts
  48. 0
      bin/u_panel/src/gui/fe/src/app/index.ts
  49. 4
      bin/u_panel/src/gui/fe/src/app/models/agent.model.ts
  50. 24
      bin/u_panel/src/gui/fe/src/app/models/index.ts
  51. 16
      bin/u_panel/src/gui/fe/src/app/models/job.model.ts
  52. 17
      bin/u_panel/src/gui/fe/src/app/models/payload.model.ts
  53. 4
      bin/u_panel/src/gui/fe/src/app/models/result.model.ts
  54. 138
      bin/u_panel/src/gui/fe/src/app/services/api.service.ts
  55. 17
      bin/u_panel/src/gui/fe/src/app/services/error.service.ts
  56. 0
      bin/u_panel/src/gui/fe/src/app/services/index.ts
  57. 0
      bin/u_panel/src/gui/fe/src/app/utils.ts
  58. 6
      bin/u_panel/src/main.rs
  59. 1
      bin/u_server/Cargo.toml
  60. 36
      bin/u_server/src/db.rs
  61. 2
      bin/u_server/src/error.rs
  62. 110
      bin/u_server/src/handlers.rs
  63. 69
      bin/u_server/src/u_server.rs
  64. 0
      integration-tests/Cargo.lock
  65. 5
      integration-tests/Cargo.toml
  66. 25
      integration-tests/docker-compose.yml
  67. 2
      integration-tests/docker.py
  68. 18
      integration-tests/integration_tests.py
  69. 2
      integration-tests/integration_tests.sh
  70. 0
      integration-tests/src/main.rs
  71. 7
      integration-tests/tests/fixtures/agent.rs
  72. 0
      integration-tests/tests/fixtures/connections.rs
  73. 0
      integration-tests/tests/fixtures/env.rs
  74. 0
      integration-tests/tests/fixtures/mod.rs
  75. 0
      integration-tests/tests/helpers/jobs.rs
  76. 0
      integration-tests/tests/helpers/mod.rs
  77. 14
      integration-tests/tests/helpers/panel.rs
  78. 28
      integration-tests/tests/integration_tests/api.rs
  79. 4
      integration-tests/tests/integration_tests/behaviour.rs
  80. 0
      integration-tests/tests/integration_tests/connection.rs
  81. 0
      integration-tests/tests/integration_tests/mod.rs
  82. 14
      integration-tests/tests/lib.rs
  83. 0
      integration-tests/utils.py
  84. 6
      integration/tests/lib.rs
  85. 93
      lib/u_lib/src/api.rs
  86. 4
      lib/u_lib/src/cache.rs
  87. 14
      lib/u_lib/src/combined_result.rs
  88. 22
      lib/u_lib/src/error/mod.rs
  89. 108
      lib/u_lib/src/jobs.rs
  90. 23
      lib/u_lib/src/logging.rs
  91. 12
      lib/u_lib/src/messaging.rs
  92. 32
      lib/u_lib/src/misc.rs
  93. 61
      lib/u_lib/src/models/jobs/meta.rs
  94. 21
      lib/u_lib/src/models/mod.rs
  95. 33
      lib/u_lib/src/models/payload.rs
  96. 16
      lib/u_lib/src/ufs/mod.rs

84
Cargo.lock generated

@ -235,9 +235,9 @@ dependencies = [
[[package]]
name = "aho-corasick"
version = "0.7.20"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04"
dependencies = [
"memchr",
]
@ -368,9 +368,9 @@ dependencies = [
[[package]]
name = "bumpalo"
version = "3.12.0"
version = "3.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8"
[[package]]
name = "byteorder"
@ -489,9 +489,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
[[package]]
name = "cpufeatures"
version = "0.2.6"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181"
checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58"
dependencies = [
"libc",
]
@ -558,6 +558,16 @@ dependencies = [
"typenum",
]
[[package]]
name = "ctor"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd4056f63fce3b82d852c3da92b08ea59959890813a7f4ce9c0ff85b10cf301b"
dependencies = [
"quote",
"syn 2.0.15",
]
[[package]]
name = "cxx"
version = "1.0.94"
@ -1219,9 +1229,10 @@ dependencies = [
]
[[package]]
name = "integration"
name = "integration-tests"
version = "0.1.0"
dependencies = [
"ctor",
"once_cell",
"reqwest",
"rstest 0.17.0",
@ -1298,9 +1309,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.141"
version = "0.2.142"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317"
[[package]]
name = "libflate"
@ -1333,9 +1344,9 @@ dependencies = [
[[package]]
name = "linux-raw-sys"
version = "0.3.3"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b085a4f2cde5781fc4b1717f2e86c62f5cda49de7ba99a7c2eae02b61c9064c"
checksum = "36eb31c1778188ae1e64398743890d0877fef36d11521ac60406b42016e8c2cf"
[[package]]
name = "local-channel"
@ -1568,9 +1579,9 @@ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
[[package]]
name = "openssl"
version = "0.10.50"
version = "0.10.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e30d8bc91859781f0a943411186324d580f2bbeb71b452fe91ae344806af3f1"
checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56"
dependencies = [
"bitflags",
"cfg-if 1.0.0",
@ -1600,9 +1611,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]]
name = "openssl-sys"
version = "0.9.85"
version = "0.9.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d3d193fb1488ad46ffe3aaabc912cc931d02ee8518fe2959aea8ef52718b0c0"
checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e"
dependencies = [
"cc",
"libc",
@ -1830,13 +1841,13 @@ dependencies = [
[[package]]
name = "regex"
version = "1.7.3"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
"regex-syntax 0.7.1",
]
[[package]]
@ -1845,7 +1856,7 @@ version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
dependencies = [
"regex-syntax",
"regex-syntax 0.6.29",
]
[[package]]
@ -1854,6 +1865,12 @@ version = "0.6.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
[[package]]
name = "regex-syntax"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c"
[[package]]
name = "reqwest"
version = "0.11.16"
@ -2003,9 +2020,9 @@ dependencies = [
[[package]]
name = "rustix"
version = "0.37.12"
version = "0.37.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "722529a737f5a942fdbac3a46cee213053196737c5eaa3386d52e85b786f2659"
checksum = "d9b864d3c18a5785a05953adeed93e2dca37ed30f18e69bba9f30079d51f363f"
dependencies = [
"bitflags",
"errno 0.3.1",
@ -2148,6 +2165,20 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_qs"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0431a35568651e363364210c91983c1da5eb29404d9f0928b67d4ebcfa7d330c"
dependencies = [
"futures",
"percent-encoding",
"serde",
"thiserror",
"tracing",
"warp",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@ -2599,13 +2630,13 @@ dependencies = [
[[package]]
name = "tracing-attributes"
version = "0.1.23"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a"
checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
"syn 2.0.15",
]
[[package]]
@ -2631,9 +2662,9 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
version = "0.3.16"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70"
checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77"
dependencies = [
"matchers",
"nu-ansi-term",
@ -2772,6 +2803,7 @@ dependencies = [
"rstest 0.12.0",
"serde",
"serde_json",
"serde_qs",
"thiserror",
"tokio",
"tracing",

@ -6,7 +6,7 @@ members = [
"bin/u_run",
"bin/u_server",
"lib/u_lib",
"integration",
"integration-tests",
]
[workspace.dependencies]

@ -63,7 +63,7 @@ upx -9 $BINS
'''
[tasks.build]
dependencies = ["cargo_update", "cargo_build", "release_tasks"]
dependencies = ["cargo_build", "release_tasks"]
clear = true
[tasks.run]
@ -80,7 +80,7 @@ alias = "unit-tests"
dependencies = ["cargo_update"]
script = '''
[[ ! -d "./target/${TARGET}/${PROFILE_OVERRIDE}" ]] && echo 'No target folder. Build project first' && exit 1
cd ./integration
cd ./integration-tests
bash integration_tests.sh ${@}
'''
@ -88,7 +88,7 @@ bash integration_tests.sh ${@}
alias = "integration-tests"
[tasks.test]
dependencies = ["unit", "integration"]
dependencies = ["unit", "integration-tests"]
[tasks.gen_schema]
script = './scripts/gen_schema.sh'

@ -16,14 +16,14 @@ use u_lib::{
models::AssignedJobById,
};
pub async fn process_request(jobs: Vec<AssignedJobById>, client: &HttpClient) {
async fn process_request(jobs: Vec<AssignedJobById>, client: &HttpClient) {
if !jobs.is_empty() {
for jr in &jobs {
if !JobCache::contains(jr.job_id) {
info!("Fetching job: {}", &jr.job_id);
let fetched_job = loop {
//todo: use payload cache
match client.get_job(jr.job_id, true).await {
match client.get_full_job(jr.job_id).await {
Ok(result) => break result,
Err(err) => {
debug!("{:?} \nretrying...", err);
@ -64,7 +64,7 @@ async fn error_reporting(client: HttpClient) -> ! {
match ErrChan::recv().await {
Some(err) => {
'retry: for _ in 0..3 {
match client.report(&Reportable::Error(err.clone())).await {
match client.report([Reportable::Error(err.clone())]).await {
Ok(_) => break 'retry,
Err(e) => {
debug!("Reporting error: {:?}", e);
@ -98,7 +98,7 @@ async fn agent_loop(client: HttpClient) -> ! {
.collect();
if !result.is_empty() {
if let Err(err) = client.report(&result).await {
if let Err(err) = client.report(result).await {
ErrChan::send(err, "report").await;
}
}
@ -119,7 +119,7 @@ pub fn run_forever() -> ! {
.next()
.unwrap()
);
init_logger(Some(logfile_uid));
init_logger(Some(&logfile_uid));
} else {
#[cfg(unix)]
u_lib::unix::daemonize()
@ -139,7 +139,7 @@ pub fn run_forever() -> ! {
}
Err(e) => {
error!("client init failed: {}", e);
exit(7)
exit(7) // todo: wtf?
}
}
})

@ -4,7 +4,7 @@ use u_lib::{
api::HttpClient,
jobs::join_payload,
messaging::AsMsg,
models::{Agent, AssignedJob, RawJob},
models::{Agent, AssignedJob, BriefMode, BriefOrFullJob, RawJob},
types::Id,
types::PanelResult,
UError, UResult,
@ -14,13 +14,16 @@ use u_lib::{
pub struct Args {
#[structopt(subcommand)]
cmd: Cmd,
#[structopt(short, long, default_value)]
brief: BriefMode,
}
#[derive(StructOpt, Debug)]
enum Cmd {
Agents(RUD),
Jobs(JobCRUD),
Map(JobMapCRUD),
Map(MapCRUD),
Payloads(RUD),
Ping,
Serve,
}
@ -35,13 +38,7 @@ enum JobCRUD {
}
#[derive(StructOpt, Debug)]
enum JobCmd {
#[structopt(external_subcommand)]
Cmd(Vec<String>),
}
#[derive(StructOpt, Debug)]
enum JobMapCRUD {
enum MapCRUD {
Create {
#[structopt(parse(try_from_str = parse_uuid))]
agent_id: Id,
@ -92,35 +89,48 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult<Value> {
let raw_job = from_str::<RawJob>(&job)
.map_err(|e| UError::DeserializeError(e.to_string(), job))?;
let job = raw_job.validated()?;
let fat_job = join_payload(job)?;
let full_job = join_payload(job)?;
into_value(client.upload_jobs(&fat_job).await?)
into_value(
client
.upload_jobs([&BriefOrFullJob::Full(full_job)])
.await?,
)
}
JobCRUD::RUD(RUD::Read { id }) => match id {
//todo: use vec not to break frontend api, possibly refactor later
Some(id) => into_value(vec![client.get_job(id, false).await?]),
Some(id) => into_value(vec![client.get_job(id, args.brief).await?]),
None => into_value(client.get_jobs().await?),
},
JobCRUD::RUD(RUD::Update { item }) => {
let raw_job = from_str::<RawJob>(&item)
.map_err(|e| UError::DeserializeError(e.to_string(), item))?;
let job = raw_job.validated()?;
into_value(client.update_job(&join_payload(job)?).await?)
let full_job = join_payload(job)?;
into_value(client.update_job(&BriefOrFullJob::Full(full_job)).await?)
}
JobCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?),
},
Cmd::Map(action) => match action {
JobMapCRUD::Create {
MapCRUD::Create {
agent_id,
job_idents,
} => into_value(client.set_jobs(agent_id, &job_idents).await?),
JobMapCRUD::RUD(RUD::Read { id }) => into_value(client.get_agent_jobs(id).await?),
JobMapCRUD::RUD(RUD::Update { item }) => {
MapCRUD::RUD(RUD::Read { id }) => into_value(client.get_assigned_jobs(id).await?),
MapCRUD::RUD(RUD::Update { item }) => {
let assigned = from_str::<AssignedJob>(&item)
.map_err(|e| UError::DeserializeError(e.to_string(), item))?;
into_value(client.update_result(&assigned).await?)
}
JobMapCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?),
MapCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?),
},
Cmd::Payloads(action) => match action {
RUD::Read { id } => match id {
None => into_value(client.get_payloads().await?),
Some(id) => into_value(client.get_payload(id, args.brief).await?),
},
RUD::Update { item: _item } => todo!(),
RUD::Delete { id } => into_value(client.del(id).await?),
},
Cmd::Ping => into_value(client.ping().await?),
Cmd::Serve => {

@ -1,14 +1,13 @@
import { NgModule } from '@angular/core';
import { RouterModule, Routes } from '@angular/router';
import { AgentComponent } from './core/tables/agent.component';
import { JobComponent } from './core/tables/job.component';
import { ResultComponent } from './core/tables/result.component';
import { AgentInfoDialogComponent } from './core/tables/dialogs/agent_info.component';
import { JobComponent, ResultComponent, AgentComponent, PayloadComponent } from './components/tables';
//import { AgentInfoDialogComponent } from './core/tables/dialogs/agent-info-dialog.component';
const routes: Routes = [
{ path: '', redirectTo: 'agents', pathMatch: 'full' },
{ path: 'agents', component: AgentComponent },
{ path: 'jobs', component: JobComponent },
{ path: 'payloads', component: PayloadComponent },
{ path: 'results', component: ResultComponent },
];

@ -3,3 +3,4 @@
[active]="rla.isActive" [routerLinkActiveOptions]="{ exact: true }">{{tab.name}}</a>
</nav>
<router-outlet></router-outlet>
<global-error></global-error>

@ -1,4 +1,4 @@
import { Component, ViewChild, AfterViewInit } from '@angular/core';
import { Component } from '@angular/core';
@Component({
selector: 'app-root',
@ -9,6 +9,7 @@ export class AppComponent {
tabs = [
{ name: 'Agents', link: '/agents' },
{ name: 'Jobs', link: '/jobs' },
{ name: 'Results', link: '/results' }
{ name: 'Results', link: '/results' },
{ name: 'Payloads', link: '/payloads' }
];
}

@ -8,22 +8,25 @@ import { MatTableModule } from '@angular/material/table';
import { MatFormFieldModule } from '@angular/material/form-field';
import { MatButtonModule } from '@angular/material/button'
import { MatInputModule } from '@angular/material/input';
import { MatSelectModule } from '@angular/material/select';
import { MatProgressSpinnerModule } from '@angular/material/progress-spinner';
import { HttpClientModule } from '@angular/common/http';
import { MatDialogModule } from '@angular/material/dialog';
import { MatIconModule } from '@angular/material/icon';
import { FormsModule } from '@angular/forms';
import { AgentComponent, JobComponent, ResultComponent } from './core/tables';
import { AgentComponent, JobComponent, ResultComponent, PayloadComponent } from './components/tables';
import {
AgentInfoDialogComponent,
AssignJobDialogComponent,
JobInfoDialogComponent,
ResultInfoDialogComponent
} from './core/tables/dialogs';
ResultInfoDialogComponent,
PayloadInfoDialogComponent
} from './components/dialogs';
import { APP_BASE_HREF } from '@angular/common';
import { MatTooltipModule } from '@angular/material/tooltip';
import { MatSnackBarModule } from '@angular/material/snack-bar';
import { MatListModule } from '@angular/material/list';
import { GlobalErrorComponent } from './components/global-error/global-error.component';
@NgModule({
declarations: [
@ -34,7 +37,10 @@ import { MatListModule } from '@angular/material/list';
AgentInfoDialogComponent,
JobInfoDialogComponent,
ResultInfoDialogComponent,
AssignJobDialogComponent
AssignJobDialogComponent,
PayloadComponent,
PayloadInfoDialogComponent,
GlobalErrorComponent
],
imports: [
BrowserModule,
@ -50,6 +56,7 @@ import { MatListModule } from '@angular/material/list';
MatIconModule,
MatTooltipModule,
MatSnackBarModule,
MatSelectModule,
MatListModule,
FormsModule,
BrowserAnimationsModule

@ -1,12 +1,12 @@
import { Component, Inject } from '@angular/core';
import { MAT_DIALOG_DATA } from '@angular/material/dialog';
import { AgentModel } from '../../models/agent.model';
import { AgentModel } from '../../../models/agent.model';
import { EventEmitter } from '@angular/core';
@Component({
selector: 'agent-info-dialog',
templateUrl: 'agent-info-dialog.html',
styleUrls: ['info-dialog.component.less']
templateUrl: 'agent-info-dialog.component.html',
styleUrls: ['../base-info-dialog.component.less']
})
export class AgentInfoDialogComponent {
is_preview = true;

@ -0,0 +1,28 @@
import { Component, Inject } from '@angular/core';
import { MAT_DIALOG_DATA } from '@angular/material/dialog';
import { ApiTableService } from '../../../services';
@Component({
selector: 'assign-job-dialog',
templateUrl: 'assign-job-dialog.component.html',
styleUrls: []
})
export class AssignJobDialogComponent {
rows: string[] = [];
selected_rows: string[] = [];
constructor(
@Inject(MAT_DIALOG_DATA) public agent_id: string,
private dataSource: ApiTableService,
) {
dataSource.getJobs().subscribe(resp => {
this.rows = resp.map(j => `${j.id} ${j.alias}`)
})
}
assignSelectedJobs() {
const job_ids = this.selected_rows.map(row => row.split(' ', 1)[0]).join(' ');
const request = `${this.agent_id} ${job_ids}`
this.dataSource.createResult(request)
}
}

@ -0,0 +1,5 @@
export * from './agent-info-dialog/agent-info-dialog.component';
export * from './result-info-dialog/result-info-dialog.component';
export * from './job-info-dialog/job-info-dialog.component';
export * from './assign-job-dialog/assign-job-dialog.component';
export * from './payload-info-dialog/payload-info-dialog.component';

@ -0,0 +1,52 @@
<h2 mat-dialog-title *ngIf="isPreview">Job info</h2>
<h2 mat-dialog-title *ngIf="!isPreview">Editing job info</h2>
<mat-dialog-content>
<div class="info-dialog-forms-box-smol">
<mat-form-field class="info-dlg-field">
<mat-label>ID</mat-label>
<input matInput disabled value="{{data.job.id}}">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Alias</mat-label>
<input matInput [readonly]="isPreview" [(ngModel)]="data.job.alias">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Args</mat-label>
<input matInput [readonly]="isPreview" [(ngModel)]="data.job.argv">
</mat-form-field>
</div>
<div class="info-dialog-forms-box-smol">
<mat-form-field class="info-dlg-field">
<mat-label>Type</mat-label>
<input matInput [readonly]="isPreview" [(ngModel)]="data.job.exec_type">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Platform</mat-label>
<input matInput [readonly]="isPreview" [(ngModel)]="data.job.target_platforms">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Schedule</mat-label>
<input matInput [readonly]="isPreview" [(ngModel)]="data.job.schedule">
</mat-form-field>
</div>
<div class="info-dialog-forms-box">
<mat-form-field class="info-dlg-field">
<mat-label>Payload</mat-label>
<mat-select [disabled]="isPreview" [(value)]="data.job.payload">
<mat-option *ngFor="let pld of allPayloads" [value]="pld[0]">{{ pld[1] }}</mat-option>
</mat-select>
</mat-form-field>
<mat-form-field class="info-dlg-field" floatLabel="always">
<mat-label>Payload data</mat-label>
<textarea matInput cdkTextareaAutosize *ngIf="!isTooBigPayload" [readonly]="isPreview"
[(ngModel)]="decodedPayload">
</textarea>
<input matInput *ngIf="isTooBigPayload" disabled placeholder="Payload is too big to display">
</mat-form-field>
</div>
</mat-dialog-content>
<mat-dialog-actions align="end">
<button mat-raised-button *ngIf="isPreview" (click)="isPreview = false">Edit</button>
<button mat-raised-button *ngIf="!isPreview" (click)="updateJob()">Save</button>
<button mat-button mat-dialog-close>Cancel</button>
</mat-dialog-actions>

@ -0,0 +1,46 @@
import { Component, Inject } from '@angular/core';
import { MAT_DIALOG_DATA } from '@angular/material/dialog';
import { EventEmitter } from '@angular/core';
import { BriefOrFullJobModel } from '../../../models/job.model';
import { ApiTableService } from 'src/app/services';
import { BriefOrFullPayloadModel, isFullPayload } from 'src/app/models';
@Component({
selector: 'job-info-dialog',
templateUrl: 'job-info-dialog.component.html',
styleUrls: ['../base-info-dialog.component.less']
})
export class JobInfoDialogComponent {
isPreview = true;
isTooBigPayload = false;
decodedPayload = "";
//[id, name]
allPayloads: [string, string][] = [];
onSave = new EventEmitter();
constructor(@Inject(MAT_DIALOG_DATA) public data: BriefOrFullJobModel, dataSource: ApiTableService) {
if (data.payload !== null) {
this.showPayload(data.payload)
}
dataSource.getPayloads().subscribe(resp => {
this.allPayloads = resp.map(r => [r.id, r.name])
})
}
showPayload(payload: BriefOrFullPayloadModel) {
if (isFullPayload(payload)) {
this.decodedPayload = new TextDecoder().decode(new Uint8Array(payload.data))
} else {
this.isTooBigPayload = true
}
}
updateJob() {
// if (this.decodedPayload.length > 0) {
// this.data.payload = Array.from(new TextEncoder().encode(this.decodedPayload))
// }
// this.onSave.emit(this.data);
}
}

@ -0,0 +1,24 @@
<h2 mat-dialog-title>Result</h2>
<mat-dialog-content>
<div class="info-dialog-forms-box-smol">
<mat-form-field class="info-dlg-field" cdkFocusInitial>
<mat-label>ID</mat-label>
<input matInput readonly value="{{data.id}}">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Name</mat-label>
<input matInput value="{{data.name}}">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>MIME-type</mat-label>
<input matInput value="{{data.mime_type}}">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Size</mat-label>
<input matInput value="{{data.size}}">
</mat-form-field>
</div>
</mat-dialog-content>
<mat-dialog-actions align="end">
<button mat-button mat-dialog-close>Close</button>
</mat-dialog-actions>

@ -0,0 +1,14 @@
import { Component, Inject } from '@angular/core';
import { MAT_DIALOG_DATA } from '@angular/material/dialog';
import { PayloadModel } from 'src/app/models/payload.model';
@Component({
selector: 'payload-info-dialog',
templateUrl: 'payload-info-dialog.component.html',
styleUrls: []
})
export class PayloadInfoDialogComponent {
constructor(@Inject(MAT_DIALOG_DATA) public data: PayloadModel) { }
}

@ -1,11 +1,11 @@
import { Component, Inject } from '@angular/core';
import { MAT_DIALOG_DATA } from '@angular/material/dialog';
import { ResultModel } from '../../models/result.model';
import { ResultModel } from '../../../models/result.model';
@Component({
selector: 'result-info-dialog',
templateUrl: 'result-info-dialog.html',
styleUrls: ['info-dialog.component.less']
templateUrl: 'result-info-dialog.component.html',
styleUrls: ['../base-info-dialog.component.less']
})
export class ResultInfoDialogComponent {
decodedResult: string;

@ -0,0 +1,34 @@
import { Component, OnInit } from '@angular/core';
import { MatSnackBar, MatSnackBarConfig } from '@angular/material/snack-bar';
import { ErrorService } from 'src/app/services/error.service';
@Component({
selector: 'global-error',
templateUrl: './global-error.component.html',
styleUrls: ['./global-error.component.less']
})
export class GlobalErrorComponent implements OnInit {
constructor(
private snackBar: MatSnackBar,
private errorService: ErrorService
) { }
ngOnInit() {
this.errorService.error$.subscribe(err => {
const _config = (duration: number): MatSnackBarConfig => {
return {
horizontalPosition: 'right',
verticalPosition: 'bottom',
duration
}
}
const error = true;
const cfg = error ? _config(0) : _config(2000)
if (err != '') {
this.snackBar.open(err, 'Ok', cfg)
}
})
}
}

@ -6,7 +6,7 @@
</div>
<mat-form-field appearance="standard">
<mat-label>Filter</mat-label>
<input matInput (keyup)="apply_filter($event)" #input>
<input matInput (keyup)="applyFilter($event)" #input>
</mat-form-field>
<button id="refresh_btn" mat-raised-button color="primary" (click)="loadTableData()">Refresh</button>

@ -0,0 +1,42 @@
import { Component, OnInit } from '@angular/core';
import { TablesComponent } from '../base-table/base-table.component';
import { AgentModel, Area } from '../../../models';
import { AssignJobDialogComponent, AgentInfoDialogComponent } from '../../dialogs';
@Component({
selector: 'agent-table',
templateUrl: './agent-table.component.html',
styleUrls: ['../base-table/base-table.component.less'],
})
export class AgentComponent extends TablesComponent<AgentModel> implements OnInit {
area = 'agents' as Area
displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions']
showItemDialog(id: string) {
this.dataSource.getAgent(id).subscribe(resp => {
const dialog = this.infoDialog.open(AgentInfoDialogComponent, {
data: resp,
width: '1000px',
});
const saveSub = dialog.componentInstance.onSave.subscribe(result => {
this.dataSource.updateAgent(result).subscribe(_ => {
alert('Saved')
this.loadTableData()
})
})
dialog.afterClosed().subscribe(result => {
saveSub.unsubscribe()
this.router.navigate(['.'], { relativeTo: this.route })
})
})
}
assignJobs(id: string) {
const dialog = this.infoDialog.open(AssignJobDialogComponent, {
data: id,
width: '1000px',
});
}
}

@ -0,0 +1,58 @@
import { OnInit, Directive, Component } from '@angular/core';
import { ApiTableService } from '../../..';
import { MatTableDataSource } from '@angular/material/table';
import { MatDialog } from '@angular/material/dialog';
import { ApiModel, Area } from '../../../models';
import { ActivatedRoute, Router } from '@angular/router';
@Directive()
export abstract class TablesComponent<T extends ApiModel> implements OnInit {
abstract area: Area;
table_data: MatTableDataSource<T> = new MatTableDataSource;
isLoadingResults = true;
constructor(
public dataSource: ApiTableService,
public infoDialog: MatDialog,
public route: ActivatedRoute,
public router: Router,
) { }
ngOnInit() {
this.loadTableData();
this.route.queryParams.subscribe(params => {
const id = params['id']
const new_agent = params['new']
if (id) {
this.showItemDialog(id);
}
if (new_agent) {
this.showItemDialog(null);
}
})
//interval(10000).subscribe(_ => this.loadTableData());
}
loadTableData() {
this.isLoadingResults = true;
this.dataSource.getMany(this.area).subscribe(resp => {
this.isLoadingResults = false;
this.table_data.data = resp;
})
}
applyFilter(event: Event) {
const filterValue = (event.target as HTMLInputElement).value;
this.table_data.filter = filterValue.trim().toLowerCase();
}
deleteItem(id: string) {
if (confirm(`Delete ${id}?`)) {
this.dataSource.delete(id, this.area)
}
}
abstract displayedColumns: string[];
abstract showItemDialog(id: string | null): void;
}

@ -0,0 +1,5 @@
export * from './agent-table/agent-table.component';
export * from './base-table/base-table.component';
export * from './job-table/job-table.component';
export * from './payload-table/payload-table.component';
export * from './result-table/result-table.component';

@ -6,7 +6,7 @@
</div>
<mat-form-field appearance="standard">
<mat-label>Filter</mat-label>
<input matInput (keyup)="apply_filter($event)" #input>
<input matInput (keyup)="applyFilter($event)" #input>
</mat-form-field>
<button id="refresh_btn" mat-raised-button color="basic" (click)="loadTableData()">Refresh</button>
<button id="new_btn" mat-raised-button color="primary" routerLink='.' [queryParams]="{new: true}">Add
@ -39,7 +39,7 @@
<ng-container matColumnDef="platform">
<th mat-header-cell *matHeaderCellDef>Platform</th>
<td mat-cell *matCellDef="let row">
{{row.platform}}
{{row.target_platforms}}
</td>
</ng-container>

@ -0,0 +1,50 @@
import { Component, OnInit } from '@angular/core';
import { TablesComponent } from '../base-table/base-table.component';
import { Area, JobModel } from '../../../models';
import { JobInfoDialogComponent } from '../../dialogs';
@Component({
selector: 'job-table',
templateUrl: './job-table.component.html',
styleUrls: ['../base-table/base-table.component.less'],
providers: [{ provide: 'area', useValue: 'jobs' }]
})
export class JobComponent extends TablesComponent<JobModel> {
area = 'jobs' as Area;
displayedColumns = ['id', 'alias', 'platform', 'schedule', 'exec_type', 'actions']
showItemDialog(id: string | null) {
const show_dlg = (id: string, edit: boolean) => {
this.dataSource.getJob(id).subscribe(resp => {
var dialog = this.infoDialog.open(JobInfoDialogComponent, {
data: resp,
width: '1000px',
});
if (edit) {
dialog.componentInstance.isPreview = false
}
const saveSub = dialog.componentInstance.onSave.subscribe(result => {
this.dataSource.updateJob(result)
.subscribe(_ => {
alert("Saved")
this.loadTableData()
})
})
dialog.afterClosed().subscribe(result => {
saveSub.unsubscribe()
this.router.navigate(['.'], { relativeTo: this.route })
})
})
}
if (id) {
show_dlg(id, false)
} else {
this.dataSource.create(null, this.area).subscribe(resp => {
show_dlg(resp[0], true)
})
}
}
}

@ -0,0 +1,60 @@
<div class="mat-elevation-z8">
<div class="table-container">
<div class="loading-shade" *ngIf="isLoadingResults">
<mat-spinner *ngIf="isLoadingResults"></mat-spinner>
</div>
<mat-form-field appearance="standard">
<mat-label>Filter</mat-label>
<input matInput (keyup)="applyFilter($event)" #input>
</mat-form-field>
<button id="refresh_btn" mat-raised-button color="primary" (click)="loadTableData()">Refresh</button>
<table mat-table fixedLayout="true" [dataSource]="table_data" class="data-table" matSort matSortActive="id"
matSortDisableClear matSortDirection="desc">
<ng-container matColumnDef="name">
<th mat-header-cell *matHeaderCellDef>Name</th>
<td mat-cell *matCellDef="let row">
{{row.name}}
</td>
</ng-container>
<ng-container matColumnDef="mime_type">
<th mat-header-cell *matHeaderCellDef>MIME-type</th>
<td mat-cell *matCellDef="let row">
{{row.mime_type}}
</td>
</ng-container>
<ng-container matColumnDef="size">
<th mat-header-cell *matHeaderCellDef>Size</th>
<td mat-cell *matCellDef="let row">
{{row.size}}
</td>
</ng-container>
<ng-container matColumnDef="actions">
<th mat-header-cell *matHeaderCellDef></th>
<td mat-cell *matCellDef="let row">
<button mat-icon-button routerLink='.' [queryParams]="{id: row.id}">
<mat-icon>more_horiz</mat-icon>
</button>
|
<button mat-icon-button (click)="deleteItem(row.id)">
<mat-icon>delete</mat-icon>
</button>
</td>
</ng-container>
<tr mat-header-row *matHeaderRowDef="displayedColumns"></tr>
<tr mat-row class="data-table-row" *matRowDef="let row; columns: displayedColumns;"></tr>
<tr class="mat-row" *matNoDataRow>
<td class="mat-cell">No data</td>
</tr>
</table>
</div>
<!-- <mat-paginator [length]="resultsLength" [pageSize]="30" aria-label="Select page of GitHub search results">
</mat-paginator> -->
</div>

@ -0,0 +1,30 @@
import { Component } from '@angular/core';
import { Area } from 'src/app/models';
import { PayloadModel } from 'src/app/models/payload.model';
import { PayloadInfoDialogComponent } from '../../dialogs';
import { TablesComponent } from '../base-table/base-table.component';
@Component({
selector: 'payload-table',
templateUrl: './payload-table.component.html',
styleUrls: ['../base-table/base-table.component.less'],
providers: [{ provide: 'area', useValue: 'payloads' }]
})
export class PayloadComponent extends TablesComponent<PayloadModel> {
area = 'payloads' as Area
displayedColumns = ["name", "mime_type", "size"];
showItemDialog(id: string) {
this.dataSource.getPayload(id).subscribe(resp => {
const dialog = this.infoDialog.open(PayloadInfoDialogComponent, {
data: resp,
width: '1000px',
});
dialog.afterClosed().subscribe(_ => {
this.router.navigate(['.'], { relativeTo: this.route })
})
})
}
}

@ -6,7 +6,7 @@
</div>
<mat-form-field appearance="standard">
<mat-label>Filter</mat-label>
<input matInput (keyup)="apply_filter($event)" #input>
<input matInput (keyup)="applyFilter($event)" #input>
</mat-form-field>
<button id="refresh_btn" mat-raised-button color="primary" (click)="loadTableData()">Refresh</button>
@ -49,7 +49,7 @@
</ng-container>
<ng-container matColumnDef="last_updated">
<th mat-header-cell *matHeaderCellDef>ID</th>
<th mat-header-cell *matHeaderCellDef>Last updated</th>
<td mat-cell *matCellDef="let row">
{{row.updated.secs_since_epoch * 1000| date:'long'}}
</td>

@ -0,0 +1,36 @@
import { Component, OnInit } from '@angular/core';
import { TablesComponent } from '../base-table/base-table.component';
import { Area, ResultModel } from '../../../models';
import { ResultInfoDialogComponent } from '../../dialogs';
@Component({
selector: 'results-table',
templateUrl: './result-table.component.html',
styleUrls: ['../base-table/base-table.component.less'],
providers: [{ provide: 'area', useValue: 'map' }]
})
export class ResultComponent extends TablesComponent<ResultModel> {
area = 'map' as Area
displayedColumns = [
'id',
'alias',
'agent_id',
'job_id',
'state',
'last_updated',
'actions'
];
showItemDialog(id: string) {
this.dataSource.getResult(id).subscribe(resp => {
const dialog = this.infoDialog.open(ResultInfoDialogComponent, {
data: resp,
width: '1000px',
});
dialog.afterClosed().subscribe(_ => {
this.router.navigate(['.'], { relativeTo: this.route })
})
})
}
}

@ -1,14 +0,0 @@
export * from './agent.model';
export * from './result.model';
export * from './job.model';
export interface UTCDate {
secs_since_epoch: number,
nanos_since_epoch: number
}
export abstract class ApiModel { }
export interface Empty extends ApiModel { }
export type Area = "agents" | "jobs" | "map";

@ -1,11 +0,0 @@
import { ApiModel } from ".";
export interface JobModel extends ApiModel {
alias: string,
argv: string,
id: string,
exec_type: string,
platform: string,
payload: number[] | null,
schedule: string | null,
}

@ -1,53 +0,0 @@
import { environment } from 'src/environments/environment';
import { HttpClient } from '@angular/common/http';
import { firstValueFrom } from 'rxjs';
import { ApiModel, Empty, Area } from '../models';
interface ServerResponse<T extends ApiModel> {
status: "ok" | "err",
data: T | string
}
export class ApiTableService<T extends ApiModel> {
area: Area;
constructor(private http: HttpClient, area: Area) {
this.area = area;
}
requestUrl = `${environment.server}/cmd/`;
async req<R extends ApiModel>(cmd: string): Promise<ServerResponse<R>> {
return await firstValueFrom(this.http.post<ServerResponse<R>>(this.requestUrl, cmd))
}
async getOne(id: string, area: string = this.area): Promise<ServerResponse<T>> {
const resp = await this.req<T[]>(`${area} read ${id}`)
if (resp.data.length === 0) {
return {
status: 'err',
data: `${id} not found in ${area}`
}
}
return {
status: resp.status,
data: resp.data[0]
}
}
async getMany(): Promise<ServerResponse<T[]>> {
return await this.req(`${this.area} read`)
}
async update(item: T): Promise<ServerResponse<Empty>> {
return await this.req(`${this.area} update '${JSON.stringify(item)}'`)
}
async delete(id: string): Promise<ServerResponse<Empty>> {
return await this.req(`${this.area} delete ${id}`)
}
async create(item: string): Promise<ServerResponse<string[]>> {
return await this.req(`${this.area} create ${item}`)
}
}

@ -1,52 +0,0 @@
import { Component, OnDestroy, OnInit } from '@angular/core';
import { TablesComponent } from './table.component';
import { AgentModel } from '../models';
import { AgentInfoDialogComponent } from './dialogs/agent_info.component';
import { HttpErrorResponse } from '@angular/common/http';
import { AssignJobDialogComponent } from './dialogs';
@Component({
selector: 'agent-table',
templateUrl: './agent.component.html',
styleUrls: ['./table.component.less']
})
export class AgentComponent extends TablesComponent<AgentModel> implements OnInit {
//dialogSubscr!: Subscription;
area = 'agents' as const;
displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions']
show_item_dialog(id: string) {
this.data_source!.getOne(id).then(resp => {
if (resp.status === 'ok') {
const dialog = this.infoDialog.open(AgentInfoDialogComponent, {
data: resp.data as AgentModel,
width: '1000px',
});
const saveSub = dialog.componentInstance.onSave.subscribe(result => {
this.data_source!.update(result).then(_ => {
this.openSnackBar('Saved', false)
this.loadTableData()
})
.catch((err: HttpErrorResponse) => this.openSnackBar(err.error))
})
dialog.afterClosed().subscribe(result => {
saveSub.unsubscribe()
this.router.navigate(['.'], { relativeTo: this.route })
})
} else {
this.openSnackBar(resp.data)
}
}).catch((err: HttpErrorResponse) => this.openSnackBar(err.error))
}
assignJobs(id: string) {
const dialog = this.infoDialog.open(AssignJobDialogComponent, {
data: id,
width: '1000px',
});
}
}

@ -1,33 +0,0 @@
import { Component, Inject } from '@angular/core';
import { MAT_DIALOG_DATA } from '@angular/material/dialog';
import { HttpClient } from '@angular/common/http';
import { ApiTableService } from '../../services';
import { JobModel } from '../../models';
import { MatListOption } from '@angular/material/list';
@Component({
selector: 'assign-job-dialog',
templateUrl: 'assign-job-dialog.html',
styleUrls: []
})
export class AssignJobDialogComponent {
rows: string[] = [];
selected_rows: string[] = [];
constructor(@Inject(MAT_DIALOG_DATA) public agent_id: string, private http: HttpClient) {
new ApiTableService(http, "jobs").getMany().then(result => {
if (result.status == "ok") {
const jobs = result.data as JobModel[]
this.rows = jobs.map(j => `${j.id} ${j.alias}`)
} else {
alert(result.data as string)
}
}).catch(err => alert(err))
}
assignSelectedJobs() {
const job_ids = this.selected_rows.map(row => row.split(' ', 1)[0]).join(' ');
const request = `${this.agent_id} ${job_ids}`
new ApiTableService(this.http, "map").create(request).catch(err => alert(err))
}
}

@ -1,4 +0,0 @@
export * from './agent_info.component';
export * from './result_info.component';
export * from './job_info.component';
export * from './assign_job.component';

@ -1,44 +0,0 @@
<h2 mat-dialog-title *ngIf="is_preview">Job info</h2>
<h2 mat-dialog-title *ngIf="!is_preview">Editing job info</h2>
<mat-dialog-content>
<div class="info-dialog-forms-box-smol">
<mat-form-field class="info-dlg-field" cdkFocusInitial>
<mat-label>ID</mat-label>
<input matInput disabled value="{{data.id}}">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Alias</mat-label>
<input matInput [readonly]="is_preview" [(ngModel)]="data.alias">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Args</mat-label>
<input matInput [readonly]="is_preview" [(ngModel)]="data.argv">
</mat-form-field>
</div>
<div class="info-dialog-forms-box-smol">
<mat-form-field class="info-dlg-field">
<mat-label>Type</mat-label>
<input matInput [readonly]="is_preview" [(ngModel)]="data.exec_type">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Platform</mat-label>
<input matInput [readonly]="is_preview" [(ngModel)]="data.platform">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Schedule</mat-label>
<input matInput [readonly]="is_preview" [(ngModel)]="data.schedule">
</mat-form-field>
</div>
<div class="info-dialog-forms-box">
<mat-form-field class="info-dlg-field">
<mat-label>Payload</mat-label>
<textarea matInput cdkTextareaAutosize [readonly]="is_preview" [(ngModel)]="decodedPayload">
</textarea>
</mat-form-field>
</div>
</mat-dialog-content>
<mat-dialog-actions align="end">
<button mat-raised-button *ngIf="is_preview" (click)="is_preview = false">Edit</button>
<button mat-raised-button *ngIf="!is_preview" (click)="updateJob()">Save</button>
<button mat-button mat-dialog-close>Cancel</button>
</mat-dialog-actions>

@ -1,30 +0,0 @@
import { Component, Inject } from '@angular/core';
import { MAT_DIALOG_DATA } from '@angular/material/dialog';
import { EventEmitter } from '@angular/core';
import { JobModel } from '../../models/job.model';
@Component({
selector: 'job-info-dialog',
templateUrl: 'job-info-dialog.html',
styleUrls: ['info-dialog.component.less']
})
export class JobInfoDialogComponent {
is_preview = true;
decodedPayload: string;
onSave = new EventEmitter();
constructor(@Inject(MAT_DIALOG_DATA) public data: JobModel) {
if (data.payload !== null) {
this.decodedPayload = new TextDecoder().decode(new Uint8Array(data.payload))
} else {
this.decodedPayload = ""
}
}
updateJob() {
if (this.decodedPayload.length > 0) {
this.data.payload = Array.from(new TextEncoder().encode(this.decodedPayload))
}
this.onSave.emit(this.data);
}
}

@ -1,3 +0,0 @@
export * from './agent.component';
export * from './job.component';
export * from './result.component';

@ -1,59 +0,0 @@
import { Component, OnInit } from '@angular/core';
import { TablesComponent } from './table.component';
import { JobModel } from '../models';
import { JobInfoDialogComponent } from './dialogs';
import { HttpErrorResponse } from '@angular/common/http';
@Component({
selector: 'job-table',
templateUrl: './job.component.html',
styleUrls: ['./table.component.less']
})
export class JobComponent extends TablesComponent<JobModel> {
area = 'jobs' as const;
displayedColumns = ['id', 'alias', 'platform', 'schedule', 'exec_type', 'actions']
show_item_dialog(id: string | null) {
const show_dlg = (id: string, edit: boolean) => {
this.data_source!.getOne(id).then(resp => {
if (resp.status === 'ok') {
var dialog = this.infoDialog.open(JobInfoDialogComponent, {
data: resp.data as JobModel,
width: '1000px',
});
if (edit) {
dialog.componentInstance.is_preview = false
}
const saveSub = dialog.componentInstance.onSave.subscribe(result => {
this.data_source!.update(result)
.then(_ => {
this.openSnackBar("Saved", false)
this.loadTableData()
})
.catch((err: HttpErrorResponse) => this.openSnackBar(err.error))
})
dialog.afterClosed().subscribe(result => {
saveSub.unsubscribe()
this.router.navigate(['.'], { relativeTo: this.route })
})
} else {
this.openSnackBar(resp.data)
}
}).catch((err: any) => this.openSnackBar(err))
}
if (id) {
show_dlg(id, false)
} else {
this.data_source!.create('"{}"').then(resp => {
if (resp.status === 'ok') {
show_dlg(resp.data[0], true)
} else {
this.openSnackBar(resp.data)
}
}).catch((err: HttpErrorResponse) => this.openSnackBar(err.error))
}
}
}

@ -1,41 +0,0 @@
import { Component, OnInit } from '@angular/core';
import { TablesComponent } from './table.component';
import { ResultModel } from '../models';
import { ResultInfoDialogComponent } from './dialogs';
import { HttpErrorResponse } from '@angular/common/http';
@Component({
selector: 'results-table',
templateUrl: './result.component.html',
styleUrls: ['./table.component.less']
})
export class ResultComponent extends TablesComponent<ResultModel> {
area = 'map' as const;
displayedColumns = [
'id',
'alias',
'agent_id',
'job_id',
'state',
'last_updated',
'actions'
];
show_item_dialog(id: string) {
this.data_source!.getOne(id).then(resp => {
if (resp.status === 'ok') {
const dialog = this.infoDialog.open(ResultInfoDialogComponent, {
data: resp.data as ResultModel,
width: '1000px',
});
dialog.afterClosed().subscribe(result => {
this.router.navigate(['.'], { relativeTo: this.route })
})
} else {
this.openSnackBar(resp.data)
}
}).catch((err: HttpErrorResponse) => this.openSnackBar(err.message))
}
}

@ -1,84 +0,0 @@
import { OnInit, Directive } from '@angular/core';
import { HttpClient } from '@angular/common/http';
import { ApiTableService } from '../';
import { MatTableDataSource } from '@angular/material/table';
import { MatDialog } from '@angular/material/dialog';
import { ApiModel, Area } from '../models';
import { ActivatedRoute, Router } from '@angular/router';
import { interval } from 'rxjs';
import { MatSnackBar, MatSnackBarConfig } from '@angular/material/snack-bar';
@Directive()
export abstract class TablesComponent<T extends ApiModel> implements OnInit {
abstract area: Area;
data_source!: ApiTableService<T>;
table_data!: MatTableDataSource<T>;
isLoadingResults = true;
constructor(
public httpClient: HttpClient,
public infoDialog: MatDialog,
public route: ActivatedRoute,
public router: Router,
public snackBar: MatSnackBar
) {
this.table_data = new MatTableDataSource;
}
ngOnInit() {
this.data_source = new ApiTableService(this.httpClient, this.area);
this.loadTableData();
this.route.queryParams.subscribe(params => {
const id = params['id']
const new_agent = params['new']
if (id) {
this.show_item_dialog(id);
}
if (new_agent) {
this.show_item_dialog(null);
}
})
//interval(10000).subscribe(_ => this.loadTableData());
}
async loadTableData() {
this.isLoadingResults = true;
//possibly needs try/catch
const data = await this.data_source!.getMany();
this.isLoadingResults = false;
if (typeof data.data !== 'string') {
this.table_data.data = data.data
} else {
alert(`Error: ${data}`)
};
}
apply_filter(event: Event) {
const filterValue = (event.target as HTMLInputElement).value;
this.table_data.filter = filterValue.trim().toLowerCase();
}
deleteItem(id: string) {
if (confirm(`Delete ${id}?`)) {
this.data_source!.delete(id).catch(this.openSnackBar)
}
}
openSnackBar(message: any, error: boolean = true) {
const msg = JSON.stringify(message)
const _config = (duration: number): MatSnackBarConfig => {
return {
horizontalPosition: 'right',
verticalPosition: 'bottom',
duration
}
}
const cfg = error ? _config(0) : _config(2000)
this.snackBar.open(msg, 'Ok', cfg);
}
abstract displayedColumns: string[];
abstract show_item_dialog(id: string | null): void;
}

@ -1,6 +1,6 @@
import { UTCDate, ApiModel } from ".";
import { UTCDate } from ".";
export interface AgentModel extends ApiModel {
export interface AgentModel {
alias: string | null,
hostname: string,
host_info: string,

@ -0,0 +1,24 @@
import { AgentModel } from './agent.model';
import { JobModel } from './job.model';
import { PayloadModel } from './payload.model';
import { ResultModel } from './result.model';
export * from './agent.model';
export * from './result.model';
export * from './job.model';
export * from './payload.model';
export interface UTCDate {
secs_since_epoch: number,
nanos_since_epoch: number
}
export type Area = "agents" | "jobs" | "map" | "payloads";
export type ApiModel = AgentModel | JobModel | ResultModel | PayloadModel | Empty;
export interface Empty { }
export function getAreaByModel(_: AgentModel): Area {
return "agents"
}

@ -0,0 +1,16 @@
import { BriefOrFullPayloadModel } from './'
export interface JobModel {
alias: string | null,
argv: string,
id: string,
exec_type: string,
target_platforms: string,
payload: string | null,
schedule: string | null,
}
export interface BriefOrFullJobModel {
job: JobModel,
payload: BriefOrFullPayloadModel | null,
}

@ -0,0 +1,17 @@
export interface PayloadModel {
id: string,
mime_type: string,
name: string,
size: number,
}
export interface FullPayloadModel {
meta: PayloadModel,
data: number[]
}
export type BriefOrFullPayloadModel = PayloadModel | FullPayloadModel;
export function isFullPayload(payload: BriefOrFullPayloadModel): payload is FullPayloadModel {
return (payload as FullPayloadModel).data !== undefined
}

@ -1,6 +1,6 @@
import { UTCDate, ApiModel } from ".";
import { UTCDate } from ".";
export interface ResultModel extends ApiModel {
export interface ResultModel {
agent_id: string,
alias: string,
created: UTCDate,

@ -0,0 +1,138 @@
import { environment } from 'src/environments/environment';
import { HttpClient, HttpErrorResponse } from '@angular/common/http';
import { Observable, map, catchError, throwError } from 'rxjs';
import { ApiModel, getAreaByModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, BriefOrFullJobModel } from '../models';
import { Injectable, Inject } from '@angular/core';
import { ErrorService } from './error.service';
type Status = "ok" | "err";
interface ServerResponse<T extends ApiModel> {
status: Status,
data: T | string
}
@Injectable({
providedIn: 'root'
})
export class ApiTableService {
constructor(
private http: HttpClient,
private errorService: ErrorService
) {
}
requestUrl = `${environment.server}/cmd/`;
req<R extends ApiModel>(cmd: string): Observable<ServerResponse<R>> {
return this.http.post<ServerResponse<R>>(this.requestUrl, cmd).pipe(
catchError(this.errorHandler)
)
}
getOne<T extends ApiModel>(id: string, area: Area, brief: 'yes' | 'no' | 'auto' | null = null): Observable<T> {
const request = `${area} read ${id}` + (brief !== null ? `-b=${brief}` : '')
const resp = this.req<T[]>(request).pipe(
map(resp => {
if (resp.data.length === 0) {
return {
status: 'err' as Status,
data: `${id} not found in ${area}`
}
}
return {
status: resp.status,
data: resp.data[0]
}
}));
return this.filterErrStatus(resp)
}
getAgent(id: string): Observable<AgentModel> {
return this.getOne(id, 'agents')
}
getJob(id: string): Observable<BriefOrFullJobModel> {
return this.getOne(id, 'jobs')
}
getResult(id: string): Observable<ResultModel> {
return this.getOne(id, 'map')
}
getPayload(id: string): Observable<PayloadModel> {
return this.getOne(id, 'payloads')
}
getMany(area: Area): Observable<any[]> {
return this.filterErrStatus(this.req(`${area} read`))
}
getAgents(): Observable<AgentModel[]> {
return this.getMany('agents')
}
getJobs(): Observable<AgentModel[]> {
return this.getMany('jobs')
}
getResults(): Observable<AgentModel[]> {
return this.getMany('map')
}
getPayloads(): Observable<PayloadModel[]> {
return this.getMany('payloads')
}
update<T extends ApiModel>(item: T, area: Area): Observable<Empty> {
return this.filterErrStatus(this.req(`${area} update '${JSON.stringify(item)}'`))
}
updateAgent(item: AgentModel): Observable<Empty> {
return this.update(item, 'agents')
}
updateJob(item: JobModel): Observable<Empty> {
return this.update(item, 'jobs')
}
updateResult(item: ResultModel): Observable<Empty> {
return this.update(item, 'map')
}
updatePayload(item: PayloadModel): Observable<Empty> {
return this.update(item, 'payloads')
}
delete(id: string, area: Area): Observable<Empty> {
return this.filterErrStatus(this.req(`${area} delete ${id}`))
}
create(item: string | null, area: Area): Observable<string[]> {
if (!item) {
item = '"{}"'
}
return this.filterErrStatus(this.req(`${area} create ${item}`))
}
createResult(item: string): Observable<string[]> {
return this.create(item, 'map')
}
filterErrStatus<R extends ApiModel>(obs: Observable<ServerResponse<R>>): Observable<R> {
return obs.pipe(
map(r => {
if (r.status == 'err') {
throw new Error(r.data as string)
}
return r.data as R
}),
catchError(this.errorHandler.bind(this)))
}
errorHandler<R>(err: HttpErrorResponse, _: R) {
this.errorService.handle(err.message);
return throwError(() => new Error(err.message));
}
}

@ -0,0 +1,17 @@
import { Injectable } from '@angular/core';
import { Subject } from 'rxjs';
@Injectable({
providedIn: 'root'
})
export class ErrorService {
error$ = new Subject<string>();
handle(msg: string) {
this.error$.next(msg)
}
clear() {
this.handle('')
}
}

@ -13,13 +13,13 @@ use u_lib::logging::init_logger;
#[actix_web::main]
async fn main() -> AnyResult<()> {
init_logger(None);
let env = AccessEnv::load()?;
let client = HttpClient::new(&env.u_server, Some(env.admin_auth_token)).await?;
let args = Args::from_args();
init_logger(None::<&str>);
let result = process_cmd(client, args).await.to_string();
println!("{result}");
Ok(())
}

@ -20,6 +20,7 @@ tokio = { workspace = true, features = ["macros"] }
uuid = { workspace = true, features = ["serde", "v4"] }
u_lib = { path = "../../lib/u_lib", features = ["server"] }
warp = { version = "0.3.1", features = ["tls"] }
serde_qs = { version = "0.12.0", features = ["warp"] }
[dev-dependencies]
rstest = "0.12"

@ -3,7 +3,7 @@ use diesel::{pg::PgConnection, prelude::*, result::Error as DslError, Connection
use std::mem::drop;
use u_lib::{
db::PgAsyncPool,
models::{schema, Agent, AssignedJob, JobModel, JobState, PayloadMeta, ThinJob},
models::{schema, Agent, AssignedJob, BriefJob, JobModel, JobState, PayloadMeta},
platform::Platform,
types::Id,
};
@ -50,7 +50,7 @@ pub struct UDB<'c> {
}
impl UDB<'_> {
pub fn insert_jobs(&mut self, jobs: &[ThinJob]) -> Result<()> {
pub fn insert_jobs(&mut self, jobs: &[BriefJob]) -> Result<()> {
use schema::{jobs, payloads};
let (jobs, payloads_opt): (Vec<_>, Vec<_>) = jobs
@ -76,7 +76,7 @@ impl UDB<'_> {
.map_err(with_err_ctx("Can't insert jobs"))
}
pub fn get_job(&mut self, id: Id) -> Result<Option<ThinJob>> {
pub fn get_job(&mut self, id: Id) -> Result<Option<BriefJob>> {
use schema::{jobs, payloads};
let maybe_job_with_payload = jobs::table
@ -86,7 +86,7 @@ impl UDB<'_> {
.optional()
.map_err(with_err_ctx(format!("Can't get job {id}")))?;
Ok(maybe_job_with_payload.map(|(job, payload_meta)| ThinJob { job, payload_meta }))
Ok(maybe_job_with_payload.map(|(job, payload_meta)| BriefJob { job, payload_meta }))
}
pub fn get_jobs(&mut self) -> Result<Vec<JobModel>> {
@ -97,7 +97,25 @@ impl UDB<'_> {
.map_err(with_err_ctx("Can't get jobs"))
}
pub fn find_job_by_alias(&mut self, alias: &str) -> Result<Option<ThinJob>> {
pub fn get_payload_meta(&mut self, id: Id) -> Result<Option<PayloadMeta>> {
use schema::payloads;
payloads::table
.filter(payloads::id.eq(id))
.first(self.conn)
.optional()
.map_err(with_err_ctx(format!("Can't get payload {id}")))
}
pub fn get_payload_metas(&mut self) -> Result<Vec<PayloadMeta>> {
use schema::payloads;
payloads::table
.load(self.conn)
.map_err(with_err_ctx("Can't get payloads"))
}
pub fn find_job_by_alias(&mut self, alias: &str) -> Result<Option<BriefJob>> {
use schema::{jobs, payloads};
let maybe_job_with_payload = jobs::table
@ -107,7 +125,7 @@ impl UDB<'_> {
.optional()
.map_err(with_err_ctx(format!("Can't get job by alias {alias}")))?;
Ok(maybe_job_with_payload.map(|(job, payload_meta)| ThinJob { job, payload_meta }))
Ok(maybe_job_with_payload.map(|(job, payload_meta)| BriefJob { job, payload_meta }))
}
pub fn insert_result(&mut self, result: &AssignedJob) -> Result<()> {
@ -150,7 +168,11 @@ impl UDB<'_> {
}
//TODO: filters possibly could work in a wrong way, check
pub fn get_exact_jobs(&mut self, id: Option<Id>, personal: bool) -> Result<Vec<AssignedJob>> {
pub fn get_assigned_jobs(
&mut self,
id: Option<Id>,
personal: bool,
) -> Result<Vec<AssignedJob>> {
use schema::results;
let mut q = results::table.into_boxed();

@ -56,7 +56,7 @@ impl RejResponse {
pub fn internal() -> Self {
Self {
message: "INTERNAL_SERVER_ERROR".to_string(),
message: "INTERNAL SERVER ERROR".to_string(),
status: StatusCode::INTERNAL_SERVER_ERROR,
}
}

@ -3,22 +3,24 @@ use std::sync::Arc;
use crate::db::{PgRepo, UDB};
use crate::error::Error;
use serde::Deserialize;
use u_lib::ufs;
use u_lib::{
api::retypes,
jobs::{join_payload, split_payload},
messaging::{AsMsg, Reportable},
misc::OneOrVec,
messaging::Reportable,
models::*,
types::Id,
};
use warp::reject::not_found;
use warp::Rejection;
const MAX_READABLE_PAYLOAD_SIZE: i64 = 1024 * 32;
type EndpResult<T> = Result<T, Rejection>;
#[derive(Deserialize)]
pub struct GetJobQuery {
force_payload: bool,
pub struct PayloadFlags {
brief: BriefMode,
}
pub struct Endpoints;
@ -44,24 +46,32 @@ impl Endpoints {
pub async fn get_job(
repo: Arc<PgRepo>,
id: Id,
params: GetJobQuery,
params: Option<PayloadFlags>,
) -> EndpResult<retypes::GetJob> {
let Some(job) = repo.interact(move |mut db| db.get_job(id)).await? else {
return Err(not_found())
};
let make_full_job = |j| -> Result<BriefOrFullJob, Error> {
let full_job = join_payload(j).map_err(Error::from)?;
Ok(BriefOrFullJob::Full(full_job))
};
if let Some(meta) = &job.payload_meta {
let max_readable_payload_size = 1024 * 8;
if !params.force_payload && meta.size > max_readable_payload_size {
return Ok(FatJob {
job: job.job,
payload_meta: job.payload_meta,
payload_data: None,
});
Ok(match params.map(|p| p.brief) {
Some(BriefMode::Yes) => BriefOrFullJob::Brief(job),
Some(BriefMode::Auto) | None => {
if job
.payload_meta
.as_ref()
.map(|m| m.size > MAX_READABLE_PAYLOAD_SIZE)
.unwrap_or(false)
{
BriefOrFullJob::Brief(job)
} else {
make_full_job(job)?
}
}
}
Ok(join_payload(job).map_err(Error::from)?)
Some(BriefMode::No) => make_full_job(job)?,
})
}
pub async fn get_jobs(repo: Arc<PgRepo>) -> EndpResult<retypes::GetJobs> {
@ -70,15 +80,48 @@ impl Endpoints {
.map_err(From::from)
}
pub async fn get_agent_jobs(
pub async fn get_assigned_jobs(
repo: Arc<PgRepo>,
id: Option<Id>,
) -> EndpResult<retypes::GetAgentJobs> {
repo.interact(move |mut db| db.get_exact_jobs(id, false))
repo.interact(move |mut db| db.get_assigned_jobs(id, false))
.await
.map_err(From::from)
}
pub async fn get_payloads(repo: Arc<PgRepo>) -> EndpResult<retypes::GetPayloads> {
repo.interact(move |mut db| db.get_payload_metas())
.await
.map_err(From::from)
}
pub async fn get_payload(
repo: Arc<PgRepo>,
id: Id,
params: Option<PayloadFlags>,
) -> EndpResult<retypes::GetPayload> {
let Some(meta) = repo.interact(move |mut db| db.get_payload_meta(id)).await? else {
return Err(not_found())
};
Ok(match params.map(|p| p.brief) {
Some(BriefMode::Yes) => BriefOrFullPayload::Brief(meta),
None | Some(BriefMode::Auto) if meta.size > MAX_READABLE_PAYLOAD_SIZE => {
BriefOrFullPayload::Brief(meta)
}
_ => {
let payload_data = ufs::read(&meta.name).map_err(|e| {
error!("payload reading failed: {}", e);
Error::from(e.downcast::<Error>().expect("wrong error type"))
})?;
BriefOrFullPayload::Full(FullPayload {
meta,
data: payload_data,
})
}
})
}
pub async fn get_personal_jobs(
repo: Arc<PgRepo>,
id: Id,
@ -104,7 +147,7 @@ impl Endpoints {
}
}
let assigned_jobs = db.get_exact_jobs(Some(id), true)?;
let assigned_jobs = db.get_assigned_jobs(Some(id), true)?;
for job in &assigned_jobs {
db.update_job_status(job.id, JobState::Running)?;
@ -121,12 +164,15 @@ impl Endpoints {
pub async fn upload_jobs(
repo: Arc<PgRepo>,
msg: Vec<FatJob>,
msg: Vec<BriefOrFullJob>,
) -> EndpResult<retypes::UploadJobs> {
let jobs = msg
.into_iter()
.map(|meta| Ok(split_payload(meta)?))
.collect::<Result<Vec<ThinJob>, Error>>()?;
.map(|meta| match meta {
BriefOrFull::Full(job) => Ok(split_payload(job)?),
BriefOrFull::Brief(job) => Ok(job),
})
.collect::<Result<Vec<BriefJob>, Error>>()?;
repo.interact(move |mut db| db.insert_jobs(&jobs))
.await
@ -179,13 +225,13 @@ impl Endpoints {
.map_err(From::from)
}
pub async fn report<Data: OneOrVec<Reportable> + AsMsg + Send + Sync + 'static>(
pub async fn report(
repo: Arc<PgRepo>,
msg: Data,
msg: Vec<Reportable>,
agent_id: Id,
) -> EndpResult<retypes::Report> {
repo.transaction(move |mut db| {
for entry in msg.into_vec() {
for entry in msg {
match entry {
Reportable::Assigned(mut result) => {
let result_agent_id = &result.agent_id;
@ -240,8 +286,14 @@ impl Endpoints {
Ok(())
}
pub async fn update_job(repo: Arc<PgRepo>, job: FatJob) -> EndpResult<retypes::UpdateJob> {
let thin_job = split_payload(job).map_err(Error::from)?;
pub async fn update_job(
repo: Arc<PgRepo>,
job: BriefOrFullJob,
) -> EndpResult<retypes::UpdateJob> {
let thin_job = match job {
BriefOrFullJob::Full(job) => split_payload(job).map_err(Error::from)?,
BriefOrFullJob::Brief(job) => job,
};
repo.interact(move |mut db| db.update_job(&thin_job.job))
.await?;
@ -256,8 +308,4 @@ impl Endpoints {
.await?;
Ok(())
}
pub async fn download(_file_id: String) -> EndpResult<Vec<u8>> {
todo!()
}
}

@ -1,10 +1,6 @@
#[macro_use]
extern crate tracing;
#[cfg(test)]
#[macro_use]
extern crate rstest;
mod db;
mod error;
mod handlers;
@ -22,11 +18,13 @@ use u_lib::{
use warp::{
body,
log::{custom, Info},
reply::{json, reply, Json, Response},
reply::{json, Json, Response},
Filter, Rejection, Reply,
};
use crate::handlers::{Endpoints, GetJobQuery};
const DEFAULT_RESP: &str = "null";
use crate::handlers::{Endpoints, PayloadFlags};
fn into_message<M: AsMsg>(msg: M) -> Json {
json(&msg)
@ -36,8 +34,15 @@ pub fn init_endpoints(
auth_token: &str,
db: PgRepo,
) -> impl Filter<Extract = (impl Reply,), Error = Rejection> + Clone {
fn make_optional<T>(
f: impl Filter<Extract = (T,), Error = Rejection> + Clone,
) -> impl Filter<Extract = (Option<T>,), Error = Infallible> + Clone {
f.map(Some)
.or_else(|_| async { Ok::<(Option<T>,), Infallible>((None,)) })
}
let path = |p: &'static str| warp::post().and(warp::path(p));
let infallible_none = |_| async { Result::<(Option<Id>,), Infallible>::Ok((None,)) };
let create_qs_cfg = || serde_qs::Config::new(1, true);
let with_db = {
let adb = Arc::new(db);
@ -46,20 +51,22 @@ pub fn init_endpoints(
let get_agents = path("get_agents")
.and(with_db.clone())
.and(warp::path::param::<Id>().map(Some).or_else(infallible_none))
.and(make_optional(warp::path::param::<Id>()))
.and_then(Endpoints::get_agents)
.map(into_message);
let upload_jobs = path("upload_jobs")
.and(with_db.clone())
.and(body::json::<Vec<FatJob>>())
.and(body::json::<Vec<BriefOrFullJob>>())
.and_then(Endpoints::upload_jobs)
.map(into_message);
let get_job = path("get_job")
.and(with_db.clone())
.and(warp::path::param::<Id>())
.and(warp::query::<GetJobQuery>())
.and(make_optional(serde_qs::warp::query::<PayloadFlags>(
create_qs_cfg(),
)))
.and_then(Endpoints::get_job)
.map(into_message);
@ -68,10 +75,10 @@ pub fn init_endpoints(
.and_then(Endpoints::get_jobs)
.map(into_message);
let get_agent_jobs = path("get_agent_jobs")
let get_assigned_jobs = path("get_assigned_jobs")
.and(with_db.clone())
.and(warp::path::param::<Id>().map(Some).or_else(infallible_none))
.and_then(Endpoints::get_agent_jobs)
.and(make_optional(warp::path::param::<Id>()))
.and_then(Endpoints::get_assigned_jobs)
.map(into_message);
let get_personal_jobs = path("get_personal_jobs")
@ -108,7 +115,7 @@ pub fn init_endpoints(
let update_job = path("update_job")
.and(with_db.clone())
.and(body::json::<FatJob>())
.and(body::json::<BriefOrFullJob>())
.and_then(Endpoints::update_job)
.map(ok);
@ -118,12 +125,21 @@ pub fn init_endpoints(
.and_then(Endpoints::update_assigned_job)
.map(ok);
let download = path("download")
.and(warp::path::param::<String>())
.and_then(Endpoints::download)
.map(ok);
let get_payloads = path("get_payloads")
.and(with_db.clone())
.and_then(Endpoints::get_payloads)
.map(into_message);
let get_payload = path("get_payload")
.and(with_db.clone())
.and(warp::path::param::<Id>())
.and(make_optional(serde_qs::warp::query::<PayloadFlags>(
create_qs_cfg(),
)))
.and_then(Endpoints::get_payload)
.map(into_message);
let ping = path("ping").map(reply);
let ping = path("ping").map(|| DEFAULT_RESP);
let auth_token = format!("Bearer {auth_token}",).into_boxed_str();
let auth_header = warp::header::exact("authorization", Box::leak(auth_token));
@ -134,17 +150,14 @@ pub fn init_endpoints(
.or(upload_jobs)
.or(del)
.or(set_jobs)
.or(get_agent_jobs)
.or(get_assigned_jobs)
.or(update_agent.or(update_job).or(update_assigned_job))
.or(download)
.or(get_payloads)
.or(get_payload)
.or(ping))
.and(auth_header);
let agent_zone = get_job
.or(get_jobs)
.or(get_personal_jobs)
.or(report)
.or(download);
let agent_zone = get_job.or(get_jobs).or(get_personal_jobs).or(report);
auth_zone.or(agent_zone)
}
@ -154,7 +167,7 @@ pub async fn preload_jobs(repo: &PgRepo) -> Result<(), ServerError> {
let job_alias = "agent_hello";
let if_job_exists = db.find_job_by_alias(job_alias)?;
if if_job_exists.is_none() {
let agent_hello = RawJob::builder()
let agent_hello = RawJob::brief_job_builder()
.with_type(JobType::Init)
.with_alias(job_alias)
.build()
@ -221,7 +234,7 @@ fn logger(info: Info<'_>) {
}
fn ok<T>(_: T) -> impl Reply {
"null"
DEFAULT_RESP
}
/*

@ -1,5 +1,5 @@
[package]
name = "integration"
name = "integration-tests"
version = "0.1.0"
authors = ["plazmoid <kronos44@mail.ru>"]
edition = "2021"
@ -7,6 +7,7 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
ctor = "0.2.0"
once_cell = "1.10.0"
reqwest = { workspace = true }
rstest = "0.17"
@ -20,5 +21,5 @@ u_lib = { path = "../lib/u_lib", features = ["panel", "server"] }
[[test]]
name = "integration"
name = "integration-tests"
path = "tests/lib.rs"

@ -30,10 +30,10 @@ services:
environment:
RUST_LOG: warp=info,u_server_lib=debug
healthcheck:
test: ss -tlpn | grep 63714
interval: 5s
timeout: 2s
retries: 2
test: ss -tlpn | grep 63714
interval: 5s
timeout: 2s
retries: 2
u_db:
image: localhost/unki/u_db
@ -51,11 +51,11 @@ services:
target: /u_db_entrypoint.sh
command: /u_db_entrypoint.sh
healthcheck:
# test if db's port is open and db is created
test: ss -tlpn | grep 5432 && psql -lqt -U $${POSTGRES_USER} | grep -qw $${POSTGRES_DATABASE}
interval: 5s
timeout: 5s
retries: 3
# test if db's port is open and db is created
test: ss -tlpn | grep 5432 && psql -lqt -U $${POSTGRES_USER} | grep -qw $${POSTGRES_DATABASE}
interval: 5s
timeout: 5s
retries: 3
u_agent:
user: *user
@ -83,13 +83,13 @@ services:
volumes:
- ${HOME}/.cargo/registry/:/usr/local/cargo/registry/
- ../__Cargo_integration.toml:/tests/Cargo.toml
- ./:/tests/integration/
- ./:/tests/integration-tests/
- ../certs:/tests/certs
- ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_panel:/u_panel
- ../lib/u_lib:/tests/lib/u_lib
- ../logs:/tests/integration/logs:rw
- ../logs:/tests/integration-tests/logs:rw
working_dir:
/tests/integration/
/tests/integration-tests/
depends_on:
u_agent:
condition: service_started
@ -100,4 +100,5 @@ services:
- ../.env.private
environment:
RUST_BACKTRACE: 1
RUST_LOG: debug,hyper=info,reqwest=info
U_SERVER: u_server

@ -90,7 +90,7 @@ class Compose:
]
def __init__(self):
self.container_tpl = 'integration-%s-%d'
self.container_tpl = 'integration-tests-%s-%d'
self.cmd_container = self.container_tpl % ('tests_runner', 1)
self.ALL_CONTAINERS = [self.container_tpl %
(c, 1) for c in self.ALL_IMAGES]

@ -6,7 +6,7 @@ from docker import rebuild_images_if_needed, Compose
from pathlib import Path
from utils import *
CARGO_INTEGRATION_TOML = Path('../__Cargo_integration.toml')
CARGO_INTEGRATION_TESTS_TOML = Path('../__Cargo_integration.toml')
CLUSTER = Compose()
@ -22,12 +22,12 @@ def usage_exit():
fail(usage)
def create_integration_workspace():
if CARGO_INTEGRATION_TOML.exists():
CARGO_INTEGRATION_TOML.unlink()
def create_integration_tests_workspace():
if CARGO_INTEGRATION_TESTS_TOML.exists():
CARGO_INTEGRATION_TESTS_TOML.unlink()
workspace = toml.load('../Cargo.toml')
workspace['workspace']['members'] = ['integration']
with open(CARGO_INTEGRATION_TOML, 'w') as fo:
workspace['workspace']['members'] = ['integration-tests']
with open(CARGO_INTEGRATION_TESTS_TOML, 'w') as fo:
toml.dump(workspace, fo)
@ -44,7 +44,7 @@ def run_tests():
def _cleanup():
if not preserve_containers and not only_setup_cluster:
CLUSTER.down()
CARGO_INTEGRATION_TOML.unlink(missing_ok=True)
CARGO_INTEGRATION_TESTS_TOML.unlink(missing_ok=True)
def abort_handler(s, _):
warn(f'Received signal: {s}, gracefully stopping...')
@ -57,12 +57,12 @@ def run_tests():
for s in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP):
signal.signal(s, abort_handler)
rebuild_images_if_needed(force_rebuild)
create_integration_workspace()
create_integration_tests_workspace()
try:
CLUSTER.up()
CLUSTER.is_alive()
if not only_setup_cluster:
CLUSTER.run('cargo test --test integration')
CLUSTER.run('cargo test --test integration-tests')
except Exception as e:
#CLUSTER.print_containers_logs()
fail(e)

@ -3,6 +3,6 @@ set -e
export DOCKER_UID=$(id -u)
export DOCKER_GID=$(id -g)
rm ../logs/u_agent*
rm ../logs/u_agent* || true
[[ "$@" =~ "--release" ]] && export PROFILE=release || export PROFILE=debug
python integration_tests.py $@

@ -1,5 +1,6 @@
use super::connections::*;
use super::run_async;
use u_lib::unwrap_enum;
use u_lib::{api::HttpClient, jobs::split_payload, messaging::Reportable, models::*, types::Id};
pub struct RegisteredAgent {
@ -13,6 +14,7 @@ pub fn registered_agent(client: &HttpClient) -> RegisteredAgent {
let agent = Agent::with_current_platform();
let agent_id = agent.id;
println!("registering agent {agent_id}");
debug!("registering agent1 {agent_id}");
let resp = client
.get_personal_jobs(agent_id)
.await
@ -20,12 +22,13 @@ pub fn registered_agent(client: &HttpClient) -> RegisteredAgent {
.pop()
.unwrap();
let job_id = resp.job_id;
let job = client.get_job(job_id, true).await.unwrap();
let job = client.get_job(job_id, BriefMode::No).await.unwrap();
let job = unwrap_enum!(job, BriefOrFull::Full);
assert_eq!(job.job.alias, Some("agent_hello".to_string()));
let mut agent_data = AssignedJob::from((&split_payload(job).unwrap().job, resp));
agent_data.set_result(&agent);
client
.report(&Reportable::Assigned(agent_data))
.report([Reportable::Assigned(agent_data)])
.await
.unwrap();
RegisteredAgent { id: agent_id }

@ -10,13 +10,25 @@ pub struct Panel;
impl Panel {
fn run(args: &[&str]) -> Output {
Command::new(PANEL_BINARY).args(args).output().unwrap()
Command::new(PANEL_BINARY)
.env("RUST_LOG", "u_lib=debug")
.args(args)
.output()
.unwrap()
}
pub fn output_argv<T: DeserializeOwned>(argv: &[&str]) -> PanelResult<T> {
let result = Self::run(argv);
let output = ProcOutput::from_output(&result);
let stderr = output.get_stderr();
if !stderr.is_empty() {
println!(
"\n*** PANEL DEBUG OUTPUT START***\n{}\n*** PANEL DEBUG OUTPUT END ***\n",
String::from_utf8_lossy(stderr)
);
}
match from_slice(output.get_stdout()) {
Ok(r) => r,
Err(e) => {

@ -14,35 +14,39 @@
// ping(&self)
use crate::fixtures::connections::*;
use u_lib::jobs::join_payload;
use u_lib::models::RawJob;
use u_lib::models::{BriefOrFullJob, RawJob};
#[rstest]
#[tokio::test]
async fn test_jobs_endpoints(client_panel: &HttpClient) {
let job_alias = "henlo";
let job = RawJob::builder()
let mut job = RawJob::brief_job_builder()
.with_shell("echo henlo")
.with_alias(job_alias)
.build()
.unwrap();
let job_id = job.job.id;
let mut fat_job = join_payload(job).unwrap();
client_panel.upload_jobs(&fat_job).await.unwrap();
client_panel
.upload_jobs([&BriefOrFullJob::Brief(job.clone())])
.await
.unwrap();
let fetched_job = client_panel.get_job(job_id, false).await.unwrap();
assert_eq!(fat_job, fetched_job);
let fetched_job = client_panel.get_brief_job(job_id).await.unwrap();
assert_eq!(job, fetched_job);
fat_job.job.alias = Some("henlo2".to_string());
client_panel.update_job(&fat_job).await.unwrap();
job.job.alias = Some("henlo2".to_string());
client_panel
.update_job(&BriefOrFullJob::Brief(job.clone()))
.await
.unwrap();
let fetched_job = client_panel.get_job(job_id, false).await.unwrap();
assert_eq!(fat_job, fetched_job);
let fetched_job = client_panel.get_brief_job(job_id).await.unwrap();
assert_eq!(job, fetched_job);
client_panel.del(job_id).await.unwrap();
let not_found_err = client_panel.get_job(job_id, false).await.unwrap_err();
let not_found_err = client_panel.get_brief_job(job_id).await.unwrap_err();
assert!(not_found_err.to_string().contains("404 Not Found"))
}

@ -21,7 +21,7 @@ async fn setup_tasks() {
let agents: Vec<Agent> = Panel::check_output("agents read");
let agent_id = agents[0].id;
let job_alias = "passwd_contents";
let job = RawJob::builder()
let job = RawJob::brief_job_builder()
.with_alias(job_alias)
.with_raw_payload(b"cat /etc/passwd".as_slice())
.with_shell("/bin/bash {}")
@ -54,7 +54,7 @@ async fn large_payload() {
let agent = &Panel::check_output::<Vec<Agent>>("agents read")[0];
let agent_id = agent.id;
let job_alias = "large_payload";
let job = RawJob::builder()
let job = RawJob::brief_job_builder()
.with_alias(job_alias)
.with_payload_path("./tests/bin/echoer")
.with_shell("{} type echo")

@ -0,0 +1,14 @@
mod fixtures;
mod helpers;
mod integration_tests;
#[macro_use]
extern crate rstest;
#[macro_use]
extern crate tracing;
#[ctor::ctor]
fn __init() {
u_lib::logging::init_logger(None);
}

@ -1,6 +0,0 @@
mod fixtures;
mod helpers;
mod integration;
#[macro_use]
extern crate rstest;

@ -7,11 +7,11 @@ use reqwest::{header, header::HeaderMap, Certificate, Client, Identity, Method,
use serde::de::DeserializeOwned;
use serde_json::{from_str, Value};
use crate::unwrap_enum;
use crate::{
config::{get_self_id, MASTER_PORT},
conv::opt_to_string,
messaging::{self, AsMsg},
misc::OneOrVecRef,
models::*,
types::Id,
UError, UResult,
@ -25,7 +25,9 @@ pub mod retypes {
pub type GetPersonalJobs = Vec<AssignedJobById>;
pub type Report = ();
pub type GetJob = FatJob;
pub type GetJob = BriefOrFullJob;
pub type GetFullJob = FullJob;
pub type GetBriefJob = BriefJob;
pub type GetJobs = Vec<JobModel>;
pub type GetAgents = Vec<Agent>;
pub type UpdateAgent = ();
@ -36,6 +38,8 @@ pub mod retypes {
pub type SetJobs = Vec<Id>;
pub type GetAgentJobs = Vec<AssignedJob>;
pub type Ping = ();
pub type GetPayloads = Vec<PayloadMeta>;
pub type GetPayload = BriefOrFullPayload;
}
#[derive(Clone, Debug)]
@ -109,6 +113,8 @@ impl HttpClient {
.post(self.base_url.join(url).unwrap())
.json(payload);
debug!("url = {url}");
let response = request
.send()
.await
@ -130,34 +136,47 @@ impl HttpClient {
}
.map_err(From::from);
debug!("url = {}, resp = {:?}", url, result);
debug!("response = {:?}", result);
result
}
// get jobs for client
pub async fn get_personal_jobs(&self, agent_id: Id) -> Result<Vec<AssignedJobById>> {
/// get jobs for agent
pub async fn get_personal_jobs(&self, agent_id: Id) -> Result<retypes::GetPersonalJobs> {
self.req(format!("get_personal_jobs/{}", agent_id)).await
}
// send something to server
pub async fn report(&self, payload: impl OneOrVecRef<messaging::Reportable>) -> Result<()> {
self.req_with_payload("report", &payload.as_vec()).await
/// send something to server
pub async fn report(
&self,
payload: impl IntoIterator<Item = messaging::Reportable>,
) -> Result<retypes::Report> {
self.req_with_payload("report", &payload.into_iter().collect::<Vec<_>>())
.await
}
// download payload
pub async fn dl(&self, file: &str) -> Result<Vec<u8>> {
/// download payload
pub async fn _dl(&self, file: &str) -> Result<Vec<u8>> {
self.req(format!("dl/{file}")).await
}
/// get exact job
pub async fn get_job(&self, job: Id, force_payload: bool) -> Result<FatJob> {
self.req(format!("get_job/{job}?force_payload={force_payload}"))
.await
pub async fn get_job(&self, job: Id, brief: BriefMode) -> Result<retypes::GetJob> {
self.req(format!("get_job/{job}?brief={brief}")).await
}
pub async fn get_full_job(&self, job: Id) -> Result<retypes::GetFullJob> {
let job = self.get_job(job, BriefMode::No).await?;
Ok(unwrap_enum!(job, BriefOrFullJob::Full))
}
pub async fn get_brief_job(&self, job: Id) -> Result<retypes::GetBriefJob> {
let job = self.get_job(job, BriefMode::Yes).await?;
Ok(unwrap_enum!(job, BriefOrFullJob::Brief))
}
/// get all available jobs
pub async fn get_jobs(&self) -> Result<Vec<JobModel>> {
pub async fn get_jobs(&self) -> Result<retypes::GetJobs> {
self.req("get_jobs").await
}
}
@ -166,34 +185,37 @@ impl HttpClient {
#[cfg(feature = "panel")]
impl HttpClient {
/// agent listing
pub async fn get_agents(&self, agent: Option<Id>) -> Result<Vec<Agent>> {
pub async fn get_agents(&self, agent: Option<Id>) -> Result<retypes::GetAgents> {
self.req(format!("get_agents/{}", opt_to_string(agent)))
.await
}
/// update agent
pub async fn update_agent(&self, agent: &Agent) -> Result<()> {
pub async fn update_agent(&self, agent: &Agent) -> Result<retypes::UpdateAgent> {
self.req_with_payload("update_agent", agent).await
}
/// update job
pub async fn update_job(&self, job: &FatJob) -> Result<()> {
pub async fn update_job(&self, job: &BriefOrFullJob) -> Result<retypes::UpdateJob> {
self.req_with_payload("update_job", job).await
}
/// update result
pub async fn update_result(&self, result: &AssignedJob) -> Result<()> {
pub async fn update_result(&self, result: &AssignedJob) -> Result<retypes::UpdateResult> {
self.req_with_payload("update_result", result).await
}
/// create and upload job
pub async fn upload_jobs(&self, payload: impl OneOrVecRef<FatJob>) -> Result<()> {
self.req_with_payload("upload_jobs", &payload.as_vec())
pub async fn upload_jobs(
&self,
payload: impl IntoIterator<Item = &BriefOrFullJob>,
) -> Result<retypes::UploadJobs> {
self.req_with_payload("upload_jobs", &payload.into_iter().collect::<Vec<_>>())
.await
}
/// delete something
pub async fn del(&self, item: Id) -> Result<()> {
pub async fn del(&self, item: Id) -> Result<retypes::Del> {
self.req(format!("del/{item}")).await
}
@ -201,19 +223,34 @@ impl HttpClient {
pub async fn set_jobs(
&self,
agent: Id,
job_idents: impl OneOrVecRef<String>,
) -> Result<Vec<Id>> {
self.req_with_payload(format!("set_jobs/{agent}"), &job_idents.as_vec())
.await
job_idents: impl IntoIterator<Item = impl Into<String>>,
) -> Result<retypes::SetJobs> {
self.req_with_payload(
format!("set_jobs/{agent}"),
&job_idents
.into_iter()
.map(|i| i.into())
.collect::<Vec<String>>(),
)
.await
}
/// get jobs for any agent
pub async fn get_agent_jobs(&self, agent: Option<Id>) -> Result<Vec<AssignedJob>> {
self.req(format!("get_agent_jobs/{}", opt_to_string(agent)))
pub async fn get_assigned_jobs(&self, agent: Option<Id>) -> Result<retypes::GetAgentJobs> {
self.req(format!("get_assigned_jobs/{}", opt_to_string(agent)))
.await
}
pub async fn get_payloads(&self) -> Result<retypes::GetPayloads> {
self.req("get_payloads").await
}
pub async fn get_payload(&self, payload: Id, brief: BriefMode) -> Result<retypes::GetPayload> {
self.req(format!("get_payload/{payload}?brief={brief}"))
.await
}
pub async fn ping(&self) -> Result<()> {
pub async fn ping(&self) -> Result<retypes::Ping> {
self.req("ping").await
}
}

@ -1,10 +1,10 @@
use crate::models::ThinJob;
use crate::models::BriefJob;
use crate::types::Id;
use lazy_static::lazy_static;
use parking_lot::{RwLock, RwLockReadGuard};
use std::{collections::HashMap, ops::Deref};
type Val = ThinJob;
type Val = BriefJob;
type Cache = HashMap<Id, Val>;
lazy_static! {

@ -1,6 +1,5 @@
use std::fmt::Debug;
use crate::misc::OneOrVec;
use anyhow::Error;
pub struct CombinedResult<T, E: Debug = Error> {
@ -16,17 +15,12 @@ impl<T, E: Debug> CombinedResult<T, E> {
}
}
pub fn ok(&mut self, result: impl OneOrVec<T>) {
self.ok.extend(result.into_vec());
pub fn push_ok(&mut self, result: T) {
self.ok.push(result);
}
pub fn err<I: Into<E>>(&mut self, err: impl OneOrVec<I>) {
self.err.extend(
err.into_vec()
.into_iter()
.map(Into::into)
.collect::<Vec<_>>(),
);
pub fn push_err(&mut self, err: impl Into<E>) {
self.err.push(err.into());
}
pub fn unwrap(self) -> Vec<T> {

@ -3,8 +3,8 @@ mod chan;
pub use chan::*;
use crate::ufs;
use reqwest::Error as ReqError;
use serde::{Deserialize, Serialize};
use std::io;
use thiserror::Error;
use uuid::Uuid;
@ -33,6 +33,9 @@ pub enum UError {
#[error(transparent)]
FSError(#[from] ufs::Error),
#[error("I/O error: {0}")]
IOError(String),
#[error("Wrong auth token")]
WrongToken,
@ -45,12 +48,12 @@ pub enum UError {
#[error("Deserialize from json error: {0}, body: {1}")]
DeserializeError(String, String),
#[error("{0}\n{1}")]
#[error("{0}\nContext: {1}")]
Contexted(Box<UError>, String),
}
impl From<ReqError> for UError {
fn from(e: ReqError) -> Self {
impl From<reqwest::Error> for UError {
fn from(e: reqwest::Error) -> Self {
UError::NetError(
e.to_string(),
e.url().map(|u| u.to_string()).unwrap_or_default(),
@ -59,6 +62,12 @@ impl From<ReqError> for UError {
}
}
impl From<io::Error> for UError {
fn from(err: io::Error) -> Self {
UError::IOError(err.to_string())
}
}
impl From<anyhow::Error> for UError {
fn from(e: anyhow::Error) -> Self {
let ctx = e
@ -73,7 +82,10 @@ impl From<anyhow::Error> for UError {
Ok(err) => UError::Contexted(Box::new(err), ctx),
Err(err) => match err.downcast::<ufs::Error>() {
Ok(err) => UError::Contexted(Box::new(UError::FSError(err)), ctx),
Err(err) => UError::Runtime(err.to_string()),
Err(err) => match err.downcast::<reqwest::Error>() {
Ok(err) => UError::Contexted(Box::new(UError::from(err)), ctx),
Err(err) => UError::Runtime(err.to_string()),
},
},
}
}

@ -1,8 +1,9 @@
use crate::{
combined_result::CombinedResult,
executor::{ExecResult, Waiter},
misc::OneOrVec,
models::{Agent, AssignedJob, AssignedJobById, FatJob, JobType, RawJob, ThinJob},
models::{
Agent, AssignedJob, AssignedJobById, BriefJob, FullJob, FullPayload, JobType, RawJob,
},
proc_output::ProcOutput,
ufs,
};
@ -16,9 +17,9 @@ pub struct AnonymousJobBatch {
}
impl AnonymousJobBatch {
pub fn from_meta_with_id(jobs: impl OneOrVec<(ThinJob, AssignedJobById)>) -> Self {
pub fn from_meta_with_id(jobs: impl IntoIterator<Item = (BriefJob, AssignedJobById)>) -> Self {
let mut waiter = Waiter::new();
for (job, ids) in jobs.into_vec() {
for (job, ids) in jobs {
waiter.push(run_assigned_job(job, ids));
}
Self {
@ -27,9 +28,8 @@ impl AnonymousJobBatch {
}
}
pub fn from_meta(jobs: impl OneOrVec<ThinJob>) -> Self {
pub fn from_meta(jobs: impl IntoIterator<Item = BriefJob>) -> Self {
let jobs_ids: Vec<_> = jobs
.into_vec()
.into_iter()
.map(|job| {
let job_id = job.job.id;
@ -76,30 +76,30 @@ pub struct NamedJobBatch<const FINISHED: bool = false> {
}
impl NamedJobBatch {
pub fn from_shell(
named_jobs: impl OneOrVec<(&'static str, &'static str)>,
) -> CombinedResult<Self> {
pub fn from_shell(named_jobs: Vec<(&'static str, &'static str)>) -> CombinedResult<Self> {
let mut result = CombinedResult::new();
let jobs: Vec<_> = named_jobs
.into_vec()
.into_iter()
.filter_map(|(alias, cmd)| {
match RawJob::builder().with_shell(cmd).with_alias(alias).build() {
match RawJob::brief_job_builder()
.with_shell(cmd)
.with_alias(alias)
.build()
{
Ok(jpm) => Some(jpm),
Err(e) => {
result.err(e);
result.push_err(e);
None
}
}
})
.collect();
result.ok(Self::from_meta(jobs));
result.push_ok(Self::from_meta(jobs));
result
}
pub fn from_meta(named_jobs: impl OneOrVec<ThinJob>) -> Self {
pub fn from_meta(named_jobs: Vec<BriefJob>) -> Self {
let (job_names, jobs): (Vec<_>, Vec<_>) = named_jobs
.into_vec()
.into_iter()
.map(|job| (job.job.alias.clone().unwrap(), job))
.unzip();
@ -134,8 +134,8 @@ impl NamedJobBatch<true> {
}
}
pub async fn run_assigned_job(job: ThinJob, ids: AssignedJobById) -> ExecResult {
let ThinJob { job, payload_meta } = job;
pub async fn run_assigned_job(job: BriefJob, ids: AssignedJobById) -> ExecResult {
let BriefJob { job, payload_meta } = job;
let mut result = AssignedJob::from((&job, ids));
match job.exec_type {
JobType::Shell => {
@ -181,40 +181,38 @@ pub async fn run_assigned_job(job: ThinJob, ids: AssignedJobById) -> ExecResult
Ok(result)
}
pub fn split_payload(job: FatJob) -> Result<ThinJob, ufs::Error> {
let FatJob {
job,
payload_meta,
payload_data,
} = job;
if let Some(meta) = &payload_meta {
if let Some(data) = payload_data {
if ufs::in_index(&meta.name) {
ufs::edit(&meta.name, data)?;
} else {
ufs::put(&meta.name, data)?;
}
pub fn split_payload(job: FullJob) -> Result<BriefJob, ufs::Error> {
let FullJob { job, payload } = job;
if let Some(payload) = &payload {
if ufs::exists_in_index(&payload.meta.name) {
ufs::edit(&payload.meta.name, &payload.data)?;
} else {
ufs::put(&payload.meta.name, &payload.data)?;
}
}
Ok(ThinJob { job, payload_meta })
}
pub fn join_payload(job: ThinJob) -> Result<FatJob, ufs::Error> {
let ThinJob { job, payload_meta } = job;
let payload_data = payload_meta
.as_ref()
.map(|p| ufs::read(&p.name))
.transpose()?;
Ok(FatJob {
Ok(BriefJob {
job,
payload_meta,
payload_data,
payload_meta: payload.map(|p| p.meta),
})
}
pub fn join_payload(job: BriefJob) -> Result<FullJob, ufs::Error> {
let BriefJob { job, payload_meta } = job;
let payload = match payload_meta {
Some(meta) => {
let payload_data = ufs::read(&meta.name)?;
Some(FullPayload {
meta,
data: payload_data,
})
}
None => None,
};
Ok(FullJob { job, payload })
}
#[cfg(test)]
mod tests {
use crate::{
@ -267,12 +265,15 @@ mod tests {
#[case] payload: Option<&[u8]>,
#[case] expected_result: &str,
) -> TestResult {
let mut job = RawJob::builder().with_shell(cmd);
let mut job = RawJob::brief_job_builder().with_shell(cmd);
if let Some(p) = payload {
job = job.with_raw_payload(p);
}
let job = job.build().unwrap();
let result = AnonymousJobBatch::from_meta(job).wait_one().await.unwrap();
let result = AnonymousJobBatch::from_meta([job])
.wait_one()
.await
.unwrap();
let result = result.to_str_result();
assert_eq!(result.trim(), expected_result);
Ok(())
@ -283,8 +284,8 @@ mod tests {
const SLEEP_SECS: u64 = 1;
let now = SystemTime::now();
let longest_job = RawJob::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap();
let longest_job = AnonymousJobBatch::from_meta(longest_job).spawn().await;
let ls = AnonymousJobBatch::from_meta(RawJob::from_shell("ls").unwrap())
let longest_job = AnonymousJobBatch::from_meta([longest_job]).spawn().await;
let ls = AnonymousJobBatch::from_meta([RawJob::from_shell("ls").unwrap()])
.wait_one()
.await
.unwrap();
@ -328,7 +329,10 @@ mod tests {
#[tokio::test]
async fn test_failing_shell_job() -> TestResult {
let job = RawJob::from_shell("lol_kek_puk").unwrap();
let job_result = AnonymousJobBatch::from_meta(job).wait_one().await.unwrap();
let job_result = AnonymousJobBatch::from_meta([job])
.wait_one()
.await
.unwrap();
let output = job_result.to_str_result();
assert!(output.contains("No such file"));
assert!(job_result.retcode.is_none());
@ -344,7 +348,7 @@ mod tests {
#[case] payload: Option<&[u8]>,
#[case] err_str: &str,
) -> TestResult {
let mut job = RawJob::builder().with_shell(cmd);
let mut job = RawJob::brief_job_builder().with_shell(cmd);
if let Some(p) = payload {
job = job.with_raw_payload(p);
}
@ -357,12 +361,12 @@ mod tests {
#[tokio::test]
async fn test_different_job_types() -> TestResult {
let mut jobs = NamedJobBatch::from_meta(vec![
RawJob::builder()
RawJob::brief_job_builder()
.with_shell("sleep 3")
.with_alias("sleeper")
.build()
.unwrap(),
RawJob::builder()
RawJob::brief_job_builder()
.with_type(JobType::Init)
.with_alias("gatherer")
.build()

@ -1,28 +1,35 @@
use std::env;
use std::io::{stderr, stdout};
use std::path::Path;
use tracing_appender::rolling;
use tracing_subscriber::{fmt, prelude::*, registry, EnvFilter};
pub fn init_logger(logfile: Option<impl AsRef<Path> + Send + Sync + 'static>) {
pub fn init_logger(logfile: Option<&str>) {
if env::var("RUST_LOG").is_err() {
env::set_var("RUST_LOG", "info")
}
let output_layer = if cfg!(test) {
fmt::layer().with_writer(stdout).with_test_writer().boxed()
} else {
fmt::layer().with_writer(stderr).boxed()
};
let reg = registry()
.with(EnvFilter::from_default_env())
.with(fmt::layer());
.with(output_layer);
match logfile {
Some(file) => reg
.with(
Some(file) => {
let file_path = Path::new(file).with_extension("log");
reg.with(
fmt::layer()
.with_writer(move || {
rolling::never("logs", file.as_ref().with_extension("log"))
})
.with_writer(move || rolling::never("logs", &file_path))
.with_ansi(false),
)
.init(),
.init()
}
None => reg.init(),
};
}

@ -11,13 +11,15 @@ impl AsMsg for Agent {}
impl AsMsg for AssignedJob {}
impl AsMsg for AssignedJobById {}
impl AsMsg for JobModel {}
impl AsMsg for FatJob {}
impl AsMsg for FullJob {}
impl AsMsg for Reportable {}
impl AsMsg for String {}
impl AsMsg for ThinJob {}
impl AsMsg for PayloadMeta {}
impl AsMsg for FullPayload {}
impl AsMsg for BriefJob {}
impl<B: AsMsg, F: AsMsg> AsMsg for BriefOrFull<B, F> {}
impl AsMsg for Id {}
impl AsMsg for i32 {}
impl AsMsg for u8 {}
impl AsMsg for String {}
impl AsMsg for Vec<u8> {}
impl AsMsg for () {}
impl<M: AsMsg> AsMsg for Vec<M> {}

@ -1,35 +1,3 @@
pub trait OneOrVec<T> {
fn into_vec(self) -> Vec<T>;
}
impl<T> OneOrVec<T> for T {
fn into_vec(self) -> Vec<T> {
vec![self]
}
}
impl<T> OneOrVec<T> for Vec<T> {
fn into_vec(self) -> Vec<T> {
self
}
}
pub trait OneOrVecRef<T> {
fn as_vec(&self) -> Vec<&T>;
}
impl<T> OneOrVecRef<T> for &T {
fn as_vec(&self) -> Vec<&T> {
vec![self]
}
}
impl<T> OneOrVecRef<T> for &Vec<T> {
fn as_vec(&self) -> Vec<&T> {
self.iter().collect()
}
}
#[macro_export]
macro_rules! unwrap_enum {
($src:expr, $t:path) => {

@ -1,10 +1,9 @@
use std::fmt;
use super::JobType;
use crate::conv::bytes_to_string;
#[cfg(feature = "server")]
use crate::models::schema::*;
use crate::models::PayloadMeta;
use crate::models::{FullPayload, PayloadMeta};
use crate::platform;
use crate::types::Id;
use crate::{ufs, UError, UResult};
@ -12,7 +11,6 @@ use crate::{ufs, UError, UResult};
use diesel::{Identifiable, Insertable, Queryable};
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::process::Command;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[cfg_attr(
@ -35,14 +33,13 @@ pub struct JobModel {
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct FatJob {
pub struct FullJob {
pub job: JobModel,
pub payload_meta: Option<PayloadMeta>,
pub payload_data: Option<Vec<u8>>,
pub payload: Option<FullPayload>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ThinJob {
pub struct BriefJob {
pub job: JobModel,
pub payload_meta: Option<PayloadMeta>,
}
@ -108,9 +105,9 @@ impl fmt::Debug for RawJob<'_> {
}
}
impl From<ThinJob> for RawJob<'_> {
fn from(job: ThinJob) -> Self {
let ThinJob { job, payload_meta } = job;
impl From<BriefJob> for RawJob<'_> {
fn from(job: BriefJob) -> Self {
let BriefJob { job, payload_meta } = job;
RawJob {
alias: job.alias,
argv: job.argv,
@ -125,15 +122,15 @@ impl From<ThinJob> for RawJob<'_> {
}
impl<'p> RawJob<'p> {
pub fn validated(self) -> UResult<ThinJob> {
pub fn validated(self) -> UResult<BriefJob> {
JobBuilder { inner: self }.build()
}
pub fn from_shell(cmd: impl Into<String>) -> UResult<ThinJob> {
Self::builder().with_shell(cmd).build()
pub fn from_shell(cmd: impl Into<String>) -> UResult<BriefJob> {
Self::brief_job_builder().with_shell(cmd).build()
}
pub fn builder() -> JobBuilder<'p> {
pub fn brief_job_builder() -> JobBuilder<'p> {
JobBuilder::default()
}
}
@ -177,43 +174,27 @@ impl<'p> JobBuilder<'p> {
self
}
pub fn build(self) -> UResult<ThinJob> {
pub fn build(self) -> UResult<BriefJob> {
let mut inner = self.inner;
let raw_into_job = |raw: RawJob| -> UResult<ThinJob> {
let payload_id = raw.payload_path.as_ref().map(|_| Id::new_v4());
let raw_into_job = |raw: RawJob| -> UResult<BriefJob> {
let payload_meta = raw
.payload_path
.as_ref()
.map(|payload_ident| PayloadMeta::from_existing_meta(payload_ident))
.transpose()?;
Ok(ThinJob {
Ok(BriefJob {
job: JobModel {
alias: raw.alias,
argv: raw.argv,
id: raw.id,
exec_type: raw.exec_type,
target_platforms: raw.target_platforms,
payload: payload_id,
payload: payload_meta.as_ref().map(|meta| meta.id),
schedule: raw.schedule,
},
payload_meta: raw
.payload_path
.map(|payload_ident| {
let ufs_meta = ufs::read_meta(&payload_ident)?;
let payload_meta = PayloadMeta {
id: payload_id.unwrap(),
mime_type: bytes_to_string(
&Command::new("file")
.arg("-b")
.arg("--mime-type")
.arg(&ufs_meta.path)
.output()
.map_err(|e| UError::JobBuildError(e.to_string()))?
.stdout,
),
name: payload_ident.clone(),
size: ufs_meta.size as i64,
};
Ok::<_, UError>(payload_meta)
})
.transpose()?,
payload_meta,
})
};

@ -4,4 +4,25 @@ mod payload;
#[cfg(feature = "server")]
pub mod schema;
use crate::messaging::AsMsg;
pub use crate::models::{agent::*, jobs::*, payload::*};
use serde::{Deserialize, Serialize};
use strum::{Display as StrumDisplay, EnumString};
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(untagged)]
pub enum BriefOrFull<B: AsMsg, F: AsMsg> {
Brief(B),
Full(F),
}
pub type BriefOrFullJob = BriefOrFull<BriefJob, FullJob>;
pub type BriefOrFullPayload = BriefOrFull<PayloadMeta, FullPayload>;
#[derive(Default, Debug, StrumDisplay, EnumString, Deserialize)]
pub enum BriefMode {
Yes,
#[default]
Auto,
No,
}

@ -1,5 +1,6 @@
use crate::types::Id;
use crate::{conv::bytes_to_string, types::Id, ufs, UError};
use serde::{Deserialize, Serialize};
use std::process::Command;
#[cfg(feature = "server")]
use crate::models::schema::*;
@ -18,3 +19,33 @@ pub struct PayloadMeta {
pub name: String,
pub size: i64,
}
impl PayloadMeta {
pub fn from_existing_meta(payload_ufs_ident: &str) -> Result<PayloadMeta, UError> {
let ufs_meta = ufs::read_meta(&payload_ufs_ident)?;
let mime_type = bytes_to_string(
&Command::new("file")
.arg("-b")
.arg("--mime-type")
.arg(&ufs_meta.path)
.output()
.map_err(|e| UError::IOError(e.to_string()))?
.stdout,
)
.trim()
.to_string();
Ok(PayloadMeta {
id: Id::new_v4(),
mime_type,
name: payload_ufs_ident.to_owned(),
size: ufs_meta.size as i64,
})
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct FullPayload {
pub meta: PayloadMeta,
pub data: Vec<u8>,
}

@ -1,4 +1,4 @@
// This module is aiming to store obfuscated payloads, get them by name,
// This module is aiming to store (obfuscated?) payloads, get them by name,
// rename, update, delete or prepare to execute via memfd_create (unix)
use anyhow::{Context, Result};
@ -45,8 +45,8 @@ impl FileMeta {
}
/// Check if file exists in index.
/// File may present in fs but not in index, thus fn will return false.
pub fn in_index(name: impl AsRef<str>) -> bool {
/// File may present in fs but not in index, fn will return false then.
pub fn exists_in_index(name: impl AsRef<str>) -> bool {
read_meta(name).is_ok()
}
@ -86,7 +86,7 @@ pub fn put(name: impl AsRef<str>, data: impl AsRef<[u8]>) -> Result<()> {
let name = name.as_ref();
let data_hash = hash_data(&data);
if in_index(&name) {
if exists_in_index(&name) {
return Err(Error::already_exists(&name)).context("put_exists");
}
@ -158,11 +158,11 @@ pub fn rename(old_name: impl AsRef<str>, new_name: impl AsRef<str>) -> Result<()
return Ok(());
}
if !in_index(old_name) {
if !exists_in_index(old_name) {
return Err(Error::not_found(old_name)).context("rename");
}
if in_index(new_name) {
if exists_in_index(new_name) {
return Err(Error::already_exists(new_name)).context("rename");
}
@ -190,7 +190,7 @@ pub fn update_payload_data(name: impl AsRef<str>, data: impl AsRef<[u8]>) -> Res
if external {
index::remove(name);
} else if in_index(&name) {
} else if exists_in_index(&name) {
remove(&name).context("upd")?;
}
@ -202,7 +202,7 @@ pub fn put_external(path: impl AsRef<Path>) -> Result<()> {
let path = path.as_ref();
let path_str = path.as_os_str().to_string_lossy().to_string();
if in_index(&path_str) {
if exists_in_index(&path_str) {
return Ok(());
}

Loading…
Cancel
Save