implement payload table

pull/9/head
plazmoid 1 year ago
parent 69b1d3d901
commit 886d4833fb
  1. 11
      .cargo/config.toml
  2. 1146
      Cargo.lock
  3. 1
      Cargo.toml
  4. 13
      Makefile.toml
  5. 4
      bin/u_panel/src/argparse.rs
  6. 4
      bin/u_panel/src/gui/fe/src/app/app.module.ts
  7. 4
      bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html
  8. 1
      bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts
  9. 20
      bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.html
  10. 43
      bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.ts
  11. 45
      bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html
  12. 9
      bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts
  13. 8
      bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.html
  14. 13
      bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html
  15. 7
      bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts
  16. 4
      bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts
  17. 3
      bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.html
  18. 49
      bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts
  19. 5
      bin/u_panel/src/gui/fe/src/app/models/payload.model.ts
  20. 2
      bin/u_panel/src/gui/fe/src/app/models/result.model.ts
  21. 11
      bin/u_panel/src/gui/fe/src/app/services/api.service.ts
  22. 4
      bin/u_panel/src/gui/mod.rs
  23. 17
      bin/u_server/src/db.rs
  24. 103
      bin/u_server/src/handlers.rs
  25. 9
      bin/u_server/src/u_server.rs
  26. 2
      images/tests_runner.Dockerfile
  27. 2
      integration-tests/tests/fixtures/agent.rs
  28. 31
      integration-tests/tests/integration_tests/endpoints.rs
  29. 52
      lib/u_lib/src/api.rs
  30. 2
      lib/u_lib/src/models/jobs/meta.rs
  31. 2
      lib/u_lib/src/models/mod.rs
  32. 7
      lib/u_lib/src/models/payload.rs
  33. 2
      migrations/2020-10-24-111622_create_all/up.sql

@ -1,6 +1,15 @@
[build]
rustflags = [
"-L", "/home/ortem/src/rust/unki/static/lib",
"-L/usr/lib/musl/lib",
"-L/home/ortem/src/rust/unki/static/lib",
"--remap-path-prefix=/home/ortem/src/rust/unki=src",
"--remap-path-prefix=/home/ortem/.cargo=cargo"
]
target = "x86_64-unknown-linux-musl"
[env]
STATIC_PREFIX = "static"
PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true"
PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = { value = "static/bin/pg_config", relative = true }
OPENSSL_STATIC = "true"
OPENSSL_DIR = { value = "static", relative = true }

1146
Cargo.lock generated

File diff suppressed because it is too large Load Diff

@ -8,6 +8,7 @@ members = [
"lib/u_lib",
"integration-tests",
]
resolver = "2"
[workspace.dependencies]
anyhow = "=1.0.63"

@ -22,13 +22,6 @@ default_to_workspace = false
[env]
TARGET = "x86_64-unknown-linux-musl"
CARGO = "cargo"
ROOTDIR = "${CARGO_MAKE_WORKING_DIRECTORY}"
STATIC_PREFIX = "${ROOTDIR}/static"
PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true"
PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = "${STATIC_PREFIX}/bin/pg_config"
OPENSSL_STATIC = "true"
OPENSSL_DIR = "${STATIC_PREFIX}"
[tasks.build_static_libs]
script = "./scripts/build_musl_libs.sh"
@ -69,6 +62,12 @@ clear = true
[tasks.run]
disabled = true
[tasks.run_front]
script = '''
cd ./bin/u_panel/src/gui/fe
ng serve
'''
[tasks.unit-tests]
command = "${CARGO}"
args = ["test", "--target", "${TARGET}", "--lib", "--", "${@}"]

@ -9,7 +9,7 @@ pub struct Args {
#[structopt(subcommand)]
cmd: Cmd,
#[structopt(short, long, default_value)]
brief: BriefMode,
brief: Brief,
}
#[derive(StructOpt, Debug)]
@ -141,7 +141,7 @@ pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult<Value> {
PayloadCRUD::Create { item } => {
let payload = from_str::<RawPayload>(&item)
.map_err(|e| UError::DeserializeError(e.to_string(), item))?;
into_value(client.upload_payloads([&payload]).await?)
into_value(client.upload_payload(&payload).await?)
}
PayloadCRUD::Read { id } => match id {
None => into_value(client.get_payloads().await?),

@ -28,6 +28,7 @@ import { MatSnackBarModule } from '@angular/material/snack-bar';
import { MatListModule } from '@angular/material/list';
import { GlobalErrorComponent } from './components/global-error/global-error.component';
import { PayloadOverviewComponent } from './components/payload-overview/payload-overview.component';
import { NewPayloadDialogComponent } from './components/dialogs/new-payload-dialog/new-payload-dialog.component';
@NgModule({
declarations: [
@ -42,7 +43,8 @@ import { PayloadOverviewComponent } from './components/payload-overview/payload-
PayloadComponent,
PayloadInfoDialogComponent,
GlobalErrorComponent,
PayloadOverviewComponent
PayloadOverviewComponent,
NewPayloadDialogComponent
],
imports: [
BrowserModule,

@ -37,7 +37,9 @@
</mat-select>
</mat-form-field>
</div>
<payload-overview *ngIf="data.payload" [preview]="true" [payload]="data.payload"></payload-overview>
<div class="info-dialog-forms-box">
<payload-overview *ngIf="data.payload" [preview]="true" [payload]="data.payload.data"></payload-overview>
</div>
</mat-dialog-content>
<mat-dialog-actions align="end">
<button mat-raised-button *ngIf="isPreview" (click)="isPreview = false">Edit</button>

@ -3,7 +3,6 @@ import { MAT_DIALOG_DATA } from '@angular/material/dialog';
import { EventEmitter } from '@angular/core';
import { Job, JobModel } from '../../../models/job.model';
import { ApiTableService } from 'src/app/services';
import { PayloadModel } from 'src/app/models';
@Component({
selector: 'job-info-dialog',

@ -0,0 +1,20 @@
<h2 mat-dialog-title>New payload</h2>
<mat-dialog-content>
<div class="info-dialog-forms-box-smol">
<mat-form-field class="info-dlg-field" cdkFocusInitial>
<mat-label>Name</mat-label>
<input matInput [(ngModel)]="payload.name">
</mat-form-field>
<input type="file" class="file-input" (change)="onFileSelected($event)" #fileUpload>
</div>
<div class="info-dialog-forms-box">
<mat-form-field class="info-dlg-field" *ngIf="!uploadMode">
<mat-label>Data</mat-label>
<textarea matInput [(ngModel)]="decodedPayload"></textarea>
</mat-form-field>
</div>
</mat-dialog-content>
<mat-dialog-actions align="end">
<button mat-raised-button (click)="save()">Save</button>
<button mat-button mat-dialog-close>Close</button>
</mat-dialog-actions>

@ -0,0 +1,43 @@
import { Component, EventEmitter, Inject } from '@angular/core';
import { MAT_DIALOG_DATA } from '@angular/material/dialog';
import { NewPayloadModel } from 'src/app/models/payload.model';
@Component({
selector: 'new-payload-dialog',
templateUrl: 'new-payload-dialog.component.html',
styleUrls: ['../base-info-dialog.component.less']
})
export class NewPayloadDialogComponent {
decodedPayload = "";
uploadMode = false;
onSave = new EventEmitter<NewPayloadModel>();
constructor(@Inject(MAT_DIALOG_DATA) public payload: NewPayloadModel) { }
save() {
if (this.payload.data.length == 0) {
this.payload.data = Array.from(new TextEncoder().encode(this.decodedPayload));
}
this.onSave.emit(this.payload);
}
onFileSelected(event: any) {
const file: File = event.target.files[0];
if (file) {
this.uploadMode = true
const reader = new FileReader();
reader.onload = e => {
this.payload.name = file.name;
const result = e.target?.result;
if (result instanceof ArrayBuffer) {
const d = Array.from(new Uint8Array(result));
this.payload.data = d;
console.log(this.payload.data)
} else {
alert!("no file")
}
}
reader.readAsArrayBuffer(file)
}
}
}

@ -1,27 +1,34 @@
<h2 mat-dialog-title>Payload</h2>
<h2 mat-dialog-title *ngIf="isPreview">Payload</h2>
<h2 mat-dialog-title *ngIf="!isPreview">Editing payload</h2>
<mat-dialog-content>
<div class="info-dialog-forms-box-smol">
<mat-form-field class="info-dlg-field" cdkFocusInitial>
<mat-label>ID</mat-label>
<input matInput readonly value="{{payload.id}}">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Name</mat-label>
<input matInput value="{{payload.name}}">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>MIME-type</mat-label>
<input matInput value="{{payload.mime_type}}">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Size</mat-label>
<input matInput value="{{payload.size}}">
</mat-form-field>
<div class="info-dialog-forms-box">
<div class="info-dialog-forms-box-smol">
<mat-form-field class="info-dlg-field" cdkFocusInitial>
<mat-label>ID</mat-label>
<input matInput disabled value="{{payload.id}}">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Name</mat-label>
<input matInput [readonly]="isPreview" [(ngModel)]="payload.name">
</mat-form-field>
</div>
<div class="info-dialog-forms-box-smol">
<mat-form-field class="info-dlg-field">
<mat-label>MIME-type</mat-label>
<input matInput disabled value="{{payload.mime_type}}">
</mat-form-field>
<mat-form-field class="info-dlg-field">
<mat-label>Size</mat-label>
<input matInput disabled value="{{payload.size}}">
</mat-form-field>
</div>
</div>
<div class="info-dialog-forms-box">
<payload-overview [preview]="true" [payload]="payload"></payload-overview>
<payload-overview [preview]="isPreview" [payload]="payload.data"></payload-overview>
</div>
</mat-dialog-content>
<mat-dialog-actions align="end">
<button mat-raised-button *ngIf="isPreview" (click)="isPreview = false">Edit</button>
<button mat-raised-button *ngIf="!isPreview" (click)="updatePayload()">Save</button>
<button mat-button mat-dialog-close>Close</button>
</mat-dialog-actions>

@ -1,4 +1,4 @@
import { Component, Inject } from '@angular/core';
import { Component, EventEmitter, Inject } from '@angular/core';
import { MAT_DIALOG_DATA } from '@angular/material/dialog';
import { PayloadModel } from 'src/app/models/payload.model';
@ -8,6 +8,13 @@ import { PayloadModel } from 'src/app/models/payload.model';
styleUrls: ['../base-info-dialog.component.less']
})
export class PayloadInfoDialogComponent {
isPreview = true;
onSave = new EventEmitter<PayloadModel>();
constructor(@Inject(MAT_DIALOG_DATA) public payload: PayloadModel) { }
updatePayload() {
this.onSave.emit(this.payload);
}
}

@ -39,13 +39,7 @@
</mat-form-field>
</div>
<div class="info-dialog-forms-box">
<p>
<mat-form-field class="info-dlg-field">
<mat-label>Result</mat-label>
<textarea matInput cdkTextareaAutosize readonly value="{{decodedResult}}">
</textarea>
</mat-form-field>
</p>
<payload-overview [preview]="true" [payload]="data.result"></payload-overview>
</div>
</mat-dialog-content>
<mat-dialog-actions align="end">

@ -1,9 +1,6 @@
<div class="info-dialog-forms-box">
<mat-form-field class="info-dlg-field" floatLabel="always">
<mat-label>Payload data</mat-label>
<textarea matInput cdkTextareaAutosize *ngIf="!isTooBigPayload" [readonly]="isPreview"
[(ngModel)]="decodedPayload">
<mat-form-field class="info-dlg-field" floatLabel="always">
<mat-label>Payload data</mat-label>
<textarea matInput cdkTextareaAutosize *ngIf="!isTooBigPayload" [readonly]="isPreview" [(ngModel)]="decodedPayload">
</textarea>
<input matInput *ngIf="isTooBigPayload" disabled placeholder="Payload is too big to display">
</mat-form-field>
</div>
<input matInput *ngIf="isTooBigPayload" disabled placeholder="Payload is too big to display">
</mat-form-field>

@ -1,5 +1,4 @@
import { Component, Input, OnInit } from '@angular/core';
import { PayloadModel } from 'src/app/models';
@Component({
selector: 'payload-overview',
@ -7,14 +6,14 @@ import { PayloadModel } from 'src/app/models';
styleUrls: ['./payload-overview.component.less']
})
export class PayloadOverviewComponent implements OnInit {
@Input() payload!: PayloadModel;
@Input() payload: number[] | null = null;
@Input("preview") isPreview = true;
isTooBigPayload = false;
decodedPayload = "";
ngOnInit() {
if (this.payload.data !== null) {
this.decodedPayload = new TextDecoder().decode(new Uint8Array(this.payload.data))
if (this.payload !== null) {
this.decodedPayload = new TextDecoder().decode(new Uint8Array(this.payload))
} else {
this.isTooBigPayload = true
}

@ -22,11 +22,11 @@ export abstract class TableComponent<T extends ApiModel> implements OnInit {
this.loadTableData();
this.route.queryParams.subscribe(params => {
const id = params['id']
const new_agent = params['new']
const new_item = params['new']
if (id) {
this.showItemDialog(id);
}
if (new_agent) {
if (new_item) {
this.showItemDialog(null);
}
})

@ -9,6 +9,9 @@
<input matInput (keyup)="applyFilter($event)" #input>
</mat-form-field>
<button id="refresh_btn" mat-raised-button color="primary" (click)="loadTableData()">Refresh</button>
<button id="new_btn" mat-raised-button color="primary" routerLink='.' [queryParams]="{new: true}">Add
payload</button>
<table mat-table fixedLayout="true" [dataSource]="table_data" class="data-table" matSort matSortActive="id"
matSortDisableClear matSortDirection="desc">

@ -1,7 +1,8 @@
import { Component } from '@angular/core';
import { Area } from 'src/app/models';
import { PayloadModel } from 'src/app/models/payload.model';
import { NewPayloadModel, PayloadModel } from 'src/app/models/payload.model';
import { PayloadInfoDialogComponent } from '../../dialogs';
import { NewPayloadDialogComponent } from '../../dialogs/new-payload-dialog/new-payload-dialog.component';
import { TableComponent } from '../base-table/base-table.component';
@Component({
@ -14,17 +15,53 @@ export class PayloadComponent extends TableComponent<PayloadModel> {
area = 'payloads' as Area
displayedColumns = ["name", "mime_type", "size", 'actions'];
showItemDialog(id: string) {
this.dataSource.getPayload(id).subscribe(resp => {
const dialog = this.infoDialog.open(PayloadInfoDialogComponent, {
data: resp,
showItemDialog(id: string | null) {
if (id === null) {
const payload: NewPayloadModel = {
name: "",
data: []
}
const dialog = this.infoDialog.open(NewPayloadDialogComponent, {
data: payload,
width: '1000px',
});
dialog.componentInstance.onSave.subscribe(result => {
this.dataSource.createPayload(result)
.subscribe(_ => {
alert("Created")
this.loadTableData()
})
dialog.close()
})
dialog.afterClosed().subscribe(_ => {
this.router.navigate(['.'], { relativeTo: this.route })
})
})
} else {
this.dataSource.getPayload(id as string).subscribe(resp => {
const dialog = this.infoDialog.open(PayloadInfoDialogComponent, {
data: resp,
width: '1000px',
});
const saveSub = dialog.componentInstance.onSave.subscribe(result => {
this.dataSource.updatePayload(result)
.subscribe(_ => {
alert("Updated")
this.loadTableData()
})
dialog.close()
})
dialog.afterClosed().subscribe(_ => {
saveSub.unsubscribe()
this.router.navigate(['.'], { relativeTo: this.route })
})
})
}
}
}

@ -5,3 +5,8 @@ export interface PayloadModel {
size: number,
data: number[] | null
}
export interface NewPayloadModel {
name: string,
data: number[]
}

@ -6,7 +6,7 @@ export interface ResultModel {
created: UTCDate,
id: string,
job_id: string,
result: number[],
result: number[] | null,
state: "Queued" | "Running" | "Finished",
retcode: number | null,
updated: UTCDate,

@ -1,7 +1,7 @@
import { environment } from 'src/environments/environment';
import { HttpClient, HttpErrorResponse } from '@angular/common/http';
import { Observable, map, catchError, throwError } from 'rxjs';
import { ApiModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, Job } from '../models';
import { ApiModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, Job, NewPayloadModel } from '../models';
import { Injectable, Inject } from '@angular/core';
import { ErrorService } from './error.service';
@ -119,6 +119,10 @@ export class ApiTableService {
return this.create(item, 'map')
}
createPayload(item: NewPayloadModel): Observable<string[]> {
return this.create(item, 'payloads')
}
filterErrStatus<R extends ApiModel>(obs: Observable<ServerResponse<R>>): Observable<R> {
return obs.pipe(
map(r => {
@ -131,7 +135,8 @@ export class ApiTableService {
}
errorHandler(err: HttpErrorResponse, caught: any) {
this.errorService.handle(caught.data ?? err.message);
return throwError(() => new Error(err.message));
var error = err.error.data !== undefined ? JSON.stringify(err.error.data) : err.message;
this.errorService.handle(error);
return throwError(() => new Error());
}
}

@ -65,10 +65,8 @@ async fn send_cmd(
let response = if result.is_ok() {
HttpResponse::Ok().body(result_string)
} else if result.is_err() {
HttpResponse::BadRequest().body(result_string)
} else {
unreachable!()
HttpResponse::BadRequest().body(result_string)
};
Ok(response)

@ -60,14 +60,14 @@ impl UDB<'_> {
.map_err(with_err_ctx("Can't insert jobs"))
}
pub fn insert_payloads(&mut self, payloads: &[Payload]) -> Result<()> {
pub fn insert_payload(&mut self, payload: &Payload) -> Result<()> {
use schema::payloads;
diesel::insert_into(payloads::table)
.values(payloads)
.values(payload)
.execute(self.conn)
.map(drop)
.map_err(with_err_ctx(format!("Can't insert payloads {payloads:?}")))
.map_err(with_err_ctx(format!("Can't insert payload {payload:?}")))
}
pub fn get_job(&mut self, id: Id) -> Result<Option<Job>> {
@ -119,6 +119,17 @@ impl UDB<'_> {
.map_err(with_err_ctx("Can't get payloads"))
}
pub fn payload_exists(&mut self, payload_id: Id) -> Result<bool> {
use schema::payloads;
payloads::table
.find(payload_id)
.first::<Payload>(self.conn)
.optional()
.map(|r| r.is_some())
.map_err(with_err_ctx("Can't check payload {payload_id}"))
}
pub fn get_job_by_alias(&mut self, alias: &str) -> Result<Option<Job>> {
use schema::{jobs, payloads};

@ -3,7 +3,7 @@ use std::sync::Arc;
use crate::db::{PgRepo, UDB};
use crate::error::Error;
use serde::Deserialize;
use u_lib::{api::retypes, messaging::Reportable, models::*, types::Id};
use u_lib::{api::api_types, messaging::Reportable, models::*, types::Id};
use warp::reject::not_found;
use warp::Rejection;
@ -11,13 +11,13 @@ type EndpResult<T> = Result<T, Rejection>;
#[derive(Deserialize)]
pub struct PayloadFlags {
brief: BriefMode,
brief: Brief,
}
pub struct Endpoints;
impl Endpoints {
pub async fn get_agents(repo: Arc<PgRepo>, id: Option<Id>) -> EndpResult<retypes::GetAgents> {
pub async fn get_agents(repo: Arc<PgRepo>, id: Option<Id>) -> EndpResult<api_types::GetAgents> {
repo.interact(move |mut db| {
Ok(match id {
Some(id) => {
@ -38,20 +38,20 @@ impl Endpoints {
repo: Arc<PgRepo>,
id: Id,
params: Option<PayloadFlags>,
) -> EndpResult<retypes::GetJob> {
) -> EndpResult<api_types::GetJob> {
let Some(mut job) = repo.interact(move |mut db| db.get_job(id)).await? else {
return Err(not_found())
return Err(not_found());
};
Ok(match params.map(|p| p.brief) {
Some(BriefMode::Yes) => job,
Some(BriefMode::Auto) | None => {
Some(Brief::Yes) => job,
Some(Brief::Auto) | None => {
if let Some(payload) = &mut job.payload {
payload.maybe_join_payload().map_err(Error::from)?;
}
job
}
Some(BriefMode::No) => {
Some(Brief::No) => {
if let Some(payload) = &mut job.payload {
payload.join_payload().map_err(Error::from)?;
}
@ -60,7 +60,7 @@ impl Endpoints {
})
}
pub async fn get_jobs(repo: Arc<PgRepo>) -> EndpResult<retypes::GetJobs> {
pub async fn get_jobs(repo: Arc<PgRepo>) -> EndpResult<api_types::GetJobs> {
repo.interact(move |mut db| db.get_jobs())
.await
.map_err(From::from)
@ -69,13 +69,13 @@ impl Endpoints {
pub async fn get_assigned_jobs(
repo: Arc<PgRepo>,
id: Option<Id>,
) -> EndpResult<retypes::GetAgentJobs> {
) -> EndpResult<api_types::GetAgentJobs> {
repo.interact(move |mut db| db.get_assigned_jobs(id, false))
.await
.map_err(From::from)
}
pub async fn get_payloads(repo: Arc<PgRepo>) -> EndpResult<retypes::GetPayloads> {
pub async fn get_payloads(repo: Arc<PgRepo>) -> EndpResult<api_types::GetPayloads> {
repo.interact(move |mut db| db.get_payloads())
.await
.map_err(From::from)
@ -85,7 +85,7 @@ impl Endpoints {
repo: Arc<PgRepo>,
name_or_id: String,
params: Option<PayloadFlags>,
) -> EndpResult<retypes::GetPayload> {
) -> EndpResult<api_types::GetPayload> {
let mut payload = match repo
.interact(move |mut db| match Id::parse_str(&name_or_id) {
Ok(id) => db.get_payload(id),
@ -98,11 +98,11 @@ impl Endpoints {
};
Ok(match params.map(|p| p.brief) {
Some(BriefMode::Yes) => {
Some(Brief::Yes) => {
payload.data = None;
payload
}
None | Some(BriefMode::Auto) => {
None | Some(Brief::Auto) => {
payload.maybe_join_payload().map_err(Error::from)?;
payload
}
@ -116,7 +116,7 @@ impl Endpoints {
pub async fn get_personal_jobs(
repo: Arc<PgRepo>,
id: Id,
) -> EndpResult<retypes::GetPersonalJobs> {
) -> EndpResult<api_types::GetPersonalJobs> {
repo.transaction(move |mut db| {
let agent = db.get_agent(id)?;
match agent {
@ -153,19 +153,33 @@ impl Endpoints {
.map_err(From::from)
}
pub async fn upload_jobs(repo: Arc<PgRepo>, msg: Vec<Job>) -> EndpResult<retypes::UploadJobs> {
let jobs = msg
.into_iter()
.map(|mut job| {
if let Some(payload) = &mut job.payload {
payload.maybe_split_payload()?;
pub async fn upload_jobs(
repo: Arc<PgRepo>,
jobs: Vec<Job>,
) -> EndpResult<api_types::UploadJobs> {
let mut checked_jobs = vec![];
for mut job in jobs {
debug!("{job:?}");
if let Some(payload) = &mut job.payload {
payload.maybe_split_payload().map_err(Error::from)?;
} else if let Some(pld_id) = job.meta.payload_id {
if repo
.interact(move |mut db| db.payload_exists(pld_id))
.await?
{
checked_jobs.push(job)
} else {
Err(Error::ProcessingError(format!(
"Payload {pld_id} not found"
)))?
}
Ok(job)
})
.collect::<Result<Vec<Job>, Error>>()?;
}
}
let (jobs, payloads_opt): (Vec<_>, Vec<_>) =
jobs.into_iter().map(|j| (j.meta, j.payload)).unzip();
let (jobs, payloads_opt): (Vec<_>, Vec<_>) = checked_jobs
.into_iter()
.map(|j| (j.meta, j.payload))
.unzip();
let payloads = payloads_opt
.into_iter()
@ -173,7 +187,9 @@ impl Endpoints {
.collect::<Vec<_>>();
repo.transaction(move |mut db| {
db.insert_payloads(&payloads)?;
for payload in payloads {
db.insert_payload(&payload)?;
}
db.insert_jobs(&jobs)
})
.await
@ -182,15 +198,11 @@ impl Endpoints {
pub async fn upload_payloads(
repo: Arc<PgRepo>,
raw_payloads: Vec<RawPayload>,
) -> EndpResult<retypes::UploadPayloads> {
let payloads = raw_payloads
.into_iter()
.map(|raw| raw.into_payload())
.collect::<Result<Vec<_>, _>>()
.map_err(Error::from)?;
raw_payload: RawPayload,
) -> EndpResult<api_types::UploadPayloads> {
let payloads = raw_payload.into_payload().map_err(Error::from)?;
repo.interact(move |mut db| db.insert_payloads(&payloads))
repo.interact(move |mut db| db.insert_payload(&payloads))
.await
.map_err(From::from)
}
@ -215,7 +227,7 @@ impl Endpoints {
repo: Arc<PgRepo>,
agent_id: Id,
job_idents: Vec<String>,
) -> EndpResult<retypes::SetJobs> {
) -> EndpResult<api_types::SetJobs> {
repo.transaction(move |mut db| {
let assigned_job_idents = job_idents
.into_iter()
@ -245,7 +257,7 @@ impl Endpoints {
repo: Arc<PgRepo>,
msg: Vec<Reportable>,
agent_id: Id,
) -> EndpResult<retypes::Report> {
) -> EndpResult<api_types::Report> {
repo.transaction(move |mut db| {
for entry in msg {
match entry {
@ -297,13 +309,16 @@ impl Endpoints {
.map_err(From::from)
}
pub async fn update_agent(repo: Arc<PgRepo>, agent: Agent) -> EndpResult<retypes::UpdateAgent> {
pub async fn update_agent(
repo: Arc<PgRepo>,
agent: Agent,
) -> EndpResult<api_types::UpdateAgent> {
repo.interact(move |mut db| db.upsert_agent(&agent))
.await
.map_err(From::from)
}
pub async fn update_job(repo: Arc<PgRepo>, job: JobMeta) -> EndpResult<retypes::UpdateJob> {
pub async fn update_job(repo: Arc<PgRepo>, job: JobMeta) -> EndpResult<api_types::UpdateJob> {
repo.interact(move |mut db| db.update_job(&job.validate()?))
.await
.map_err(From::from)
@ -312,7 +327,7 @@ impl Endpoints {
pub async fn update_assigned_job(
repo: Arc<PgRepo>,
assigned: AssignedJob,
) -> EndpResult<retypes::UpdateResult> {
) -> EndpResult<api_types::UpdateResult> {
repo.interact(move |mut db| db.update_result(&assigned))
.await
.map_err(From::from)
@ -321,7 +336,8 @@ impl Endpoints {
pub async fn update_payload(
repo: Arc<PgRepo>,
payload: Payload,
) -> EndpResult<retypes::UpdatePayload> {
) -> EndpResult<api_types::UpdatePayload> {
debug!("update payload: {payload:?}");
match payload.data {
Some(data) => {
let mut well_formed_payload =
@ -332,7 +348,10 @@ impl Endpoints {
.await
.map_err(From::from)
}
None => return Ok(()),
None => repo
.interact(move |mut db| db.update_payload(&payload))
.await
.map_err(From::from),
}
}
}

@ -5,6 +5,7 @@ mod db;
mod error;
mod handlers;
use crate::handlers::{Endpoints, PayloadFlags};
use db::PgRepo;
use error::{Error as ServerError, RejResponse};
use std::{convert::Infallible, sync::Arc};
@ -24,8 +25,6 @@ use warp::{
const DEFAULT_RESPONSE: &str = "null";
use crate::handlers::{Endpoints, PayloadFlags};
fn into_message<M: AsMsg>(msg: M) -> Json {
json(&msg)
}
@ -139,9 +138,9 @@ pub fn init_endpoints(
.and_then(Endpoints::get_payload)
.map(into_message);
let upload_payloads = path("upload_payloads")
let upload_payload = path("upload_payload")
.and(with_db.clone())
.and(body::json::<Vec<RawPayload>>())
.and(body::json::<RawPayload>())
.and_then(Endpoints::upload_payloads)
.map(ok);
@ -162,7 +161,7 @@ pub fn init_endpoints(
.or(get_payloads)
.or(get_payload)
.or(upload_jobs)
.or(upload_payloads)
.or(upload_payload)
.or(del)
.or(set_jobs)
.or(get_assigned_jobs)

@ -1,4 +1,4 @@
FROM rust:1.67
FROM rust:1.72
RUN rustup target add x86_64-unknown-linux-musl
RUN mkdir -p /tests && chmod 777 /tests

@ -21,7 +21,7 @@ pub fn registered_agent(client: &HttpClient) -> RegisteredAgent {
.pop()
.unwrap();
let job_id = resp.job_id;
let job = client.get_job(job_id, BriefMode::No).await.unwrap();
let job = client.get_job(job_id, Brief::No).await.unwrap();
assert_eq!(job.meta.alias, Some("agent_hello".to_string()));

@ -15,7 +15,7 @@
use crate::fixtures::connections::*;
use std::iter::repeat;
use u_lib::models::{BriefMode, RawJob, RawPayload, MAX_READABLE_PAYLOAD_SIZE};
use u_lib::models::{Brief, RawJob, RawPayload, MAX_READABLE_PAYLOAD_SIZE};
#[rstest]
#[tokio::test]
@ -62,16 +62,10 @@ async fn payloads_upload_update_get_del(client_panel: &HttpClient) {
data: data.clone(),
};
client_panel.upload_payloads([&payload]).await.unwrap();
client_panel.upload_payload(&payload).await.unwrap();
let mut fetched_payload = client_panel
.get_payload(&name, BriefMode::No)
.await
.unwrap();
let fetched_payload_auto = client_panel
.get_payload(&name, BriefMode::Auto)
.await
.unwrap();
let mut fetched_payload = client_panel.get_payload(&name, Brief::No).await.unwrap();
let fetched_payload_auto = client_panel.get_payload(&name, Brief::Auto).await.unwrap();
assert_eq!(fetched_payload, fetched_payload_auto);
assert_eq!(fetched_payload.data.unwrap(), data);
@ -82,30 +76,21 @@ async fn payloads_upload_update_get_del(client_panel: &HttpClient) {
fetched_payload.data = Some(big_data.clone());
client_panel.update_payload(&fetched_payload).await.unwrap();
let fetched_big_payload = client_panel
.get_payload(&name, BriefMode::Yes)
.await
.unwrap();
let fetched_big_payload_auto = client_panel
.get_payload(&name, BriefMode::Auto)
.await
.unwrap();
let fetched_big_payload = client_panel.get_payload(&name, Brief::Yes).await.unwrap();
let fetched_big_payload_auto = client_panel.get_payload(&name, Brief::Auto).await.unwrap();
assert_eq!(fetched_big_payload, fetched_big_payload_auto);
assert_eq!(fetched_big_payload.size, new_size);
assert!(fetched_big_payload.data.is_none());
let fetched_big_payload_full = client_panel
.get_payload(&name, BriefMode::No)
.await
.unwrap();
let fetched_big_payload_full = client_panel.get_payload(&name, Brief::No).await.unwrap();
assert_eq!(fetched_big_payload_full.data.unwrap(), big_data);
client_panel.del(fetched_big_payload_full.id).await.unwrap();
let not_found_err = client_panel
.get_payload(&name, BriefMode::Yes)
.get_payload(&name, Brief::Yes)
.await
.unwrap_err();
assert!(not_found_err.to_string().contains("404 Not Found"))

@ -19,7 +19,7 @@ use crate::{
const AGENT_IDENTITY: &[u8] = include_bytes!("../../../certs/alice.p12");
const ROOT_CA_CERT: &[u8] = include_bytes!("../../../certs/ca.crt");
pub mod retypes {
pub mod api_types {
use super::*;
pub type GetPersonalJobs = Vec<AssignedJobById>;
@ -145,7 +145,7 @@ impl HttpClient {
}
/// get jobs for agent
pub async fn get_personal_jobs(&self, agent_id: Id) -> Result<retypes::GetPersonalJobs> {
pub async fn get_personal_jobs(&self, agent_id: Id) -> Result<api_types::GetPersonalJobs> {
self.req(format!("get_personal_jobs/{}", agent_id)).await
}
@ -153,7 +153,7 @@ impl HttpClient {
pub async fn report(
&self,
payload: impl IntoIterator<Item = messaging::Reportable>,
) -> Result<retypes::Report> {
) -> Result<api_types::Report> {
self.req_with_payload("report", &payload.into_iter().collect::<Vec<_>>())
.await
}
@ -164,20 +164,20 @@ impl HttpClient {
}
/// get exact job
pub async fn get_job(&self, job: Id, brief: BriefMode) -> Result<retypes::GetJob> {
pub async fn get_job(&self, job: Id, brief: Brief) -> Result<api_types::GetJob> {
self.req(format!("get_job/{job}?brief={brief}")).await
}
pub async fn get_full_job(&self, job: Id) -> Result<retypes::GetJob> {
self.get_job(job, BriefMode::No).await
pub async fn get_full_job(&self, job: Id) -> Result<api_types::GetJob> {
self.get_job(job, Brief::No).await
}
pub async fn get_brief_job(&self, job: Id) -> Result<retypes::GetJob> {
self.get_job(job, BriefMode::Yes).await
pub async fn get_brief_job(&self, job: Id) -> Result<api_types::GetJob> {
self.get_job(job, Brief::Yes).await
}
/// get all available jobs
pub async fn get_jobs(&self) -> Result<retypes::GetJobs> {
pub async fn get_jobs(&self) -> Result<api_types::GetJobs> {
self.req("get_jobs").await
}
}
@ -186,27 +186,27 @@ impl HttpClient {
#[cfg(feature = "panel")]
impl HttpClient {
/// agent listing
pub async fn get_agents(&self, agent: Option<Id>) -> Result<retypes::GetAgents> {
pub async fn get_agents(&self, agent: Option<Id>) -> Result<api_types::GetAgents> {
self.req(format!("get_agents/{}", opt_to_string(agent)))
.await
}
/// update agent
pub async fn update_agent(&self, agent: &Agent) -> Result<retypes::UpdateAgent> {
pub async fn update_agent(&self, agent: &Agent) -> Result<api_types::UpdateAgent> {
self.req_with_payload("update_agent", agent).await
}
/// update job
pub async fn update_job(&self, job: &JobMeta) -> Result<retypes::UpdateJob> {
pub async fn update_job(&self, job: &JobMeta) -> Result<api_types::UpdateJob> {
self.req_with_payload("update_job", job).await
}
/// update result
pub async fn update_result(&self, result: &AssignedJob) -> Result<retypes::UpdateResult> {
pub async fn update_result(&self, result: &AssignedJob) -> Result<api_types::UpdateResult> {
self.req_with_payload("update_result", result).await
}
pub async fn update_payload(&self, payload: &Payload) -> Result<retypes::UpdatePayload> {
pub async fn update_payload(&self, payload: &Payload) -> Result<api_types::UpdatePayload> {
self.req_with_payload("update_payload", payload).await
}
@ -214,21 +214,17 @@ impl HttpClient {
pub async fn upload_jobs(
&self,
jobs: impl IntoIterator<Item = &Job>,
) -> Result<retypes::UploadJobs> {
) -> Result<api_types::UploadJobs> {
self.req_with_payload("upload_jobs", &jobs.into_iter().collect::<Vec<_>>())
.await
}
pub async fn upload_payloads(
&self,
payload: impl IntoIterator<Item = &RawPayload>,
) -> Result<retypes::UploadPayloads> {
self.req_with_payload("upload_payloads", &payload.into_iter().collect::<Vec<_>>())
.await
pub async fn upload_payload(&self, payload: &RawPayload) -> Result<api_types::UploadPayloads> {
self.req_with_payload("upload_payload", payload).await
}
/// delete something
pub async fn del(&self, item: Id) -> Result<retypes::Del> {
pub async fn del(&self, item: Id) -> Result<api_types::Del> {
self.req(format!("del/{item}")).await
}
@ -237,7 +233,7 @@ impl HttpClient {
&self,
agent: Id,
job_idents: impl IntoIterator<Item = impl Into<String>>,
) -> Result<retypes::SetJobs> {
) -> Result<api_types::SetJobs> {
self.req_with_payload(
format!("set_jobs/{agent}"),
&job_idents
@ -249,26 +245,26 @@ impl HttpClient {
}
/// get jobs for any agent
pub async fn get_assigned_jobs(&self, agent: Option<Id>) -> Result<retypes::GetAgentJobs> {
pub async fn get_assigned_jobs(&self, agent: Option<Id>) -> Result<api_types::GetAgentJobs> {
self.req(format!("get_assigned_jobs/{}", opt_to_string(agent)))
.await
}
pub async fn get_payloads(&self) -> Result<retypes::GetPayloads> {
pub async fn get_payloads(&self) -> Result<api_types::GetPayloads> {
self.req("get_payloads").await
}
pub async fn get_payload(
&self,
payload: impl AsRef<str>,
brief: BriefMode,
) -> Result<retypes::GetPayload> {
brief: Brief,
) -> Result<api_types::GetPayload> {
let payload = payload.as_ref();
self.req(format!("get_payload/{payload}?brief={brief}"))
.await
}
pub async fn ping(&self) -> Result<retypes::Ping> {
pub async fn ping(&self) -> Result<api_types::Ping> {
self.req("ping").await
}
}

@ -223,7 +223,7 @@ impl TryFrom<RawJob<'_>> for Job {
}
};
raw.meta.payload_id = payload.as_ref().map(|p| p.id);
raw.meta.payload_id = payload.as_ref().map(|p| p.id).or(raw.meta.payload_id);
Ok(Job {
meta: raw.meta.validate()?,

@ -9,7 +9,7 @@ use serde::Deserialize;
use strum::{Display as StrumDisplay, EnumString};
#[derive(Default, Debug, StrumDisplay, EnumString, Deserialize)]
pub enum BriefMode {
pub enum Brief {
Yes,
#[default]
Auto,

@ -129,12 +129,12 @@ impl Payload {
const BUFFER_LEN: usize = 4096;
let mut buffer: [u8; BUFFER_LEN] = [0; BUFFER_LEN];
let mut payload_src = if let Some(data) = &self.data {
Box::new(data.as_slice()) as Box<dyn Read>
let mut payload_src: Box<dyn Read> = if let Some(data) = &self.data {
Box::new(data.as_slice())
} else {
let payload_path = ufs::read_meta(&self.name).context("prep")?.path;
let file = File::open(&payload_path).map_err(|e| ufs::Error::new(e, &payload_path))?;
Box::new(file) as Box<dyn Read>
Box::new(file)
};
let fd = memfd_create(
@ -167,6 +167,7 @@ impl Payload {
}
}
#[cfg(unix)]
fn get_mime_type(path: impl AsRef<Path>) -> Result<String, UError> {
let path = path.as_ref();

@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS jobs (
payload_id UUID,
schedule TEXT,
FOREIGN KEY(payload_id) REFERENCES payloads(id) ON DELETE SET NULL,
FOREIGN KEY(payload_id) REFERENCES payloads(id),
PRIMARY KEY(id)
);

Loading…
Cancel
Save