root
/
unki
1
0
Fork 0
Code Issues 7 Pull Requests 1 Packages Projects Releases Wiki Activity

add stats integration test

Browse Source
pull/12/head
plazmoid 11 months ago
parent 2816e72cbd
commit 810d942721
17 changed files with 298 additions and 278 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 265
      Cargo.lock
  2. 82
      bin/u_agent/src/lib.rs
  3. 41
      bin/u_server/src/handlers.rs
  4. 2
      integration-tests/docker-compose.yml
  5. 51
      integration-tests/tests/integration_tests/behaviour.rs
  6. 1
      lib/u_lib/Cargo.toml
  7. 4
      lib/u_lib/src/jobs.rs
  8. 10
      lib/u_lib/src/models/jobs/assigned.rs
  9. 4
      lib/u_lib/src/models/mod.rs
  10. 7
      lib/u_lib/src/models/schema.rs
  11. 4
      lib/u_lib/src/models/stats.rs
  12. 20
      lib/u_lib/src/scheduler/entry.rs
  13. 52
      lib/u_lib/src/scheduler/mod.rs
  14. 14
      lib/u_lib/src/u_runner.rs
  15. 10
      lib/u_lib/src/ufs/index.rs
  16. 7
      lib/u_lib/src/ufs/mod.rs
  17. 2
      migrations/2020-10-24-111622_create_all/up.sql

265
Cargo.lock generated
Unescape View File

@ -80,7 +80,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
dependencies = [ dependencies = [
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]
@ -193,7 +193,7 @@ dependencies = [
"actix-router", "actix-router",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]
@ -292,7 +292,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]
@ -376,9 +376,9 @@ dependencies = [
[[package]] [[package]]
name = "brotli-decompressor" name = "brotli-decompressor"
version = "2.5.0" version = "2.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448" checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f"
dependencies = [ dependencies = [
"alloc-no-stdlib", "alloc-no-stdlib",
"alloc-stdlib", "alloc-stdlib",
@ -404,20 +404,19 @@ checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
[[package]] [[package]]
name = "bytestring" name = "bytestring"
version = "1.3.0" version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae" checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72"
dependencies = [ dependencies = [
"bytes", "bytes",
] ]
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.0.83" version = "1.0.84"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" checksum = "0f8e7c90afad890484a21653d08b6e209ae34770fb5ee298f9c699fcc1e5c856"
dependencies = [ dependencies = [
"jobserver",
"libc", "libc",
] ]
@ -498,9 +497,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
[[package]] [[package]]
name = "cpufeatures" name = "cpufeatures"
version = "0.2.10" version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fbc60abd742b35f2492f808e1abbb83d45f72db402e14c55057edc9c7b1e9e4" checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0"
dependencies = [ dependencies = [
"libc", "libc",
] ]
@ -585,7 +584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37e366bff8cd32dd8754b0991fb66b279dc48f598c3a18914852a6673deef583" checksum = "37e366bff8cd32dd8754b0991fb66b279dc48f598c3a18914852a6673deef583"
dependencies = [ dependencies = [
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]
@ -691,7 +690,7 @@ dependencies = [
"heck 0.4.1", "heck 0.4.1",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]
@ -703,7 +702,7 @@ dependencies = [
"diesel_table_macro_syntax", "diesel_table_macro_syntax",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]
@ -723,7 +722,7 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5"
dependencies = [ dependencies = [
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]
@ -791,9 +790,9 @@ dependencies = [
[[package]] [[package]]
name = "errno" name = "errno"
version = "0.3.5" version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" checksum = "7c18ee0ed65a5f1f81cac6b1d213b69c35fa47d4252ad41f1486dbd8226fe36e"
dependencies = [ dependencies = [
"libc", "libc",
"windows-sys", "windows-sys",
@ -857,9 +856,9 @@ dependencies = [
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.28" version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -872,9 +871,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-channel" name = "futures-channel"
version = "0.3.28" version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@ -882,15 +881,15 @@ dependencies = [
[[package]] [[package]]
name = "futures-core" name = "futures-core"
version = "0.3.28" version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c"
[[package]] [[package]]
name = "futures-executor" name = "futures-executor"
version = "0.3.28" version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-task", "futures-task",
@ -899,32 +898,32 @@ dependencies = [
[[package]] [[package]]
name = "futures-io" name = "futures-io"
version = "0.3.28" version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa"
[[package]] [[package]]
name = "futures-macro" name = "futures-macro"
version = "0.3.28" version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]
name = "futures-sink" name = "futures-sink"
version = "0.3.28" version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817"
[[package]] [[package]]
name = "futures-task" name = "futures-task"
version = "0.3.28" version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2"
[[package]] [[package]]
name = "futures-timer" name = "futures-timer"
@ -934,9 +933,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
[[package]] [[package]]
name = "futures-util" name = "futures-util"
version = "0.3.28" version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -962,9 +961,9 @@ dependencies = [
[[package]] [[package]]
name = "getrandom" name = "getrandom"
version = "0.2.10" version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"libc", "libc",
@ -1074,11 +1073,17 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]] [[package]]
name = "http" name = "http"
version = "0.2.9" version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb"
dependencies = [ dependencies = [
"bytes", "bytes",
"fnv", "fnv",
@ -1224,9 +1229,9 @@ dependencies = [
[[package]] [[package]]
name = "indexmap" name = "indexmap"
version = "2.0.2" version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
dependencies = [ dependencies = [
"equivalent", "equivalent",
"hashbrown 0.14.2", "hashbrown 0.14.2",
@ -1261,20 +1266,11 @@ version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
[[package]]
name = "jobserver"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.64" version = "0.3.65"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8"
dependencies = [ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
@ -1302,9 +1298,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.149" version = "0.2.150"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
[[package]] [[package]]
name = "libflate" name = "libflate"
@ -1328,15 +1324,15 @@ dependencies = [
[[package]] [[package]]
name = "linux-raw-sys" name = "linux-raw-sys"
version = "0.4.10" version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829"
[[package]] [[package]]
name = "local-channel" name = "local-channel"
version = "0.1.4" version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0a493488de5f18c8ffcba89eebb8532ffc562dc400490eb65b84893fae0b178" checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@ -1345,9 +1341,9 @@ dependencies = [
[[package]] [[package]]
name = "local-waker" name = "local-waker"
version = "0.1.3" version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487"
[[package]] [[package]]
name = "lock_api" name = "lock_api"
@ -1578,9 +1574,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
[[package]] [[package]]
name = "openssl" name = "openssl"
version = "0.10.57" version = "0.10.59"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33"
dependencies = [ dependencies = [
"bitflags 2.4.1", "bitflags 2.4.1",
"cfg-if 1.0.0", "cfg-if 1.0.0",
@ -1599,7 +1595,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]
@ -1610,9 +1606,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]] [[package]]
name = "openssl-sys" name = "openssl-sys"
version = "0.9.93" version = "0.9.95"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9"
dependencies = [ dependencies = [
"cc", "cc",
"libc", "libc",
@ -1644,7 +1640,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"libc", "libc",
"redox_syscall 0.4.1", "redox_syscall",
"smallvec", "smallvec",
"windows-targets", "windows-targets",
] ]
@ -1678,7 +1674,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]
@ -1701,9 +1697,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
[[package]] [[package]]
name = "platforms" name = "platforms"
version = "3.1.2" version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4503fa043bf02cee09a9582e9554b4c6403b2ef55e4612e96561d294419429f8" checksum = "14e6ab3f592e6fb464fc9712d8d6e6912de6473954635fd76a589d832cffcbb0"
[[package]] [[package]]
name = "powerfmt" name = "powerfmt"
@ -1824,15 +1820,6 @@ dependencies = [
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]]
name = "redox_syscall"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
dependencies = [
"bitflags 1.3.2",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.4.1" version = "0.4.1"
@ -2010,7 +1997,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"rust-embed-utils", "rust-embed-utils",
"syn 2.0.38", "syn 2.0.39",
"walkdir", "walkdir",
] ]
@ -2041,12 +2028,12 @@ dependencies = [
[[package]] [[package]]
name = "rustix" name = "rustix"
version = "0.38.20" version = "0.38.22"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0" checksum = "80109a168d9bc0c7f483083244543a6eb0dba02295d33ca268145e6190d6df0c"
dependencies = [ dependencies = [
"bitflags 2.4.1", "bitflags 2.4.1",
"errno 0.3.5", "errno 0.3.6",
"libc", "libc",
"linux-raw-sys", "linux-raw-sys",
"windows-sys", "windows-sys",
@ -2066,9 +2053,9 @@ dependencies = [
[[package]] [[package]]
name = "rustls-pemfile" name = "rustls-pemfile"
version = "1.0.3" version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
dependencies = [ dependencies = [
"base64", "base64",
] ]
@ -2160,29 +2147,29 @@ checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.189" version = "1.0.192"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001"
dependencies = [ dependencies = [
"serde_derive", "serde_derive",
] ]
[[package]] [[package]]
name = "serde_derive" name = "serde_derive"
version = "1.0.189" version = "1.0.192"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]
name = "serde_json" name = "serde_json"
version = "1.0.107" version = "1.0.108"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
dependencies = [ dependencies = [
"itoa", "itoa",
"ryu", "ryu",
@ -2291,9 +2278,9 @@ dependencies = [
[[package]] [[package]]
name = "smallvec" name = "smallvec"
version = "1.11.1" version = "1.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970"
[[package]] [[package]]
name = "socket2" name = "socket2"
@ -2406,9 +2393,9 @@ dependencies = [
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.38" version = "2.0.39"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -2453,13 +2440,13 @@ dependencies = [
[[package]] [[package]]
name = "tempfile" name = "tempfile"
version = "3.8.0" version = "3.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"fastrand", "fastrand",
"redox_syscall 0.3.5", "redox_syscall",
"rustix", "rustix",
"windows-sys", "windows-sys",
] ]
@ -2549,9 +2536,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "1.33.0" version = "1.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9"
dependencies = [ dependencies = [
"backtrace", "backtrace",
"bytes", "bytes",
@ -2568,13 +2555,13 @@ dependencies = [
[[package]] [[package]]
name = "tokio-macros" name = "tokio-macros"
version = "2.1.0" version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]
@ -2622,9 +2609,9 @@ dependencies = [
[[package]] [[package]]
name = "tokio-util" name = "tokio-util"
version = "0.7.9" version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15"
dependencies = [ dependencies = [
"bytes", "bytes",
"futures-core", "futures-core",
@ -2661,7 +2648,7 @@ version = "0.19.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
dependencies = [ dependencies = [
"indexmap 2.0.2", "indexmap 2.1.0",
"serde", "serde",
"serde_spanned", "serde_spanned",
"toml_datetime", "toml_datetime",
@ -2688,11 +2675,12 @@ dependencies = [
[[package]] [[package]]
name = "tracing-appender" name = "tracing-appender"
version = "0.2.2" version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf"
dependencies = [ dependencies = [
"crossbeam-channel", "crossbeam-channel",
"thiserror",
"time", "time",
"tracing-subscriber", "tracing-subscriber",
] ]
@ -2705,7 +2693,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]
@ -2720,9 +2708,9 @@ dependencies = [
[[package]] [[package]]
name = "tracing-log" name = "tracing-log"
version = "0.1.4" version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [ dependencies = [
"log", "log",
"once_cell", "once_cell",
@ -2731,9 +2719,9 @@ dependencies = [
[[package]] [[package]]
name = "tracing-subscriber" name = "tracing-subscriber"
version = "0.3.17" version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
dependencies = [ dependencies = [
"matchers", "matchers",
"nu-ansi-term", "nu-ansi-term",
@ -2805,6 +2793,7 @@ dependencies = [
"dotenv", "dotenv",
"envy", "envy",
"guess_host_triple", "guess_host_triple",
"hex",
"lazy_static", "lazy_static",
"libc", "libc",
"nix", "nix",
@ -3046,9 +3035,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]] [[package]]
name = "wasm-bindgen" name = "wasm-bindgen"
version = "0.2.87" version = "0.2.88"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"wasm-bindgen-macro", "wasm-bindgen-macro",
@ -3056,24 +3045,24 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-backend" name = "wasm-bindgen-backend"
version = "0.2.87" version = "0.2.88"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217"
dependencies = [ dependencies = [
"bumpalo", "bumpalo",
"log", "log",
"once_cell", "once_cell",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
[[package]] [[package]]
name = "wasm-bindgen-futures" name = "wasm-bindgen-futures"
version = "0.4.37" version = "0.4.38"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"js-sys", "js-sys",
@ -3083,9 +3072,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-macro" name = "wasm-bindgen-macro"
version = "0.2.87" version = "0.2.88"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2"
dependencies = [ dependencies = [
"quote", "quote",
"wasm-bindgen-macro-support", "wasm-bindgen-macro-support",
@ -3093,28 +3082,28 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-macro-support" name = "wasm-bindgen-macro-support"
version = "0.2.87" version = "0.2.88"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
"wasm-bindgen-backend", "wasm-bindgen-backend",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
[[package]] [[package]]
name = "wasm-bindgen-shared" name = "wasm-bindgen-shared"
version = "0.2.87" version = "0.2.88"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b"
[[package]] [[package]]
name = "web-sys" name = "web-sys"
version = "0.3.64" version = "0.3.65"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85"
dependencies = [ dependencies = [
"js-sys", "js-sys",
"wasm-bindgen", "wasm-bindgen",
@ -3228,9 +3217,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]] [[package]]
name = "winnow" name = "winnow"
version = "0.5.17" version = "0.5.19"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c" checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]
@ -3247,22 +3236,22 @@ dependencies = [
[[package]] [[package]]
name = "zerocopy" name = "zerocopy"
version = "0.7.14" version = "0.7.25"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69c48d63854f77746c68a5fbb4aa17f3997ece1cb301689a257af8cb80610d21" checksum = "8cd369a67c0edfef15010f980c3cbe45d7f651deac2cd67ce097cd801de16557"
dependencies = [ dependencies = [
"zerocopy-derive", "zerocopy-derive",
] ]
[[package]] [[package]]
name = "zerocopy-derive" name = "zerocopy-derive"
version = "0.7.14" version = "0.7.25"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c258c1040279e4f88763a113de72ce32dde2d50e2a94573f15dd534cea36a16d" checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.38", "syn 2.0.39",
] ]
[[package]] [[package]]

82
bin/u_agent/src/lib.rs
Unescape View File

@ -17,56 +17,54 @@ use u_lib::{
}; };
async fn process_request(assigned_jobs: Vec<AssignedJobById>, client: &HttpClient) { async fn process_request(assigned_jobs: Vec<AssignedJobById>, client: &HttpClient) {
if !assigned_jobs.is_empty() { for asgn_job in assigned_jobs {
for asgn_job in assigned_jobs { if !JobCache::contains(asgn_job.job_id) {
if !JobCache::contains(asgn_job.job_id) { info!("Fetching job: {}", &asgn_job.job_id);
info!("Fetching job: {}", &asgn_job.job_id); let mut fetched_job = loop {
let mut fetched_job = loop { //todo: use payload cache
//todo: use payload cache match client.get_full_job(asgn_job.job_id).await {
match client.get_full_job(asgn_job.job_id).await { Ok(result) => break result,
Ok(result) => break result, Err(err) => {
Err(err) => { debug!("{:?} \nretrying...", err);
debug!("{:?} \nretrying...", err); sleep(AGENT_ITERATION_INTERVAL).await;
sleep(AGENT_ITERATION_INTERVAL).await;
}
}
};
if let Some(payload) = &mut fetched_job.payload {
if let Err(e) = payload.maybe_split_payload() {
ErrChan::send(e, "pay").await;
continue;
} }
} }
JobCache::insert(fetched_job); };
if let Some(payload) = &mut fetched_job.payload {
if let Err(e) = payload.maybe_split_payload() {
ErrChan::send(e, "pay").await;
continue;
}
} }
JobCache::insert(fetched_job);
}
let job = match JobCache::get(asgn_job.job_id).as_deref() { let job = match JobCache::get(asgn_job.job_id).as_deref() {
Some(job) => job.clone(), Some(job) => job.clone(),
None => continue, None => continue,
}; };
info!("Scheduling job {}", job.meta.id.to_string());
info!("Scheduling job {}", job.meta.id.to_string()); let schedule = match job.meta.schedule.clone() {
Some(sched) => {
let schedule = match job.meta.schedule.clone() { if sched.is_empty() {
Some(sched) => { None
if sched.is_empty() { } else {
None match sched.as_str().try_into() {
} else { Ok(s) => Some(s),
match sched.as_str().try_into() { Err(err) => {
Ok(s) => Some(s), ErrChan::send(err, "sch").await;
Err(err) => { continue;
ErrChan::send(err, "sch").await;
continue;
}
} }
} }
} }
None => None, }
}; None => None,
SCHEDULER };
.add_job(schedule, PreparedJob { job, ids: asgn_job }) SCHEDULER
.await; .add_job(schedule, PreparedJob { job, ids: asgn_job })
} .await;
} }
} }

41
bin/u_server/src/handlers.rs
Unescape View File

@ -249,7 +249,10 @@ impl Endpoints {
Reportable::Assigned(mut result) => { Reportable::Assigned(mut result) => {
let result_agent_id = &result.agent_id; let result_agent_id = &result.agent_id;
if agent_id != *result_agent_id { if agent_id != *result_agent_id {
warn!("Agent ids are not equal! actual id: {agent_id}, id from job: {result_agent_id}"); warn!(
"Agent ids are not equal! actual id: {agent_id}, \
id from job: {result_agent_id}"
);
continue; continue;
} }
result.touch(); result.touch();
@ -260,24 +263,21 @@ impl Endpoints {
JobType::Init => { JobType::Init => {
result.state = JobState::Finished; result.state = JobState::Finished;
match &result.result { let mut agent: Agent = match result.deserialize() {
Some(rbytes) => { Ok(a) => a,
let mut agent: Agent = match serde_json::from_slice(&rbytes) { Err(e) => {
Ok(a) => a, error!(
Err(e) => { "Error deserializing agent \
error!("Error deserializing agent data from {agent_id}: {e}"); data from {agent_id}: {e}"
continue; );
} continue;
}; }
agent.state = AgentState::Active; };
db.upsert_agent(&agent)?; agent.state = AgentState::Active;
} db.upsert_agent(&agent)?;
None => error!("Empty agent data"), }
}}, JobType::Shell => result.state = JobState::Finished,
JobType::Shell => { JobType::Stats => result.state = JobState::Finished,
result.state = JobState::Finished
},
JobType::Stats => (),
JobType::Terminate => (), JobType::Terminate => (),
JobType::Update => (), JobType::Update => (),
} }
@ -287,7 +287,8 @@ impl Endpoints {
error!("agent {agent_id} reported: {e}"); error!("agent {agent_id} reported: {e}");
} }
Reportable::Dummy => (), Reportable::Dummy => (),
}} }
}
Ok(()) Ok(())
}) })
.await .await

2
integration-tests/docker-compose.yml
Unescape View File

@ -40,7 +40,7 @@ services:
networks: networks:
- u_net - u_net
ports: ports:
- 54321:5432 - 5432:5432
env_file: env_file:
- ../.env - ../.env
- ../.env.private - ../.env.private

51
integration-tests/tests/integration_tests/behaviour.rs
Unescape View File

@ -16,7 +16,7 @@ async fn registration(registered_agent: &RegisteredAgent) {
} }
#[tokio::test] #[tokio::test]
async fn setup_tasks() { async fn setup_jobs() {
let agents: Vec<Agent> = Panel::check_output("agents read"); let agents: Vec<Agent> = Panel::check_output("agents read");
let agent_id = agents[0].id; let agent_id = agents[0].id;
let job_alias = "passwd_contents"; let job_alias = "passwd_contents";
@ -88,3 +88,52 @@ async fn large_payload() {
}) })
.await; .await;
} }
#[tokio::test]
async fn gather_stats() {
let agent = &Panel::check_output::<Vec<Agent>>("agents read")[0];
let job_alias = "stats";
let job = RawJob::default()
.with_type(JobType::Stats)
.with_alias(job_alias)
.with_target_platforms(&agent.platform)
.try_into_job()
.unwrap();
let job_id = job.meta.id;
let agent_id = agent.id;
Panel::check_status(["jobs", "create", &to_string(&RawJob::from(job)).unwrap()]);
let assigned = AssignedJobById {
agent_id,
job_id,
..Default::default()
};
Panel::check_status(["map", "create", &to_string(&[assigned]).unwrap()]);
let result = retry_with_interval(5, AGENT_ITERATION_INTERVAL, || {
let result =
Panel::check_output::<Vec<AssignedJob>>(["map", "read", &job_id.to_string()]).remove(0);
if result.state == JobState::Finished {
Ok(result)
} else {
Err("job didn't finish")
}
})
.await;
let stats = result.deserialize::<Stats>().unwrap();
assert_eq!(stats.agent_id, agent_id);
assert!(stats.cached_jobs.contains(&job_id));
assert!(stats
.scheduled_jobs
.iter()
.find(|j| j.job_ident == "error_reporting")
.is_some());
assert!(stats
.scheduled_jobs
.iter()
.find(|j| j.job_ident == "agent_loop")
.is_some())
}

1
lib/u_lib/Cargo.toml
Unescape View File

@ -34,6 +34,7 @@ parking_lot = "0.12.1"
bincode = "1.3.3" bincode = "1.3.3"
sha3 = "0.10.7" sha3 = "0.10.7"
cron = "0.12.0" cron = "0.12.0"
hex = "0.4.3"
[target.'cfg(unix)'.dependencies] [target.'cfg(unix)'.dependencies]
daemonize = "0.5" daemonize = "0.5"

4
lib/u_lib/src/jobs.rs
Unescape View File

@ -2,9 +2,7 @@ use crate::{
cache::JobCache, cache::JobCache,
combined_result::CombinedResult, combined_result::CombinedResult,
config::get_self_id, config::get_self_id,
models::{ models::{Agent, AssignedJob, AssignedJobById, Job, JobType, PreparedJob, RawJob, Stats},
stats::Stats, Agent, AssignedJob, AssignedJobById, Job, JobType, PreparedJob, RawJob,
},
proc_output::ProcOutput, proc_output::ProcOutput,
scheduler::SCHEDULER, scheduler::SCHEDULER,
u_runner::{ExecResult, IdentifiableFuture, URunner}, u_runner::{ExecResult, IdentifiableFuture, URunner},

10
lib/u_lib/src/models/jobs/assigned.rs
Unescape View File

@ -6,10 +6,11 @@ use crate::{
conv::{bytes_to_string_truncated, systime_to_string}, conv::{bytes_to_string_truncated, systime_to_string},
models::Job, models::Job,
types::Id, types::Id,
UError, UResult,
}; };
#[cfg(feature = "server")] #[cfg(feature = "server")]
use diesel::{Identifiable, Insertable, Queryable}; use diesel::{Identifiable, Insertable, Queryable};
use serde::{Deserialize, Serialize}; use serde::{de::DeserializeOwned, Deserialize, Serialize};
use std::{borrow::Cow, fmt::Debug, time::SystemTime}; use std::{borrow::Cow, fmt::Debug, time::SystemTime};
/// A job assigned to some agents, stores execution state and result /// A job assigned to some agents, stores execution state and result
@ -145,6 +146,13 @@ impl AssignedJob {
String::from_utf8_lossy(self.to_raw_result()) String::from_utf8_lossy(self.to_raw_result())
} }
pub fn deserialize<T: DeserializeOwned>(&self) -> UResult<T> {
let body = self.result.as_deref().unwrap_or_default();
serde_json::from_slice(body).map_err(|e| {
UError::DeserializeError(e.to_string(), bytes_to_string_truncated(body, 1024))
})
}
pub fn set_result(&mut self, result: &impl Serialize) { pub fn set_result(&mut self, result: &impl Serialize) {
self.result = Some(serde_json::to_vec(result).unwrap()); self.result = Some(serde_json::to_vec(result).unwrap());
} }

4
lib/u_lib/src/models/mod.rs
Unescape View File

@ -3,9 +3,9 @@ mod jobs;
mod payload; mod payload;
#[cfg(feature = "server")] #[cfg(feature = "server")]
pub mod schema; pub mod schema;
pub mod stats; mod stats;
pub use crate::models::{agent::*, jobs::*, payload::*}; pub use crate::models::{agent::*, jobs::*, payload::*, stats::*};
use serde::Deserialize; use serde::Deserialize;
use strum::{Display as StrumDisplay, EnumString}; use strum::{Display as StrumDisplay, EnumString};

7
lib/u_lib/src/models/schema.rs
Unescape View File

@ -86,4 +86,9 @@ diesel::joinable!(jobs -> payloads (payload_id));
diesel::joinable!(results -> agents (agent_id)); diesel::joinable!(results -> agents (agent_id));
diesel::joinable!(results -> jobs (job_id)); diesel::joinable!(results -> jobs (job_id));
diesel::allow_tables_to_appear_in_same_query!(agents, jobs, payloads, results,); diesel::allow_tables_to_appear_in_same_query!(
agents,
jobs,
payloads,
results,
);

4
lib/u_lib/src/models/stats.rs
Unescape View File

@ -1,8 +1,8 @@
use serde::Serialize; use serde::{Deserialize, Serialize};
use crate::{messaging::AsMsg, scheduler::EntryStat, types::Id, ufs::IndexFileMeta}; use crate::{messaging::AsMsg, scheduler::EntryStat, types::Id, ufs::IndexFileMeta};
#[derive(Debug, Clone, Serialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Stats { pub struct Stats {
pub agent_id: Id, pub agent_id: Id,
pub running_jobs: Vec<String>, pub running_jobs: Vec<String>,

20
lib/u_lib/src/scheduler/entry.rs
Unescape View File

@ -1,7 +1,7 @@
use crate::{models::PreparedJob, types::Id, u_runner::IdentifiableFuture}; use crate::{models::PreparedJob, types::Id, u_runner::IdentifiableFuture};
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use core::fmt; use core::fmt;
use serde::Serialize; use serde::{Deserialize, Serialize};
use std::sync::Arc; use std::sync::Arc;
pub trait SchedulerJob { pub trait SchedulerJob {
@ -21,16 +21,16 @@ where
#[derive(Clone)] #[derive(Clone)]
pub struct Entry { pub struct Entry {
pub id: Id, pub entry_id: Id,
pub schedule: Option<cron::Schedule>, pub schedule: Option<cron::Schedule>,
pub next: Option<DateTime<Utc>>, pub next: Option<DateTime<Utc>>,
pub runnable: EntryType, pub runnable: EntryType,
} }
#[derive(Clone, Debug, Serialize)] #[derive(Clone, Debug, Serialize, Deserialize)]
pub struct EntryStat { pub struct EntryStat {
pub id: Id, pub entry_id: Id,
pub job_id: String, pub job_ident: String,
pub schedule: String, pub schedule: String,
pub next: Option<DateTime<Utc>>, pub next: Option<DateTime<Utc>>,
} }
@ -66,14 +66,14 @@ impl Entry {
} }
pub fn as_stat(&self) -> EntryStat { pub fn as_stat(&self) -> EntryStat {
let job_id = match &self.runnable { let job_ident = match &self.runnable {
EntryType::URunner(entry) => entry.job.meta.id.to_string(), EntryType::URunner(entry) => entry.job.meta.id.to_string(),
EntryType::Common(entry) => entry.call().job_id.to_string(), EntryType::Common(entry) => entry.call().job_ident.to_string(),
}; };
EntryStat { EntryStat {
id: self.id, entry_id: self.entry_id,
job_id, job_ident,
schedule: self schedule: self
.schedule .schedule
.as_ref() .as_ref()
@ -86,6 +86,6 @@ impl Entry {
impl fmt::Debug for Entry { impl fmt::Debug for Entry {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{} {:?} {:?}", self.id, self.schedule, self.next) write!(f, "{} {:?} {:?}", self.entry_id, self.schedule, self.next)
} }
} }

52
lib/u_lib/src/scheduler/mod.rs
Unescape View File

@ -8,7 +8,6 @@ use cron::Schedule;
use entry::Entry; use entry::Entry;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use tokio::sync::Mutex; use tokio::sync::Mutex;
use tokio::time::timeout;
use uuid::Uuid; use uuid::Uuid;
use crate::jobs::AnonymousJobBatch; use crate::jobs::AnonymousJobBatch;
@ -59,23 +58,23 @@ impl AsyncScheduler {
for entry in &mut *entries { for entry in &mut *entries {
match entry.next.as_ref() { match entry.next.as_ref() {
Some(next) => { Some(next) => {
if next.gt(&Utc::now()) {
break;
}
match &entry.runnable { match &entry.runnable {
EntryType::Common(runnable) => { EntryType::Common(runnable) => {
let cancel_timeout = let fut = runnable.call();
next.timestamp_millis() - Utc::now().timestamp_millis(); tokio::spawn(fut);
let cancel_timeout = Duration::from_millis(cancel_timeout as u64); }
EntryType::URunner(runnable) => {
tokio::spawn(timeout(cancel_timeout, runnable.call())); debug!("starting assigned job {:?}", runnable.ids);
job_batch.push(runnable.clone())
} }
EntryType::URunner(runnable) => job_batch.push(runnable.clone()),
} }
entry.set_next_run_time(); entry.set_next_run_time();
} }
None => {} None => {}
} }
if entry.next.as_ref().unwrap().gt(&Utc::now()) {
break;
}
} }
AnonymousJobBatch::from_prepared_jobs(job_batch) AnonymousJobBatch::from_prepared_jobs(job_batch)
@ -90,7 +89,7 @@ impl AsyncScheduler {
let entry_id = Uuid::new_v4(); let entry_id = Uuid::new_v4();
let runnable = runnable.into(); let runnable = runnable.into();
let mut entry = Entry { let mut entry = Entry {
id: entry_id, entry_id,
schedule, schedule,
next: None, next: None,
runnable, runnable,
@ -103,7 +102,7 @@ impl AsyncScheduler {
} }
pub async fn del_job(&self, entry_id: Id) { pub async fn del_job(&self, entry_id: Id) {
self.entries.lock().await.retain(|e| e.id != entry_id); self.entries.lock().await.retain(|e| e.entry_id != entry_id);
} }
pub async fn stats(&self) -> Vec<EntryStat> { pub async fn stats(&self) -> Vec<EntryStat> {
@ -176,33 +175,4 @@ mod tests {
assert_eq!(*v.lock().await, 2); assert_eq!(*v.lock().await, 2);
} }
/// Run task every second.
/// Task will be cancelled if not completed until next run
/// (and this task lasts 2 secs),
/// so the counter value should still be 0
#[tokio::test(flavor = "multi_thread")]
async fn cancellation() {
let counter = Arc::new(Mutex::new(0));
let scheduler = AsyncScheduler::new();
{
let counter = counter.clone();
scheduler
.add_job(Some("*/1 * * * * * *".try_into().unwrap()), move || {
let counter = counter.clone();
IdentifiableFuture::from_fut_with_ident("testf", async move {
sleep(Duration::from_secs(2)).await;
*counter.lock().await += 1;
})
})
.await;
}
scheduler.start().await;
sleep(Duration::from_secs(3)).await;
assert_eq!(*counter.lock().await, 0);
}
} }

14
lib/u_lib/src/u_runner.rs
Unescape View File

@ -24,18 +24,18 @@ lazy_static! {
} }
pub struct IdentifiableFuture<R> { pub struct IdentifiableFuture<R> {
pub job_id: String, pub job_ident: String,
fut: Pin<Box<dyn Future<Output = R> + Send + Sync + 'static>>, fut: Pin<Box<dyn Future<Output = R> + Send + Sync + 'static>>,
} }
impl<R> IdentifiableFuture<R> { impl<R> IdentifiableFuture<R> {
pub fn from_fut_with_ident( pub fn from_fut_with_ident(
job_id: impl Into<String>, job_ident: impl Into<String>,
job: impl Future<Output = R> + Send + Sync + 'static, job: impl Future<Output = R> + Send + Sync + 'static,
) -> Self { ) -> Self {
Self { Self {
fut: Box::pin(job), fut: Box::pin(job),
job_id: job_id.into(), job_ident: job_ident.into(),
} }
} }
} }
@ -49,7 +49,7 @@ impl<R> Future for IdentifiableFuture<R> {
} }
struct JoinInfo { struct JoinInfo {
job_id: String, job_ident: String,
handle: JoinHandle<JoinHandle<ExecResult>>, handle: JoinHandle<JoinHandle<ExecResult>>,
completed: bool, completed: bool,
// collectable: bool, // indicates if future can be popped from pool via pop_task_if_completed // collectable: bool, // indicates if future can be popped from pool via pop_task_if_completed
@ -91,7 +91,7 @@ impl URunner {
let handle = Handle::current(); let handle = Handle::current();
let fid = Uuid::new_v4(); let fid = Uuid::new_v4();
let tx = get_sender(); let tx = get_sender();
let job_id = executable.job_id.clone(); let job_id = executable.job_ident.clone();
self.fids.push(fid); self.fids.push(fid);
let job_wrapper = async move { let job_wrapper = async move {
debug!("inside wrapper (started): {}", fid); debug!("inside wrapper (started): {}", fid);
@ -100,7 +100,7 @@ impl URunner {
result result
}; };
let handler = JoinInfo { let handler = JoinInfo {
job_id, job_ident: job_id,
handle: spawn_blocking(move || handle.spawn(job_wrapper)), handle: spawn_blocking(move || handle.spawn(job_wrapper)),
completed: false, completed: false,
}; };
@ -155,7 +155,7 @@ impl URunner {
.lock() .lock()
.await .await
.values() .values()
.map(|v| v.job_id.clone()) .map(|v| v.job_ident.clone())
.collect() .collect()
} }

10
lib/u_lib/src/ufs/index.rs
Unescape View File

@ -34,7 +34,7 @@ static INDEX: Lazy<Mutex<Index>> = Lazy::new(|| {
pub struct IndexFileMeta { pub struct IndexFileMeta {
extension: Option<OsString>, extension: Option<OsString>,
pub external: bool, // if file is present before adding to index pub external: bool, // if file is present before adding to index
pub hash: Vec<u8>, pub hashsum: String,
pub path: PathBuf, // actual file path pub path: PathBuf, // actual file path
pub size: u64, pub size: u64,
} }
@ -42,7 +42,7 @@ pub struct IndexFileMeta {
impl IndexFileMeta { impl IndexFileMeta {
pub fn new( pub fn new(
full_path: impl Into<PathBuf>, full_path: impl Into<PathBuf>,
hash: Vec<u8>, hashsum: String,
external: bool, external: bool,
) -> Result<Self, Error> { ) -> Result<Self, Error> {
let full_path: PathBuf = full_path.into(); let full_path: PathBuf = full_path.into();
@ -53,7 +53,7 @@ impl IndexFileMeta {
path: full_path, path: full_path,
extension, extension,
external, external,
hash, hashsum,
size, size,
}) })
} }
@ -96,14 +96,14 @@ pub fn get(name: impl AsRef<str>) -> Option<IndexFileMeta> {
index.get(name.as_ref()).cloned() index.get(name.as_ref()).cloned()
} }
pub fn get_by_hash(hash: impl AsRef<[u8]>) -> Option<(String, IndexFileMeta)> { pub fn get_by_hash(hash: impl AsRef<str>) -> Option<(String, IndexFileMeta)> {
let mut index = INDEX.lock(); let mut index = INDEX.lock();
sync::deleted(&mut index); sync::deleted(&mut index);
index index
.iter() .iter()
.find(|(_name, meta)| meta.hash == hash.as_ref()) .find(|(_name, meta)| meta.hashsum == hash.as_ref())
.map(|(n, m)| (n.to_owned(), m.clone())) .map(|(n, m)| (n.to_owned(), m.clone()))
} }

7
lib/u_lib/src/ufs/mod.rs
Unescape View File

@ -93,7 +93,7 @@ pub fn edit(name: impl AsRef<str>, data: impl AsRef<[u8]>) -> Result<()> {
let meta = read_meta(&name).context("edit_meta")?; let meta = read_meta(&name).context("edit_meta")?;
let data_hash = hash_data(&data); let data_hash = hash_data(&data);
if meta.hash == data_hash { if meta.hashsum == data_hash {
return Ok(()); return Ok(());
} }
@ -205,10 +205,11 @@ pub fn cleanup() {
} }
*/ */
fn hash_data(data: impl AsRef<[u8]>) -> Vec<u8> { fn hash_data(data: impl AsRef<[u8]>) -> String {
use sha3::{Digest, Sha3_256}; use sha3::{Digest, Sha3_256};
let mut hasher = Sha3_256::new(); let mut hasher = Sha3_256::new();
hasher.update(data); hasher.update(data);
hasher.finalize().to_vec() let hashsum = hasher.finalize().to_vec();
hex::encode(hashsum)
} }

2
migrations/2020-10-24-111622_create_all/up.sql
Unescape View File

@ -1,5 +1,5 @@
CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE TYPE JobType AS ENUM ('shell', 'init', 'python', 'service'); CREATE TYPE JobType AS ENUM ('shell', 'init', 'python', 'service', 'stats');
CREATE TYPE JobState AS ENUM ('queued', 'running', 'finished'); CREATE TYPE JobState AS ENUM ('queued', 'running', 'finished');
CREATE TYPE AgentState AS ENUM ('new', 'active', 'banned'); CREATE TYPE AgentState AS ENUM ('new', 'active', 'banned');

Write Preview
Loading…
Cancel
Save