add stats integration test

pull/12/head
plazmoid 11 months ago
parent 2816e72cbd
commit 810d942721
  1. 265
      Cargo.lock
  2. 2
      bin/u_agent/src/lib.rs
  3. 25
      bin/u_server/src/handlers.rs
  4. 2
      integration-tests/docker-compose.yml
  5. 51
      integration-tests/tests/integration_tests/behaviour.rs
  6. 1
      lib/u_lib/Cargo.toml
  7. 4
      lib/u_lib/src/jobs.rs
  8. 10
      lib/u_lib/src/models/jobs/assigned.rs
  9. 4
      lib/u_lib/src/models/mod.rs
  10. 7
      lib/u_lib/src/models/schema.rs
  11. 4
      lib/u_lib/src/models/stats.rs
  12. 20
      lib/u_lib/src/scheduler/entry.rs
  13. 52
      lib/u_lib/src/scheduler/mod.rs
  14. 14
      lib/u_lib/src/u_runner.rs
  15. 10
      lib/u_lib/src/ufs/index.rs
  16. 7
      lib/u_lib/src/ufs/mod.rs
  17. 2
      migrations/2020-10-24-111622_create_all/up.sql

265
Cargo.lock generated

@ -80,7 +80,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
dependencies = [
"quote",
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]
@ -193,7 +193,7 @@ dependencies = [
"actix-router",
"proc-macro2",
"quote",
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]
@ -292,7 +292,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]
@ -376,9 +376,9 @@ dependencies = [
[[package]]
name = "brotli-decompressor"
version = "2.5.0"
version = "2.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448"
checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@ -404,20 +404,19 @@ checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
[[package]]
name = "bytestring"
version = "1.3.0"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae"
checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72"
dependencies = [
"bytes",
]
[[package]]
name = "cc"
version = "1.0.83"
version = "1.0.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
checksum = "0f8e7c90afad890484a21653d08b6e209ae34770fb5ee298f9c699fcc1e5c856"
dependencies = [
"jobserver",
"libc",
]
@ -498,9 +497,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
[[package]]
name = "cpufeatures"
version = "0.2.10"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fbc60abd742b35f2492f808e1abbb83d45f72db402e14c55057edc9c7b1e9e4"
checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0"
dependencies = [
"libc",
]
@ -585,7 +584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37e366bff8cd32dd8754b0991fb66b279dc48f598c3a18914852a6673deef583"
dependencies = [
"quote",
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]
@ -691,7 +690,7 @@ dependencies = [
"heck 0.4.1",
"proc-macro2",
"quote",
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]
@ -703,7 +702,7 @@ dependencies = [
"diesel_table_macro_syntax",
"proc-macro2",
"quote",
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]
@ -723,7 +722,7 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5"
dependencies = [
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]
@ -791,9 +790,9 @@ dependencies = [
[[package]]
name = "errno"
version = "0.3.5"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860"
checksum = "7c18ee0ed65a5f1f81cac6b1d213b69c35fa47d4252ad41f1486dbd8226fe36e"
dependencies = [
"libc",
"windows-sys",
@ -857,9 +856,9 @@ dependencies = [
[[package]]
name = "futures"
version = "0.3.28"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335"
dependencies = [
"futures-channel",
"futures-core",
@ -872,9 +871,9 @@ dependencies = [
[[package]]
name = "futures-channel"
version = "0.3.28"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb"
dependencies = [
"futures-core",
"futures-sink",
@ -882,15 +881,15 @@ dependencies = [
[[package]]
name = "futures-core"
version = "0.3.28"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c"
[[package]]
name = "futures-executor"
version = "0.3.28"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc"
dependencies = [
"futures-core",
"futures-task",
@ -899,32 +898,32 @@ dependencies = [
[[package]]
name = "futures-io"
version = "0.3.28"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa"
[[package]]
name = "futures-macro"
version = "0.3.28"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]
name = "futures-sink"
version = "0.3.28"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e"
checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817"
[[package]]
name = "futures-task"
version = "0.3.28"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2"
[[package]]
name = "futures-timer"
@ -934,9 +933,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
[[package]]
name = "futures-util"
version = "0.3.28"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104"
dependencies = [
"futures-channel",
"futures-core",
@ -962,9 +961,9 @@ dependencies = [
[[package]]
name = "getrandom"
version = "0.2.10"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
dependencies = [
"cfg-if 1.0.0",
"libc",
@ -1074,11 +1073,17 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "http"
version = "0.2.9"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb"
dependencies = [
"bytes",
"fnv",
@ -1224,9 +1229,9 @@ dependencies = [
[[package]]
name = "indexmap"
version = "2.0.2"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897"
checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
dependencies = [
"equivalent",
"hashbrown 0.14.2",
@ -1261,20 +1266,11 @@ version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
[[package]]
name = "jobserver"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
dependencies = [
"libc",
]
[[package]]
name = "js-sys"
version = "0.3.64"
version = "0.3.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a"
checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8"
dependencies = [
"wasm-bindgen",
]
@ -1302,9 +1298,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.149"
version = "0.2.150"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
[[package]]
name = "libflate"
@ -1328,15 +1324,15 @@ dependencies = [
[[package]]
name = "linux-raw-sys"
version = "0.4.10"
version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829"
[[package]]
name = "local-channel"
version = "0.1.4"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0a493488de5f18c8ffcba89eebb8532ffc562dc400490eb65b84893fae0b178"
checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8"
dependencies = [
"futures-core",
"futures-sink",
@ -1345,9 +1341,9 @@ dependencies = [
[[package]]
name = "local-waker"
version = "0.1.3"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1"
checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487"
[[package]]
name = "lock_api"
@ -1578,9 +1574,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
[[package]]
name = "openssl"
version = "0.10.57"
version = "0.10.59"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c"
checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33"
dependencies = [
"bitflags 2.4.1",
"cfg-if 1.0.0",
@ -1599,7 +1595,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]
@ -1610,9 +1606,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]]
name = "openssl-sys"
version = "0.9.93"
version = "0.9.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d"
checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9"
dependencies = [
"cc",
"libc",
@ -1644,7 +1640,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
dependencies = [
"cfg-if 1.0.0",
"libc",
"redox_syscall 0.4.1",
"redox_syscall",
"smallvec",
"windows-targets",
]
@ -1678,7 +1674,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]
@ -1701,9 +1697,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
[[package]]
name = "platforms"
version = "3.1.2"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4503fa043bf02cee09a9582e9554b4c6403b2ef55e4612e96561d294419429f8"
checksum = "14e6ab3f592e6fb464fc9712d8d6e6912de6473954635fd76a589d832cffcbb0"
[[package]]
name = "powerfmt"
@ -1824,15 +1820,6 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "redox_syscall"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_syscall"
version = "0.4.1"
@ -2010,7 +1997,7 @@ dependencies = [
"proc-macro2",
"quote",
"rust-embed-utils",
"syn 2.0.38",
"syn 2.0.39",
"walkdir",
]
@ -2041,12 +2028,12 @@ dependencies = [
[[package]]
name = "rustix"
version = "0.38.20"
version = "0.38.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0"
checksum = "80109a168d9bc0c7f483083244543a6eb0dba02295d33ca268145e6190d6df0c"
dependencies = [
"bitflags 2.4.1",
"errno 0.3.5",
"errno 0.3.6",
"libc",
"linux-raw-sys",
"windows-sys",
@ -2066,9 +2053,9 @@ dependencies = [
[[package]]
name = "rustls-pemfile"
version = "1.0.3"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2"
checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
dependencies = [
"base64",
]
@ -2160,29 +2147,29 @@ checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
[[package]]
name = "serde"
version = "1.0.189"
version = "1.0.192"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537"
checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.189"
version = "1.0.192"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5"
checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]
name = "serde_json"
version = "1.0.107"
version = "1.0.108"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65"
checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
dependencies = [
"itoa",
"ryu",
@ -2291,9 +2278,9 @@ dependencies = [
[[package]]
name = "smallvec"
version = "1.11.1"
version = "1.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a"
checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970"
[[package]]
name = "socket2"
@ -2406,9 +2393,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.38"
version = "2.0.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b"
checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a"
dependencies = [
"proc-macro2",
"quote",
@ -2453,13 +2440,13 @@ dependencies = [
[[package]]
name = "tempfile"
version = "3.8.0"
version = "3.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5"
dependencies = [
"cfg-if 1.0.0",
"fastrand",
"redox_syscall 0.3.5",
"redox_syscall",
"rustix",
"windows-sys",
]
@ -2549,9 +2536,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.33.0"
version = "1.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653"
checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9"
dependencies = [
"backtrace",
"bytes",
@ -2568,13 +2555,13 @@ dependencies = [
[[package]]
name = "tokio-macros"
version = "2.1.0"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]
@ -2622,9 +2609,9 @@ dependencies = [
[[package]]
name = "tokio-util"
version = "0.7.9"
version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d"
checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15"
dependencies = [
"bytes",
"futures-core",
@ -2661,7 +2648,7 @@ version = "0.19.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
dependencies = [
"indexmap 2.0.2",
"indexmap 2.1.0",
"serde",
"serde_spanned",
"toml_datetime",
@ -2688,11 +2675,12 @@ dependencies = [
[[package]]
name = "tracing-appender"
version = "0.2.2"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e"
checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf"
dependencies = [
"crossbeam-channel",
"thiserror",
"time",
"tracing-subscriber",
]
@ -2705,7 +2693,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]
@ -2720,9 +2708,9 @@ dependencies = [
[[package]]
name = "tracing-log"
version = "0.1.4"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2"
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [
"log",
"once_cell",
@ -2731,9 +2719,9 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
version = "0.3.17"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77"
checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
dependencies = [
"matchers",
"nu-ansi-term",
@ -2805,6 +2793,7 @@ dependencies = [
"dotenv",
"envy",
"guess_host_triple",
"hex",
"lazy_static",
"libc",
"nix",
@ -3046,9 +3035,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.87"
version = "0.2.88"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342"
checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce"
dependencies = [
"cfg-if 1.0.0",
"wasm-bindgen-macro",
@ -3056,24 +3045,24 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.87"
version = "0.2.88"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd"
checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.38",
"syn 2.0.39",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.37"
version = "0.4.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03"
checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02"
dependencies = [
"cfg-if 1.0.0",
"js-sys",
@ -3083,9 +3072,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.87"
version = "0.2.88"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -3093,28 +3082,28 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.87"
version = "0.2.88"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.38",
"syn 2.0.39",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.87"
version = "0.2.88"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b"
[[package]]
name = "web-sys"
version = "0.3.64"
version = "0.3.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b"
checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85"
dependencies = [
"js-sys",
"wasm-bindgen",
@ -3228,9 +3217,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]]
name = "winnow"
version = "0.5.17"
version = "0.5.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c"
checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b"
dependencies = [
"memchr",
]
@ -3247,22 +3236,22 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.7.14"
version = "0.7.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69c48d63854f77746c68a5fbb4aa17f3997ece1cb301689a257af8cb80610d21"
checksum = "8cd369a67c0edfef15010f980c3cbe45d7f651deac2cd67ce097cd801de16557"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.7.14"
version = "0.7.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c258c1040279e4f88763a113de72ce32dde2d50e2a94573f15dd534cea36a16d"
checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.38",
"syn 2.0.39",
]
[[package]]

@ -17,7 +17,6 @@ use u_lib::{
};
async fn process_request(assigned_jobs: Vec<AssignedJobById>, client: &HttpClient) {
if !assigned_jobs.is_empty() {
for asgn_job in assigned_jobs {
if !JobCache::contains(asgn_job.job_id) {
info!("Fetching job: {}", &asgn_job.job_id);
@ -67,7 +66,6 @@ async fn process_request(assigned_jobs: Vec<AssignedJobById>, client: &HttpClien
.add_job(schedule, PreparedJob { job, ids: asgn_job })
.await;
}
}
}
async fn error_reporting(client: HttpClient) {

@ -249,7 +249,10 @@ impl Endpoints {
Reportable::Assigned(mut result) => {
let result_agent_id = &result.agent_id;
if agent_id != *result_agent_id {
warn!("Agent ids are not equal! actual id: {agent_id}, id from job: {result_agent_id}");
warn!(
"Agent ids are not equal! actual id: {agent_id}, \
id from job: {result_agent_id}"
);
continue;
}
result.touch();
@ -260,24 +263,21 @@ impl Endpoints {
JobType::Init => {
result.state = JobState::Finished;
match &result.result {
Some(rbytes) => {
let mut agent: Agent = match serde_json::from_slice(&rbytes) {
let mut agent: Agent = match result.deserialize() {
Ok(a) => a,
Err(e) => {
error!("Error deserializing agent data from {agent_id}: {e}");
error!(
"Error deserializing agent \
data from {agent_id}: {e}"
);
continue;
}
};
agent.state = AgentState::Active;
db.upsert_agent(&agent)?;
}
None => error!("Empty agent data"),
}},
JobType::Shell => {
result.state = JobState::Finished
},
JobType::Stats => (),
JobType::Shell => result.state = JobState::Finished,
JobType::Stats => result.state = JobState::Finished,
JobType::Terminate => (),
JobType::Update => (),
}
@ -287,7 +287,8 @@ impl Endpoints {
error!("agent {agent_id} reported: {e}");
}
Reportable::Dummy => (),
}}
}
}
Ok(())
})
.await

@ -40,7 +40,7 @@ services:
networks:
- u_net
ports:
- 54321:5432
- 5432:5432
env_file:
- ../.env
- ../.env.private

@ -16,7 +16,7 @@ async fn registration(registered_agent: &RegisteredAgent) {
}
#[tokio::test]
async fn setup_tasks() {
async fn setup_jobs() {
let agents: Vec<Agent> = Panel::check_output("agents read");
let agent_id = agents[0].id;
let job_alias = "passwd_contents";
@ -88,3 +88,52 @@ async fn large_payload() {
})
.await;
}
#[tokio::test]
async fn gather_stats() {
let agent = &Panel::check_output::<Vec<Agent>>("agents read")[0];
let job_alias = "stats";
let job = RawJob::default()
.with_type(JobType::Stats)
.with_alias(job_alias)
.with_target_platforms(&agent.platform)
.try_into_job()
.unwrap();
let job_id = job.meta.id;
let agent_id = agent.id;
Panel::check_status(["jobs", "create", &to_string(&RawJob::from(job)).unwrap()]);
let assigned = AssignedJobById {
agent_id,
job_id,
..Default::default()
};
Panel::check_status(["map", "create", &to_string(&[assigned]).unwrap()]);
let result = retry_with_interval(5, AGENT_ITERATION_INTERVAL, || {
let result =
Panel::check_output::<Vec<AssignedJob>>(["map", "read", &job_id.to_string()]).remove(0);
if result.state == JobState::Finished {
Ok(result)
} else {
Err("job didn't finish")
}
})
.await;
let stats = result.deserialize::<Stats>().unwrap();
assert_eq!(stats.agent_id, agent_id);
assert!(stats.cached_jobs.contains(&job_id));
assert!(stats
.scheduled_jobs
.iter()
.find(|j| j.job_ident == "error_reporting")
.is_some());
assert!(stats
.scheduled_jobs
.iter()
.find(|j| j.job_ident == "agent_loop")
.is_some())
}

@ -34,6 +34,7 @@ parking_lot = "0.12.1"
bincode = "1.3.3"
sha3 = "0.10.7"
cron = "0.12.0"
hex = "0.4.3"
[target.'cfg(unix)'.dependencies]
daemonize = "0.5"

@ -2,9 +2,7 @@ use crate::{
cache::JobCache,
combined_result::CombinedResult,
config::get_self_id,
models::{
stats::Stats, Agent, AssignedJob, AssignedJobById, Job, JobType, PreparedJob, RawJob,
},
models::{Agent, AssignedJob, AssignedJobById, Job, JobType, PreparedJob, RawJob, Stats},
proc_output::ProcOutput,
scheduler::SCHEDULER,
u_runner::{ExecResult, IdentifiableFuture, URunner},

@ -6,10 +6,11 @@ use crate::{
conv::{bytes_to_string_truncated, systime_to_string},
models::Job,
types::Id,
UError, UResult,
};
#[cfg(feature = "server")]
use diesel::{Identifiable, Insertable, Queryable};
use serde::{Deserialize, Serialize};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use std::{borrow::Cow, fmt::Debug, time::SystemTime};
/// A job assigned to some agents, stores execution state and result
@ -145,6 +146,13 @@ impl AssignedJob {
String::from_utf8_lossy(self.to_raw_result())
}
pub fn deserialize<T: DeserializeOwned>(&self) -> UResult<T> {
let body = self.result.as_deref().unwrap_or_default();
serde_json::from_slice(body).map_err(|e| {
UError::DeserializeError(e.to_string(), bytes_to_string_truncated(body, 1024))
})
}
pub fn set_result(&mut self, result: &impl Serialize) {
self.result = Some(serde_json::to_vec(result).unwrap());
}

@ -3,9 +3,9 @@ mod jobs;
mod payload;
#[cfg(feature = "server")]
pub mod schema;
pub mod stats;
mod stats;
pub use crate::models::{agent::*, jobs::*, payload::*};
pub use crate::models::{agent::*, jobs::*, payload::*, stats::*};
use serde::Deserialize;
use strum::{Display as StrumDisplay, EnumString};

@ -86,4 +86,9 @@ diesel::joinable!(jobs -> payloads (payload_id));
diesel::joinable!(results -> agents (agent_id));
diesel::joinable!(results -> jobs (job_id));
diesel::allow_tables_to_appear_in_same_query!(agents, jobs, payloads, results,);
diesel::allow_tables_to_appear_in_same_query!(
agents,
jobs,
payloads,
results,
);

@ -1,8 +1,8 @@
use serde::Serialize;
use serde::{Deserialize, Serialize};
use crate::{messaging::AsMsg, scheduler::EntryStat, types::Id, ufs::IndexFileMeta};
#[derive(Debug, Clone, Serialize)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Stats {
pub agent_id: Id,
pub running_jobs: Vec<String>,

@ -1,7 +1,7 @@
use crate::{models::PreparedJob, types::Id, u_runner::IdentifiableFuture};
use chrono::{DateTime, Utc};
use core::fmt;
use serde::Serialize;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
pub trait SchedulerJob {
@ -21,16 +21,16 @@ where
#[derive(Clone)]
pub struct Entry {
pub id: Id,
pub entry_id: Id,
pub schedule: Option<cron::Schedule>,
pub next: Option<DateTime<Utc>>,
pub runnable: EntryType,
}
#[derive(Clone, Debug, Serialize)]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct EntryStat {
pub id: Id,
pub job_id: String,
pub entry_id: Id,
pub job_ident: String,
pub schedule: String,
pub next: Option<DateTime<Utc>>,
}
@ -66,14 +66,14 @@ impl Entry {
}
pub fn as_stat(&self) -> EntryStat {
let job_id = match &self.runnable {
let job_ident = match &self.runnable {
EntryType::URunner(entry) => entry.job.meta.id.to_string(),
EntryType::Common(entry) => entry.call().job_id.to_string(),
EntryType::Common(entry) => entry.call().job_ident.to_string(),
};
EntryStat {
id: self.id,
job_id,
entry_id: self.entry_id,
job_ident,
schedule: self
.schedule
.as_ref()
@ -86,6 +86,6 @@ impl Entry {
impl fmt::Debug for Entry {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{} {:?} {:?}", self.id, self.schedule, self.next)
write!(f, "{} {:?} {:?}", self.entry_id, self.schedule, self.next)
}
}

@ -8,7 +8,6 @@ use cron::Schedule;
use entry::Entry;
use once_cell::sync::Lazy;
use tokio::sync::Mutex;
use tokio::time::timeout;
use uuid::Uuid;
use crate::jobs::AnonymousJobBatch;
@ -59,23 +58,23 @@ impl AsyncScheduler {
for entry in &mut *entries {
match entry.next.as_ref() {
Some(next) => {
if next.gt(&Utc::now()) {
break;
}
match &entry.runnable {
EntryType::Common(runnable) => {
let cancel_timeout =
next.timestamp_millis() - Utc::now().timestamp_millis();
let cancel_timeout = Duration::from_millis(cancel_timeout as u64);
tokio::spawn(timeout(cancel_timeout, runnable.call()));
let fut = runnable.call();
tokio::spawn(fut);
}
EntryType::URunner(runnable) => {
debug!("starting assigned job {:?}", runnable.ids);
job_batch.push(runnable.clone())
}
EntryType::URunner(runnable) => job_batch.push(runnable.clone()),
}
entry.set_next_run_time();
}
None => {}
}
if entry.next.as_ref().unwrap().gt(&Utc::now()) {
break;
}
}
AnonymousJobBatch::from_prepared_jobs(job_batch)
@ -90,7 +89,7 @@ impl AsyncScheduler {
let entry_id = Uuid::new_v4();
let runnable = runnable.into();
let mut entry = Entry {
id: entry_id,
entry_id,
schedule,
next: None,
runnable,
@ -103,7 +102,7 @@ impl AsyncScheduler {
}
pub async fn del_job(&self, entry_id: Id) {
self.entries.lock().await.retain(|e| e.id != entry_id);
self.entries.lock().await.retain(|e| e.entry_id != entry_id);
}
pub async fn stats(&self) -> Vec<EntryStat> {
@ -176,33 +175,4 @@ mod tests {
assert_eq!(*v.lock().await, 2);
}
/// Run task every second.
/// Task will be cancelled if not completed until next run
/// (and this task lasts 2 secs),
/// so the counter value should still be 0
#[tokio::test(flavor = "multi_thread")]
async fn cancellation() {
let counter = Arc::new(Mutex::new(0));
let scheduler = AsyncScheduler::new();
{
let counter = counter.clone();
scheduler
.add_job(Some("*/1 * * * * * *".try_into().unwrap()), move || {
let counter = counter.clone();
IdentifiableFuture::from_fut_with_ident("testf", async move {
sleep(Duration::from_secs(2)).await;
*counter.lock().await += 1;
})
})
.await;
}
scheduler.start().await;
sleep(Duration::from_secs(3)).await;
assert_eq!(*counter.lock().await, 0);
}
}

@ -24,18 +24,18 @@ lazy_static! {
}
pub struct IdentifiableFuture<R> {
pub job_id: String,
pub job_ident: String,
fut: Pin<Box<dyn Future<Output = R> + Send + Sync + 'static>>,
}
impl<R> IdentifiableFuture<R> {
pub fn from_fut_with_ident(
job_id: impl Into<String>,
job_ident: impl Into<String>,
job: impl Future<Output = R> + Send + Sync + 'static,
) -> Self {
Self {
fut: Box::pin(job),
job_id: job_id.into(),
job_ident: job_ident.into(),
}
}
}
@ -49,7 +49,7 @@ impl<R> Future for IdentifiableFuture<R> {
}
struct JoinInfo {
job_id: String,
job_ident: String,
handle: JoinHandle<JoinHandle<ExecResult>>,
completed: bool,
// collectable: bool, // indicates if future can be popped from pool via pop_task_if_completed
@ -91,7 +91,7 @@ impl URunner {
let handle = Handle::current();
let fid = Uuid::new_v4();
let tx = get_sender();
let job_id = executable.job_id.clone();
let job_id = executable.job_ident.clone();
self.fids.push(fid);
let job_wrapper = async move {
debug!("inside wrapper (started): {}", fid);
@ -100,7 +100,7 @@ impl URunner {
result
};
let handler = JoinInfo {
job_id,
job_ident: job_id,
handle: spawn_blocking(move || handle.spawn(job_wrapper)),
completed: false,
};
@ -155,7 +155,7 @@ impl URunner {
.lock()
.await
.values()
.map(|v| v.job_id.clone())
.map(|v| v.job_ident.clone())
.collect()
}

@ -34,7 +34,7 @@ static INDEX: Lazy<Mutex<Index>> = Lazy::new(|| {
pub struct IndexFileMeta {
extension: Option<OsString>,
pub external: bool, // if file is present before adding to index
pub hash: Vec<u8>,
pub hashsum: String,
pub path: PathBuf, // actual file path
pub size: u64,
}
@ -42,7 +42,7 @@ pub struct IndexFileMeta {
impl IndexFileMeta {
pub fn new(
full_path: impl Into<PathBuf>,
hash: Vec<u8>,
hashsum: String,
external: bool,
) -> Result<Self, Error> {
let full_path: PathBuf = full_path.into();
@ -53,7 +53,7 @@ impl IndexFileMeta {
path: full_path,
extension,
external,
hash,
hashsum,
size,
})
}
@ -96,14 +96,14 @@ pub fn get(name: impl AsRef<str>) -> Option<IndexFileMeta> {
index.get(name.as_ref()).cloned()
}
pub fn get_by_hash(hash: impl AsRef<[u8]>) -> Option<(String, IndexFileMeta)> {
pub fn get_by_hash(hash: impl AsRef<str>) -> Option<(String, IndexFileMeta)> {
let mut index = INDEX.lock();
sync::deleted(&mut index);
index
.iter()
.find(|(_name, meta)| meta.hash == hash.as_ref())
.find(|(_name, meta)| meta.hashsum == hash.as_ref())
.map(|(n, m)| (n.to_owned(), m.clone()))
}

@ -93,7 +93,7 @@ pub fn edit(name: impl AsRef<str>, data: impl AsRef<[u8]>) -> Result<()> {
let meta = read_meta(&name).context("edit_meta")?;
let data_hash = hash_data(&data);
if meta.hash == data_hash {
if meta.hashsum == data_hash {
return Ok(());
}
@ -205,10 +205,11 @@ pub fn cleanup() {
}
*/
fn hash_data(data: impl AsRef<[u8]>) -> Vec<u8> {
fn hash_data(data: impl AsRef<[u8]>) -> String {
use sha3::{Digest, Sha3_256};
let mut hasher = Sha3_256::new();
hasher.update(data);
hasher.finalize().to_vec()
let hashsum = hasher.finalize().to_vec();
hex::encode(hashsum)
}

@ -1,5 +1,5 @@
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE TYPE JobType AS ENUM ('shell', 'init', 'python', 'service');
CREATE TYPE JobType AS ENUM ('shell', 'init', 'python', 'service', 'stats');
CREATE TYPE JobState AS ENUM ('queued', 'running', 'finished');
CREATE TYPE AgentState AS ENUM ('new', 'active', 'banned');

Loading…
Cancel
Save