diff --git a/.cargo/config.toml b/.cargo/config.toml index 433fea9..241bfac 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,6 +1,15 @@ [build] rustflags = [ - "-L", "/home/ortem/src/rust/unki/static/lib", + "-L/usr/lib/musl/lib", + "-L/home/ortem/src/rust/unki/static/lib", "--remap-path-prefix=/home/ortem/src/rust/unki=src", "--remap-path-prefix=/home/ortem/.cargo=cargo" ] +target = "x86_64-unknown-linux-musl" + +[env] +STATIC_PREFIX = "static" +PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true" +PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = { value = "static/bin/pg_config", relative = true } +OPENSSL_STATIC = "true" +OPENSSL_DIR = { value = "static", relative = true } diff --git a/Cargo.lock b/Cargo.lock index 8d0a08b..9719584 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,19 +4,19 @@ version = 3 [[package]] name = "actix-codec" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a7559404a7f3573127aab53c08ce37a6c6a315c374a31070f3c91cd1b4a7fe" +checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "bytes", "futures-core", "futures-sink", - "log", "memchr", "pin-project-lite", "tokio", "tokio-util", + "tracing", ] [[package]] @@ -36,17 +36,17 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0070905b2c4a98d184c4e81025253cb192aa8a73827553f38e9410801ceb35bb" +checksum = "a92ef85799cba03f76e4f7c10f533e66d87c9a7e7055f3391f09000ad8351bc9" dependencies = [ "actix-codec", "actix-rt", "actix-service", "actix-utils", "ahash", - "base64 0.21.0", - "bitflags", + "base64", + "bitflags 2.4.0", "brotli", "bytes", "bytestring", @@ -75,12 +75,12 @@ dependencies = [ [[package]] name = "actix-macros" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" +checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn", + "syn 2.0.38", ] [[package]] @@ -98,9 +98,9 @@ dependencies = [ [[package]] name = "actix-rt" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" +checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" dependencies = [ "futures-core", "tokio", @@ -108,9 +108,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e8613a75dd50cc45f473cee3c34d59ed677c0f7b44480ce3b8247d7dc519327" +checksum = "3eb13e7eef0423ea6eab0e59f6c72e7cb46d33691ad56a726b3cd07ddec2c2d4" dependencies = [ "actix-rt", "actix-service", @@ -118,8 +118,7 @@ dependencies = [ "futures-core", "futures-util", "mio", - "num_cpus", - "socket2", + "socket2 0.5.4", "tokio", "tracing", ] @@ -147,9 +146,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464e0fddc668ede5f26ec1f9557a8d44eda948732f40c6b0ad79126930eb775f" +checksum = "0e4a5b5e29603ca8c94a77c65cf874718ceb60292c5a5c3e5f4ace041af462b9" dependencies = [ "actix-codec", "actix-http", @@ -169,7 +168,6 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "http", "itoa", "language-tags", "log", @@ -181,21 +179,30 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2", - "time 0.3.19", + "socket2 0.5.4", + "time", "url", ] [[package]] name = "actix-web-codegen" -version = "4.1.0" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa9362663c8643d67b2d5eafba49e4cb2c8a053a29ed00a0bea121f17c76b13" +checksum = "eb1f50ebbb30eca122b188319a4398b3f7bb4a8cdf50ecfb73bfc6a3c3ce54f5" dependencies = [ "actix-router", "proc-macro2", "quote", - "syn", + "syn 2.0.38", +] + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", ] [[package]] @@ -212,10 +219,11 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "ahash" -version = "0.7.6" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ + "cfg-if 1.0.0", "getrandom", "once_cell", "version_check", @@ -223,9 +231,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" dependencies = [ "memchr", ] @@ -245,6 +253,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -271,13 +285,13 @@ checksum = "a26fa4d7e3f2eebadf743988fc8aec9fa9a9e82611acafd77c1462ed6262440a" [[package]] name = "async-trait" -version = "0.1.64" +version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.38", ] [[package]] @@ -298,16 +312,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] -name = "base64" -version = "0.13.1" +name = "backtrace" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] [[package]] name = "base64" -version = "0.21.0" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" + +[[package]] +name = "bincode" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] [[package]] name = "bitflags" @@ -315,26 +347,26 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + [[package]] name = "block-buffer" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] -[[package]] -name = "boxfnonce" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5988cb1d626264ac94100be357308f29ff7cbdd3b36bda27f450a4ee3f713426" - [[package]] name = "brotli" -version = "3.3.4" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -343,58 +375,49 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.3.4" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", ] -[[package]] -name = "buf_redux" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f" -dependencies = [ - "memchr", - "safemem", -] - [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "bytestring" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7f83e57d9154148e355404702e2694463241880b939570d7c97c014da7a69a1" +checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae" dependencies = [ "bytes", ] [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", + "libc", ] [[package]] @@ -411,17 +434,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.23" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", - "time 0.1.45", "wasm-bindgen", - "winapi", + "windows-targets", ] [[package]] @@ -432,23 +454,13 @@ checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ "ansi_term", "atty", - "bitflags", + "bitflags 1.3.2", "strsim", "textwrap", "unicode-width", "vec_map", ] -[[package]] -name = "codespan-reporting" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] - [[package]] name = "convert_case" version = "0.4.0" @@ -462,7 +474,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding", - "time 0.3.19", + "time", "version_check", ] @@ -478,15 +490,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.5" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -502,9 +514,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -512,9 +524,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", @@ -523,9 +535,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if 1.0.0", @@ -536,9 +548,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if 1.0.0", ] @@ -554,58 +566,29 @@ dependencies = [ ] [[package]] -name = "cxx" -version = "1.0.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.91" +name = "ctor" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" +checksum = "37e366bff8cd32dd8754b0991fb66b279dc48f598c3a18914852a6673deef583" dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", "quote", - "scratch", - "syn", + "syn 2.0.38", ] [[package]] -name = "cxxbridge-flags" -version = "1.0.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.91" +name = "daemonize" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" +checksum = "ab8bfdaacb3c887a54d41bdf48d3af8873b3f5566469f8ba21b92057509f116e" dependencies = [ - "proc-macro2", - "quote", - "syn", + "libc", ] [[package]] -name = "daemonize" -version = "0.4.1" +name = "data-encoding" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70c24513e34f53b640819f0ac9f705b673fcf4006d7aab8778bee72ebfc89815" -dependencies = [ - "boxfnonce", - "libc", -] +checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "deadpool" @@ -633,22 +616,28 @@ dependencies = [ [[package]] name = "deadpool-runtime" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaa37046cc0f6c3cc6090fbdbf73ef0b8ef4cfcc37f6befc0020f63e8cf121e1" +checksum = "63dfa964fe2a66f3fde91fc70b267fe193d822c7e603e2a675a49a7f46ad3f49" dependencies = [ "tokio", ] [[package]] name = "deadpool-sync" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1bea344b64b32537fde6e0f0179b1ede34d435636719dd40fe6a0f28218a61c" +checksum = "f8db70494c13cae4ce67b4b4dafdaf828cf0df7237ab5b9e2fcabee4965d0a0a" dependencies = [ - "deadpool", + "deadpool-runtime", ] +[[package]] +name = "deranged" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" + [[package]] name = "derive_more" version = "0.99.17" @@ -659,16 +648,16 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 1.0.109", ] [[package]] name = "diesel" -version = "2.0.3" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4391a22b19c916e50bec4d6140f29bdda3e3bb187223fe6e3ea0b6e4d1021c04" +checksum = "2268a214a6f118fce1838edba3d1561cf0e78d8de785475957a580a7f8c69d33" dependencies = [ - "bitflags", + "bitflags 2.4.0", "byteorder", "diesel_derives", "itoa", @@ -679,44 +668,53 @@ dependencies = [ [[package]] name = "diesel-derive-enum" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b10c03b954333d05bfd5be1d8a74eae2c9ca77b86e0f1c3a1ea29c49da1d6c2" +checksum = "81c5131a2895ef64741dad1d483f358c2a229a3a2d1b256778cdc5e146db64d4" dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn", + "syn 2.0.38", ] [[package]] name = "diesel_derives" -version = "2.0.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b758c91dbc3fe1fdcb0dba5bd13276c6a66422f2ef5795b58488248a310aa" +checksum = "ef8337737574f55a468005a83499da720f20c65586241ffea339db9ecdfd2b44" dependencies = [ - "proc-macro-error", + "diesel_table_macro_syntax", "proc-macro2", "quote", - "syn", + "syn 2.0.38", ] [[package]] name = "diesel_migrations" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9ae22beef5e9d6fab9225ddb073c1c6c1a7a6ded5019d5da11d1e5c5adc34e2" +checksum = "6036b3f0120c5961381b570ee20a02432d7e2d27ea60de9578799cf9156914ac" dependencies = [ "diesel", "migrations_internals", "migrations_macros", ] +[[package]] +name = "diesel_table_macro_syntax" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" +dependencies = [ + "syn 2.0.38", +] + [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -736,15 +734,15 @@ checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if 1.0.0", ] @@ -758,6 +756,12 @@ dependencies = [ "serde", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.2.8" @@ -769,6 +773,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "errno" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + [[package]] name = "errno-dragonfly" version = "0.1.2" @@ -781,18 +796,15 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" dependencies = [ "crc32fast", "miniz_oxide", @@ -821,18 +833,18 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] [[package]] name = "futures" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", @@ -845,9 +857,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", "futures-sink", @@ -855,15 +867,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" [[package]] name = "futures-executor" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" dependencies = [ "futures-core", "futures-task", @@ -872,38 +884,44 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-macro" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.38", ] [[package]] name = "futures-sink" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" [[package]] name = "futures-task" -version = "0.3.26" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-timer" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-channel", "futures-core", @@ -919,9 +937,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -929,22 +947,28 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", ] +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + [[package]] name = "guess_host_triple" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35a8ce923c7490629d84e12fa2f75e1733f1ec692a47c264f9b7fd632855afc" dependencies = [ - "errno", + "errno 0.2.8", "libc", "log", "winapi", @@ -952,9 +976,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes", "fnv", @@ -962,7 +986,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -975,14 +999,19 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" + [[package]] name = "headers" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" +checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" dependencies = [ - "base64 0.13.1", - "bitflags", + "base64", "bytes", "headers-core", "http", @@ -1026,18 +1055,15 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.6" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "http" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", @@ -1063,15 +1089,15 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.24" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -1084,7 +1110,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -1106,33 +1132,32 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows", ] [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", ] [[package]] name = "idna" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1140,9 +1165,9 @@ dependencies = [ [[package]] name = "include-flate" -version = "0.1.4" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfdcb449c721557c1cf89bbd3412bf33fa963289e26e9badbd824a960912e148" +checksum = "c2e11569346406931d20276cc460215ee2826e7cad43aa986999cb244dd7adb0" dependencies = [ "include-flate-codegen-exports", "lazy_static", @@ -1159,7 +1184,7 @@ dependencies = [ "proc-macro-hack", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1174,30 +1199,32 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", ] [[package]] -name = "instant" -version = "0.1.12" +name = "indexmap" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ - "cfg-if 1.0.0", + "equivalent", + "hashbrown 0.14.1", ] [[package]] -name = "integration" +name = "integration-tests" version = "0.1.0" dependencies = [ + "ctor", "once_cell", "reqwest", - "rstest", + "rstest 0.17.0", "serde", "serde_json", "shlex", @@ -1209,34 +1236,43 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.7.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keccak" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +dependencies = [ + "cpufeatures", +] + [[package]] name = "language-tags" version = "0.3.2" @@ -1251,15 +1287,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libflate" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97822bf791bd4d5b403713886a5fbe8bf49520fe78e323b0dc480ca1a03e50b0" +checksum = "5ff4ae71b685bbad2f2f391fe74f6b7659a34871c08b210fdc039e43bee07d18" dependencies = [ "adler32", "crc32fast", @@ -1276,23 +1312,19 @@ dependencies = [ ] [[package]] -name = "link-cplusplus" -version = "1.0.8" +name = "linux-raw-sys" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] +checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" [[package]] name = "local-channel" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f303ec0e94c6c54447f84f3b0ef7af769858a9c4ef56ef2a986d3dcd4c3fc9c" +checksum = "e0a493488de5f18c8ffcba89eebb8532ffc562dc400490eb65b84893fae0b178" dependencies = [ "futures-core", "futures-sink", - "futures-util", "local-waker", ] @@ -1304,9 +1336,9 @@ checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -1314,12 +1346,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "matchers" @@ -1327,29 +1356,29 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] [[package]] name = "migrations_internals" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c493c09323068c01e54c685f7da41a9ccf9219735c3766fbfd6099806ea08fbc" +checksum = "0f23f71580015254b020e856feac3df5878c2c7a8812297edd6c0a485ac9dada" dependencies = [ "serde", "toml", @@ -1357,9 +1386,9 @@ dependencies = [ [[package]] name = "migrations_macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a8ff27a350511de30cdabb77147501c36ef02e0451d957abea2f30caffb2b58" +checksum = "cce3325ac70e67bbab5bd837a31cae01f1a6db64e0e744a33cb03a543469ef08" dependencies = [ "migrations_internals", "proc-macro2", @@ -1379,9 +1408,9 @@ dependencies = [ [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" @@ -1395,41 +1424,41 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "wasi", + "windows-sys", ] [[package]] -name = "multipart" -version = "0.18.0" +name = "multer" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00dec633863867f29cb39df64a397cdf4a6354708ddd7759f70c7fb51c5f9182" +checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" dependencies = [ - "buf_redux", + "bytes", + "encoding_rs", + "futures-util", + "http", "httparse", "log", + "memchr", "mime", - "mime_guess", - "quick-error", - "rand", - "safemem", - "tempfile", - "twoway", + "spin 0.9.8", + "version_check", ] [[package]] @@ -1456,7 +1485,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cc", "cfg-if 0.1.10", "libc", @@ -1483,47 +1512,46 @@ dependencies = [ ] [[package]] -name = "num-integer" -version = "0.1.45" +name = "num-traits" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", - "num-traits", ] [[package]] -name = "num-traits" -version = "0.2.15" +name = "num_cpus" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "autocfg", + "hermit-abi 0.3.3", + "libc", ] [[package]] -name = "num_cpus" -version = "1.15.0" +name = "object" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ - "hermit-abi 0.2.6", - "libc", + "memchr", ] [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "openssl" -version = "0.10.45" +version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags", + "bitflags 2.4.0", "cfg-if 1.0.0", "foreign-types", "libc", @@ -1534,13 +1562,13 @@ dependencies = [ [[package]] name = "openssl-macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.38", ] [[package]] @@ -1551,11 +1579,10 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.80" +version = "0.9.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" dependencies = [ - "autocfg", "cc", "libc", "pkg-config", @@ -1580,54 +1607,54 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall", "smallvec", - "windows-sys 0.45.0", + "windows-targets", ] [[package]] name = "paste" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.38", ] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -1637,15 +1664,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "platforms" -version = "3.0.2" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d7ddaed09e0eb771a79ab0fd64609ba0afb0a8366421957936ad14cbd13630" +checksum = "4503fa043bf02cee09a9582e9554b4c6403b2ef55e4612e96561d294419429f8" [[package]] name = "ppv-lite86" @@ -1655,9 +1682,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "pq-sys" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b845d6d8ec554f972a2c5298aad68953fd64e7441e846075450b44656a016d1" +checksum = "31c0052426df997c0cbd30789eb44ca097e3541717a7b8fa36b1c464ee7edebd" dependencies = [ "vcpkg", ] @@ -1671,7 +1698,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "version_check", ] @@ -1694,24 +1721,18 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "5b1106fec09662ec6dd98ccac0f81cef56984d0b49f75c92d8cbad76e20c005c" dependencies = [ "unicode-ident", ] -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - [[package]] name = "quote" -version = "1.0.23" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -1748,9 +1769,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ "either", "rayon-core", @@ -1758,34 +1779,33 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.7.1" +version = "1.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-automata 0.3.9", + "regex-syntax 0.7.5", ] [[package]] @@ -1794,31 +1814,39 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.7.5", ] [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] -name = "remove_dir_all" -version = "0.5.3" +name = "regex-syntax" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "reqwest" -version = "0.11.14" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ - "base64 0.21.0", + "base64", "bytes", "encoding_rs", "futures-core", @@ -1839,6 +1867,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-native-tls", "tower-service", @@ -1864,7 +1893,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", @@ -1886,14 +1915,40 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 1.0.109", +] + +[[package]] +name = "rstest" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de1bb486a691878cd320c2f0d319ba91eeaa2e894066d8b5f8f117c000e9d962" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290ca1a1c8ca7edb7c3283bd44dc35dd54fdec6253a3912e201ba1072018fca8" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", + "unicode-ident", ] [[package]] name = "rust-embed" -version = "6.4.2" +version = "6.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "283ffe2f866869428c92e0d61c2f35dfb4355293cdfdc48f49e895c15f1333d1" +checksum = "a36224c3276f8c4ebc8c20f158eca7ca4359c8db89991c4925132aaaf6702661" dependencies = [ "include-flate", "rust-embed-impl", @@ -1903,27 +1958,33 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "6.3.1" +version = "6.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31ab23d42d71fb9be1b643fe6765d292c5e14d46912d13f3ae2815ca048ea04d" +checksum = "49b94b81e5b2c284684141a2fb9e2a31be90638caf040bf9afbc5a0416afe1ac" dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn", + "syn 2.0.38", "walkdir", ] [[package]] name = "rust-embed-utils" -version = "7.3.0" +version = "7.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1669d81dfabd1b5f8e2856b8bbe146c6192b0ba22162edc738ac0a5de18f054" +checksum = "9d38ff6bf570dc3bb7100fce9f7b60c33fa71d80e88da3f2580df4ff2bdded74" dependencies = [ "sha2", "walkdir", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc_version" version = "0.4.0" @@ -1933,38 +1994,55 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f25469e9ae0f3d0047ca8b93fc56843f38e6774f0914a107ff8b41be8be8e0b7" +dependencies = [ + "bitflags 2.4.0", + "errno 0.3.4", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustls" -version = "0.20.8" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", "ring", + "rustls-webpki", "sct", - "webpki", ] [[package]] name = "rustls-pemfile" -version = "0.2.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.13.1", + "base64", ] [[package]] -name = "ryu" -version = "1.0.12" +name = "rustls-webpki" +version = "0.101.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +dependencies = [ + "ring", + "untrusted", +] [[package]] -name = "safemem" -version = "0.3.3" +name = "ryu" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "same-file" @@ -1977,11 +2055,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1992,15 +2070,9 @@ checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" [[package]] name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "scratch" -version = "1.0.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" @@ -2014,11 +2086,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.8.2" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -2027,9 +2099,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" dependencies = [ "core-foundation-sys", "libc", @@ -2037,41 +2109,64 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.16" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" +checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" [[package]] name = "serde" -version = "1.0.152" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.38", ] [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", "serde", ] +[[package]] +name = "serde_qs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0431a35568651e363364210c91983c1da5eb29404d9f0928b67d4ebcfa7d330c" +dependencies = [ + "futures", + "percent-encoding", + "serde", + "thiserror", + "tracing", + "warp", +] + +[[package]] +name = "serde_spanned" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2085,10 +2180,10 @@ dependencies = [ ] [[package]] -name = "sha-1" -version = "0.10.1" +name = "sha1" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if 1.0.0", "cpufeatures", @@ -2096,10 +2191,10 @@ dependencies = [ ] [[package]] -name = "sha1" -version = "0.10.5" +name = "sha2" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if 1.0.0", "cpufeatures", @@ -2107,30 +2202,29 @@ dependencies = [ ] [[package]] -name = "sha2" -version = "0.10.6" +name = "sha3" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" dependencies = [ - "cfg-if 1.0.0", - "cpufeatures", "digest", + "keccak", ] [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] [[package]] name = "shlex" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" [[package]] name = "signal-hook-registry" @@ -2143,35 +2237,51 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", ] +[[package]] +name = "socket2" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "spin" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "strsim" version = "0.8.0" @@ -2199,7 +2309,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2229,7 +2339,7 @@ dependencies = [ "heck 0.3.3", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2241,14 +2351,25 @@ dependencies = [ "heck 0.3.3", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -2271,26 +2392,37 @@ dependencies = [ ] [[package]] -name = "tempfile" -version = "3.3.0" +name = "system-configuration" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ - "cfg-if 1.0.0", - "fastrand", + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", "libc", - "redox_syscall", - "remove_dir_all", - "winapi", ] [[package]] -name = "termcolor" -version = "1.2.0" +name = "tempfile" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ - "winapi-util", + "cfg-if 1.0.0", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", ] [[package]] @@ -2319,7 +2451,7 @@ checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2334,21 +2466,11 @@ dependencies = [ [[package]] name = "time" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "time" -version = "0.3.19" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53250a3b3fed8ff8fd988587d8925d26a83ac3845d9e03b220b37f34c2b8d6c2" +checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" dependencies = [ + "deranged", "itoa", "serde", "time-core", @@ -2357,15 +2479,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.7" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a460aeb8de6dcb0f381e1ee05f1cd56fcf5a5f6eb8187ff3d8f0b11078d38b7c" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -2387,33 +2509,32 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.25.0" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", - "memchr", "mio", "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.4", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.38", ] [[package]] @@ -2428,20 +2549,19 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.23.4" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls", "tokio", - "webpki", ] [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", @@ -2450,9 +2570,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.17.2" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" +checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", @@ -2462,9 +2582,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", @@ -2476,13 +2596,38 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.11" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" dependencies = [ "serde", ] +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.0.2", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -2509,26 +2654,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" dependencies = [ "crossbeam-channel", - "time 0.3.19", + "time", "tracing-subscriber", ] [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.38", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", "valuable", @@ -2547,9 +2692,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ "matchers", "nu-ansi-term", @@ -2571,37 +2716,28 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "tungstenite" -version = "0.17.3" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" +checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" dependencies = [ - "base64 0.13.1", "byteorder", "bytes", + "data-encoding", "http", "httparse", "log", "rand", - "sha-1", + "sha1", "thiserror", "url", "utf-8", ] -[[package]] -name = "twoway" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" -dependencies = [ - "memchr", -] - [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "u_agent" @@ -2620,6 +2756,7 @@ name = "u_lib" version = "0.1.0" dependencies = [ "anyhow", + "bincode", "chrono", "daemonize", "deadpool-diesel", @@ -2627,7 +2764,6 @@ dependencies = [ "diesel-derive-enum", "dotenv", "envy", - "futures", "guess_host_triple", "lazy_static", "libc", @@ -2636,9 +2772,10 @@ dependencies = [ "parking_lot", "platforms", "reqwest", - "rstest", + "rstest 0.12.0", "serde", "serde_json", + "sha3", "shlex", "strum 0.20.0", "thiserror", @@ -2690,11 +2827,13 @@ dependencies = [ "deadpool-diesel", "diesel", "hyper", + "mime_guess", "once_cell", "openssl", - "rstest", + "rstest 0.12.0", "serde", "serde_json", + "serde_qs", "thiserror", "tokio", "tracing", @@ -2705,24 +2844,24 @@ dependencies = [ [[package]] name = "unicase" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ "version_check", ] [[package]] name = "unicode-bidi" -version = "0.3.10" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -2741,9 +2880,9 @@ checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "untrusted" @@ -2753,9 +2892,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -2770,9 +2909,9 @@ checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "uuid" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" dependencies = [ "getrandom", "serde", @@ -2810,30 +2949,28 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "walkdir" -version = "2.3.2" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", - "winapi", "winapi-util", ] [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] [[package]] name = "warp" -version = "0.3.3" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed7b8be92646fc3d18b06147664ebc5f48d222686cb11a8755e561a735aacc6d" +checksum = "c1e92e22e03ff1230c03a1a8ee37d2f89cd489e2e541b7550d6afad96faed169" dependencies = [ "bytes", "futures-channel", @@ -2844,7 +2981,7 @@ dependencies = [ "log", "mime", "mime_guess", - "multipart", + "multer", "percent-encoding", "pin-project", "rustls-pemfile", @@ -2861,12 +2998,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2875,9 +3006,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -2885,24 +3016,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.38", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.34" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -2912,9 +3043,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2922,43 +3053,33 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.38", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "winapi" version = "0.3.9" @@ -2977,9 +3098,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -2991,34 +3112,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-sys" -version = "0.42.0" +name = "windows" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows-targets", ] [[package]] name = "windows-sys" -version = "0.45.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -3031,69 +3146,79 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "winnow" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907" +dependencies = [ + "memchr", +] [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi", + "cfg-if 1.0.0", + "windows-sys", ] [[package]] name = "zstd" -version = "0.12.3+zstd.1.5.2" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.4+zstd.1.5.4" +version = "6.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7afb4b54b8910cf5447638cb54bf4e8a65cbedd783af98b98c62ffe91f185543" +checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" dependencies = [ "libc", "zstd-sys", @@ -3101,9 +3226,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.7+zstd.1.5.4" +version = "2.0.8+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index 224420d..372e08a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,13 +6,15 @@ members = [ "bin/u_run", "bin/u_server", "lib/u_lib", - "integration", + "integration-tests", ] +resolver = "2" [workspace.dependencies] anyhow = "=1.0.63" deadpool-diesel = "0.4.0" diesel = { version = "2", features = ["postgres", "uuid"] } +mime_guess = "2.0" openssl = "0.10" reqwest = { version = "0.11", features = ["json"] } serde = { version = "1.0", features = ["derive"] } diff --git a/Makefile.toml b/Makefile.toml index 08e5742..67ef1dc 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -22,13 +22,6 @@ default_to_workspace = false [env] TARGET = "x86_64-unknown-linux-musl" CARGO = "cargo" -ROOTDIR = "${CARGO_MAKE_WORKING_DIRECTORY}" -STATIC_PREFIX = "${ROOTDIR}/static" -PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL = "true" -PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU = "${STATIC_PREFIX}/bin/pg_config" -OPENSSL_STATIC = "true" -OPENSSL_DIR = "${STATIC_PREFIX}" - [tasks.build_static_libs] script = "./scripts/build_musl_libs.sh" @@ -63,29 +56,41 @@ upx -9 $BINS ''' [tasks.build] -dependencies = ["cargo_update", "cargo_build", "release_tasks"] +dependencies = ["cargo_build", "release_tasks"] clear = true [tasks.run] disabled = true -[tasks.unit] +[tasks.run_front] +script = ''' +cd ./bin/u_panel/src/gui/fe +ng serve +''' + +[tasks.unit-tests] command = "${CARGO}" args = ["test", "--target", "${TARGET}", "--lib", "--", "${@}"] -[tasks.integration] +[tasks.ut] +alias = "unit-tests" + +[tasks.integration-tests] dependencies = ["cargo_update"] script = ''' [[ ! -d "./target/${TARGET}/${PROFILE_OVERRIDE}" ]] && echo 'No target folder. Build project first' && exit 1 -cd ./integration +cd ./integration-tests bash integration_tests.sh ${@} ''' +[tasks.it] +alias = "integration-tests" + [tasks.test] -dependencies = ["unit", "integration"] +dependencies = ["unit", "integration-tests"] [tasks.gen_schema] script = './scripts/gen_schema.sh' [tasks.deploy] -script = './scripts/deploy.sh' \ No newline at end of file +script = './scripts/deploy.sh' diff --git a/bin/u_agent/Cargo.toml b/bin/u_agent/Cargo.toml index 3b16a92..eced273 100644 --- a/bin/u_agent/Cargo.toml +++ b/bin/u_agent/Cargo.toml @@ -12,5 +12,5 @@ reqwest = { workspace = true } sysinfo = "0.10.5" tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "time"] } uuid = { workspace = true } -u_lib = { path = "../../lib/u_lib" } +u_lib = { path = "../../lib/u_lib", features = ["agent"] } diff --git a/bin/u_agent/src/lib.rs b/bin/u_agent/src/lib.rs index 08cc6a3..faa3d3f 100644 --- a/bin/u_agent/src/lib.rs +++ b/bin/u_agent/src/lib.rs @@ -5,24 +5,25 @@ use std::process::exit; use tokio::runtime::Builder; use tokio::time::{sleep, Duration}; use u_lib::{ - api::ClientHandler, + api::HttpClient, cache::JobCache, config::{get_self_id, EndpointsEnv, AGENT_ITERATION_INTERVAL}, error::ErrChan, - executor::pop_completed, - jobs::{fat_meta_to_thin, AnonymousJobBatch}, + executor, + jobs::AnonymousJobBatch, logging::init_logger, messaging::Reportable, models::AssignedJobById, }; -pub async fn process_request(jobs: Vec, client: &ClientHandler) { +async fn process_request(jobs: Vec, client: &HttpClient) { if !jobs.is_empty() { for jr in &jobs { if !JobCache::contains(jr.job_id) { info!("Fetching job: {}", &jr.job_id); - let fetched_job = loop { - match client.get_job(jr.job_id).await { + let mut fetched_job = loop { + //todo: use payload cache + match client.get_full_job(jr.job_id).await { Ok(result) => break result, Err(err) => { debug!("{:?} \nretrying...", err); @@ -30,10 +31,12 @@ pub async fn process_request(jobs: Vec, client: &ClientHandler) } } }; - match fat_meta_to_thin(fetched_job) { - Ok(thin_meta) => JobCache::insert(thin_meta), - Err(e) => ErrChan::send(e, "pld").await, + if let Some(payload) = &mut fetched_job.payload { + if let Err(e) = payload.maybe_split_payload() { + ErrChan::send(e, "pld").await; + } } + JobCache::insert(fetched_job); } } info!( @@ -58,12 +61,12 @@ pub async fn process_request(jobs: Vec, client: &ClientHandler) } } -async fn error_reporting(client: ClientHandler) -> ! { +async fn error_reporting(client: HttpClient) -> ! { loop { match ErrChan::recv().await { Some(err) => { 'retry: for _ in 0..3 { - match client.report(Reportable::Error(err.clone())).await { + match client.report([Reportable::Error(err.clone())]).await { Ok(_) => break 'retry, Err(e) => { debug!("Reporting error: {:?}", e); @@ -77,7 +80,7 @@ async fn error_reporting(client: ClientHandler) -> ! { } } -async fn agent_loop(client: ClientHandler) -> ! { +async fn agent_loop(client: HttpClient) -> ! { let self_id = get_self_id(); loop { match client.get_personal_jobs(self_id).await { @@ -87,7 +90,7 @@ async fn agent_loop(client: ClientHandler) -> ! { Err(err) => ErrChan::send(err, "processing").await, } - let result: Vec = pop_completed() + let result: Vec = executor::pop_completed() .await .into_iter() .map(|result| match result { @@ -109,7 +112,7 @@ pub fn run_forever() -> ! { let env = EndpointsEnv::load(); if cfg!(debug_assertions) { - init_logger(Some(format!( + let logfile_uid = format!( "u_agent-{}", get_self_id() .hyphenated() @@ -117,7 +120,8 @@ pub fn run_forever() -> ! { .split("-") .next() .unwrap() - ))); + ); + init_logger(Some(&logfile_uid)); } else { #[cfg(unix)] u_lib::unix::daemonize() @@ -130,14 +134,14 @@ pub fn run_forever() -> ! { .build() .unwrap() .block_on(async { - match ClientHandler::new(&env.u_server, None).await { + match HttpClient::new(&env.u_server, None).await { Ok(client) => { tokio::spawn(error_reporting(client.clone())); agent_loop(client).await } Err(e) => { error!("client init failed: {}", e); - exit(7) + exit(7) // todo: wtf? } } }) diff --git a/bin/u_panel/Cargo.toml b/bin/u_panel/Cargo.toml index e6705f6..083c7e0 100644 --- a/bin/u_panel/Cargo.toml +++ b/bin/u_panel/Cargo.toml @@ -11,7 +11,7 @@ actix-cors = "0.6.1" actix-web = "4.1" anyhow = { workspace = true } futures-util = "0.3.21" -mime_guess = "2.0.4" +mime_guess = { workspace = true } once_cell = "1.8.0" rust-embed = { version = "6.3.0", features = ["debug-embed", "compression"] } serde = { workspace = true } diff --git a/bin/u_panel/src/argparse.rs b/bin/u_panel/src/argparse.rs index 8a13e99..a4e67a6 100644 --- a/bin/u_panel/src/argparse.rs +++ b/bin/u_panel/src/argparse.rs @@ -1,124 +1,162 @@ use serde_json::{from_str, to_value, Value}; use structopt::StructOpt; use u_lib::{ - api::ClientHandler, - messaging::AsMsg, - models::{Agent, AssignedJob, FatJobMeta}, - types::Id, - types::PanelResult, - UError, UResult, + api::HttpClient, messaging::AsMsg, models::*, types::Id, types::PanelResult, UError, UResult, }; #[derive(StructOpt, Debug)] pub struct Args { #[structopt(subcommand)] cmd: Cmd, + #[structopt(short, long, default_value)] + brief: Brief, } #[derive(StructOpt, Debug)] enum Cmd { Agents(RUD), - Jobs(JobCRUD), - Map(JobMapCRUD), + Jobs(CRUD), + Map(AssignedCRUD), + Payloads(PayloadCRUD), Ping, Serve, } #[derive(StructOpt, Debug)] -enum JobCRUD { +enum CRUD { Create { - job: String, + item: String, }, #[structopt(flatten)] RUD(RUD), } #[derive(StructOpt, Debug)] -enum JobCmd { - #[structopt(external_subcommand)] - Cmd(Vec), +enum AssignedCRUD { + Create { + item: String, + }, + #[structopt(flatten)] + RUD(RUD), } #[derive(StructOpt, Debug)] -enum JobMapCRUD { +enum PayloadCRUD { Create { - #[structopt(parse(try_from_str = parse_uuid))] - agent_id: Id, - - job_idents: Vec, + item: String, + }, + Read { + id: Option, + }, + Update { + item: String, + }, + Delete { + #[structopt(parse(try_from_str = parse::uuid))] + id: Id, }, - #[structopt(flatten)] - RUD(RUD), } #[derive(StructOpt, Debug)] enum RUD { Read { - #[structopt(parse(try_from_str = parse_uuid))] + #[structopt(parse(try_from_str = parse::uuid))] id: Option, }, Update { item: String, }, Delete { - #[structopt(parse(try_from_str = parse_uuid))] + #[structopt(parse(try_from_str = parse::uuid))] id: Id, }, } -fn parse_uuid(src: &str) -> Result { - Id::parse_str(src).map_err(|e| e.to_string()) +mod parse { + use super::*; + + pub fn uuid(src: &str) -> Result { + Id::parse_str(src).map_err(|e| e.to_string()) + } } pub fn into_value(data: M) -> Value { to_value(data).unwrap() } -pub async fn process_cmd(client: ClientHandler, args: Args) -> PanelResult { +pub async fn process_cmd(client: HttpClient, args: Args) -> PanelResult { let catcher: UResult = (|| async { Ok(match args.cmd { Cmd::Agents(action) => match action { RUD::Read { id } => into_value(client.get_agents(id).await?), RUD::Update { item } => { - let agent = from_str::(&item)?; - into_value(client.update_agent(agent).await?) + let agent = from_str::(&item) + .map_err(|e| UError::DeserializeError(e.to_string(), item))?; + into_value(client.update_agent(&agent).await?) } RUD::Delete { id } => into_value(client.del(id).await?), }, Cmd::Jobs(action) => match action { - JobCRUD::Create { job } => { - let raw_job = from_str::(&job)?; - let mut job = raw_job.validated()?; + CRUD::Create { item: job } => { + let raw_job = from_str::(&job) + .map_err(|e| UError::DeserializeError(e.to_string(), job))?; + let mut job = raw_job.try_into_job()?; if let Some(payload) = &mut job.payload { - payload.read_into_self()?; + payload.join_payload()?; } - into_value(client.upload_jobs(job).await?) + into_value(client.upload_jobs([&job]).await?) } - JobCRUD::RUD(RUD::Read { id }) => match id { - //todo: use vec not to break frontend api, possibly refactor later - Some(id) => into_value(vec![client.get_job(id).await?]), + CRUD::RUD(RUD::Read { id }) => match id { + Some(id) => into_value(vec![client.get_job(id, args.brief).await?]), None => into_value(client.get_jobs().await?), }, - JobCRUD::RUD(RUD::Update { item }) => { - let raw_job = from_str::(&item)?; - let job = raw_job.validated()?; - into_value(client.update_job(job).await?) + CRUD::RUD(RUD::Update { item }) => { + let raw_job = from_str::(&item) + .map_err(|e| UError::DeserializeError(e.to_string(), item))?; + let job = raw_job.validate()?; + + // if let Some(payload) = &mut job.payload { + // payload.join_payload()?; + // } + + into_value(client.update_job(&job).await?) } - JobCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), + CRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), }, Cmd::Map(action) => match action { - JobMapCRUD::Create { - agent_id, - job_idents, - } => into_value(client.set_jobs(agent_id, job_idents).await?), - JobMapCRUD::RUD(RUD::Read { id }) => into_value(client.get_agent_jobs(id).await?), - JobMapCRUD::RUD(RUD::Update { item }) => { - let assigned = from_str::(&item)?; - into_value(client.update_result(assigned).await?) + AssignedCRUD::Create { item } => { + let payload = serde_json::from_str::>(&item) + .map_err(|e| UError::DeserializeError(e.to_string(), item))?; + into_value(client.assign_jobs(&payload).await?) + } + AssignedCRUD::RUD(RUD::Read { id }) => { + into_value(client.get_assigned_jobs(id).await?) + } + AssignedCRUD::RUD(RUD::Update { item }) => { + let assigned = from_str::(&item) + .map_err(|e| UError::DeserializeError(e.to_string(), item))?; + into_value(client.update_result(&assigned).await?) + } + AssignedCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), + }, + Cmd::Payloads(action) => match action { + PayloadCRUD::Create { item } => { + let payload = from_str::(&item) + .map_err(|e| UError::DeserializeError(e.to_string(), item))?; + into_value(client.upload_payload(&payload).await?) + } + PayloadCRUD::Read { id } => match id { + None => into_value(client.get_payloads().await?), + Some(id) => into_value(vec![client.get_payload(id, args.brief).await?]), + }, + PayloadCRUD::Update { item } => { + let payload = from_str::(&item) + .map_err(|e| UError::DeserializeError(e.to_string(), item))?; + into_value(client.update_payload(&payload).await?) } - JobMapCRUD::RUD(RUD::Delete { id }) => into_value(client.del(id).await?), + PayloadCRUD::Delete { id } => into_value(client.del(id).await?), }, Cmd::Ping => into_value(client.ping().await?), Cmd::Serve => { diff --git a/bin/u_panel/src/gui/fe/src/app/app-routing.module.ts b/bin/u_panel/src/gui/fe/src/app/app-routing.module.ts index 0ec7bf4..8178913 100644 --- a/bin/u_panel/src/gui/fe/src/app/app-routing.module.ts +++ b/bin/u_panel/src/gui/fe/src/app/app-routing.module.ts @@ -1,14 +1,13 @@ import { NgModule } from '@angular/core'; import { RouterModule, Routes } from '@angular/router'; -import { AgentComponent } from './core/tables/agent.component'; -import { JobComponent } from './core/tables/job.component'; -import { ResultComponent } from './core/tables/result.component'; -import { AgentInfoDialogComponent } from './core/tables/dialogs/agent_info.component'; +import { JobComponent, ResultComponent, AgentComponent, PayloadComponent } from './components/tables'; +//import { AgentInfoDialogComponent } from './core/tables/dialogs/agent-info-dialog.component'; const routes: Routes = [ { path: '', redirectTo: 'agents', pathMatch: 'full' }, { path: 'agents', component: AgentComponent }, { path: 'jobs', component: JobComponent }, + { path: 'payloads', component: PayloadComponent }, { path: 'results', component: ResultComponent }, ]; diff --git a/bin/u_panel/src/gui/fe/src/app/app.component.html b/bin/u_panel/src/gui/fe/src/app/app.component.html index b35dfd7..09c8847 100644 --- a/bin/u_panel/src/gui/fe/src/app/app.component.html +++ b/bin/u_panel/src/gui/fe/src/app/app.component.html @@ -2,4 +2,5 @@ {{tab.name}} - \ No newline at end of file + + \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/app.component.ts b/bin/u_panel/src/gui/fe/src/app/app.component.ts index 566f46b..a8925de 100644 --- a/bin/u_panel/src/gui/fe/src/app/app.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/app.component.ts @@ -1,4 +1,4 @@ -import { Component, ViewChild, AfterViewInit } from '@angular/core'; +import { Component } from '@angular/core'; @Component({ selector: 'app-root', @@ -9,6 +9,7 @@ export class AppComponent { tabs = [ { name: 'Agents', link: '/agents' }, { name: 'Jobs', link: '/jobs' }, - { name: 'Results', link: '/results' } + { name: 'Results', link: '/results' }, + { name: 'Payloads', link: '/payloads' } ]; } diff --git a/bin/u_panel/src/gui/fe/src/app/app.module.ts b/bin/u_panel/src/gui/fe/src/app/app.module.ts index c1572f4..5925085 100644 --- a/bin/u_panel/src/gui/fe/src/app/app.module.ts +++ b/bin/u_panel/src/gui/fe/src/app/app.module.ts @@ -8,22 +8,27 @@ import { MatTableModule } from '@angular/material/table'; import { MatFormFieldModule } from '@angular/material/form-field'; import { MatButtonModule } from '@angular/material/button' import { MatInputModule } from '@angular/material/input'; +import { MatSelectModule } from '@angular/material/select'; import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; import { HttpClientModule } from '@angular/common/http'; import { MatDialogModule } from '@angular/material/dialog'; import { MatIconModule } from '@angular/material/icon'; import { FormsModule } from '@angular/forms'; -import { AgentComponent, JobComponent, ResultComponent } from './core/tables'; +import { AgentComponent, JobComponent, ResultComponent, PayloadComponent } from './components/tables'; import { AgentInfoDialogComponent, AssignJobDialogComponent, JobInfoDialogComponent, - ResultInfoDialogComponent -} from './core/tables/dialogs'; + ResultInfoDialogComponent, + PayloadInfoDialogComponent +} from './components/dialogs'; import { APP_BASE_HREF } from '@angular/common'; import { MatTooltipModule } from '@angular/material/tooltip'; import { MatSnackBarModule } from '@angular/material/snack-bar'; import { MatListModule } from '@angular/material/list'; +import { GlobalErrorComponent } from './components/global-error/global-error.component'; +import { PayloadOverviewComponent } from './components/payload-overview/payload-overview.component'; +import { NewPayloadDialogComponent } from './components/dialogs/new-payload-dialog/new-payload-dialog.component'; @NgModule({ declarations: [ @@ -34,7 +39,12 @@ import { MatListModule } from '@angular/material/list'; AgentInfoDialogComponent, JobInfoDialogComponent, ResultInfoDialogComponent, - AssignJobDialogComponent + AssignJobDialogComponent, + PayloadComponent, + PayloadInfoDialogComponent, + GlobalErrorComponent, + PayloadOverviewComponent, + NewPayloadDialogComponent ], imports: [ BrowserModule, @@ -50,6 +60,7 @@ import { MatListModule } from '@angular/material/list'; MatIconModule, MatTooltipModule, MatSnackBarModule, + MatSelectModule, MatListModule, FormsModule, BrowserAnimationsModule diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/agent-info-dialog.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.html similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/agent-info-dialog.html rename to bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.html diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/agent_info.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.ts similarity index 73% rename from bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/agent_info.component.ts rename to bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.ts index d24d3a9..1feb025 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/agent_info.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/agent-info-dialog/agent-info-dialog.component.ts @@ -1,12 +1,12 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA } from '@angular/material/dialog'; -import { AgentModel } from '../../models/agent.model'; +import { AgentModel } from '../../../models/agent.model'; import { EventEmitter } from '@angular/core'; @Component({ selector: 'agent-info-dialog', - templateUrl: 'agent-info-dialog.html', - styleUrls: ['info-dialog.component.less'] + templateUrl: 'agent-info-dialog.component.html', + styleUrls: ['../base-info-dialog.component.less'] }) export class AgentInfoDialogComponent { is_preview = true; diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/assign-job-dialog.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.html similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/assign-job-dialog.html rename to bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.html diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts new file mode 100644 index 0000000..4cfdfc8 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/assign-job-dialog/assign-job-dialog.component.ts @@ -0,0 +1,36 @@ +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { AssignedJobByIdModel } from 'src/app/models'; +import { ApiTableService } from '../../../services'; + +@Component({ + selector: 'assign-job-dialog', + templateUrl: 'assign-job-dialog.component.html', + styleUrls: [] +}) +export class AssignJobDialogComponent { + rows: string[] = []; + selected_rows: string[] = []; + + constructor( + @Inject(MAT_DIALOG_DATA) public agent_id: string, + private dataSource: ApiTableService, + ) { + dataSource.getJobs().subscribe(resp => { + this.rows = resp.map(j => `${j.id} ${j.alias}`) + }) + } + + assignSelectedJobs() { + const assigned_jobs: AssignedJobByIdModel[] = this.selected_rows.map(row => { + const job_id = row.split(' ', 1)[0]; + return { + job_id: job_id, + agent_id: this.agent_id + } + }); + this.dataSource.createResult(assigned_jobs).subscribe(_ => { + alert("Created") + }); + } +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/info-dialog.component.less b/bin/u_panel/src/gui/fe/src/app/components/dialogs/base-info-dialog.component.less similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/info-dialog.component.less rename to bin/u_panel/src/gui/fe/src/app/components/dialogs/base-info-dialog.component.less diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/index.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/index.ts new file mode 100644 index 0000000..1a58ff1 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/index.ts @@ -0,0 +1,5 @@ +export * from './agent-info-dialog/agent-info-dialog.component'; +export * from './result-info-dialog/result-info-dialog.component'; +export * from './job-info-dialog/job-info-dialog.component'; +export * from './assign-job-dialog/assign-job-dialog.component'; +export * from './payload-info-dialog/payload-info-dialog.component'; \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html new file mode 100644 index 0000000..f581b5d --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.html @@ -0,0 +1,48 @@ +

Job info

+

Editing job info

+ +
+ + ID + + + + Alias + + + + Args + + +
+
+ + Type + + + + Platform + + + + Schedule + + +
+
+ + Payload + + {{ pld[1] }} + + +
+
+ +
+
+ + + + + \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts new file mode 100644 index 0000000..04b29c2 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/job-info-dialog/job-info-dialog.component.ts @@ -0,0 +1,28 @@ +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { EventEmitter } from '@angular/core'; +import { Job, JobModel } from '../../../models/job.model'; +import { ApiTableService } from 'src/app/services'; + +@Component({ + selector: 'job-info-dialog', + templateUrl: 'job-info-dialog.component.html', + styleUrls: ['../base-info-dialog.component.less'] +}) +export class JobInfoDialogComponent { + //[id, name] + isPreview = true; + allPayloads: [string | null, string][] = [[null, "none"]]; + + onSave = new EventEmitter(); + + constructor(@Inject(MAT_DIALOG_DATA) public data: Job, dataSource: ApiTableService) { + dataSource.getPayloads().subscribe(resp => { + this.allPayloads = this.allPayloads.concat(resp.map(r => [r.id, r.name])) + }) + } + + updateJob() { + this.onSave.emit(this.data.meta); + } +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.html new file mode 100644 index 0000000..2480108 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.html @@ -0,0 +1,20 @@ +

New payload

+ +
+ + Name + + + +
+
+ + Data + + +
+
+ + + + \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.ts new file mode 100644 index 0000000..69551bf --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/new-payload-dialog/new-payload-dialog.component.ts @@ -0,0 +1,43 @@ +import { Component, EventEmitter, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { NewPayloadModel } from 'src/app/models/payload.model'; + +@Component({ + selector: 'new-payload-dialog', + templateUrl: 'new-payload-dialog.component.html', + styleUrls: ['../base-info-dialog.component.less'] +}) +export class NewPayloadDialogComponent { + decodedPayload = ""; + uploadMode = false; + onSave = new EventEmitter(); + + constructor(@Inject(MAT_DIALOG_DATA) public payload: NewPayloadModel) { } + + save() { + if (this.payload.data.length == 0) { + this.payload.data = Array.from(new TextEncoder().encode(this.decodedPayload)); + } + this.onSave.emit(this.payload); + } + + onFileSelected(event: any) { + const file: File = event.target.files[0]; + if (file) { + this.uploadMode = true + const reader = new FileReader(); + reader.onload = e => { + this.payload.name = file.name; + const result = e.target?.result; + if (result instanceof ArrayBuffer) { + const d = Array.from(new Uint8Array(result)); + this.payload.data = d; + console.log(this.payload.data) + } else { + alert!("no file") + } + } + reader.readAsArrayBuffer(file) + } + } +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html new file mode 100644 index 0000000..cc3ce41 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.html @@ -0,0 +1,34 @@ +

Payload

+

Editing payload

+ +
+
+ + ID + + + + Name + + +
+
+ + MIME-type + + + + Size + + +
+
+
+ +
+
+ + + + + \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts new file mode 100644 index 0000000..680f996 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/payload-info-dialog/payload-info-dialog.component.ts @@ -0,0 +1,20 @@ +import { Component, EventEmitter, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { PayloadModel } from 'src/app/models/payload.model'; + +@Component({ + selector: 'payload-info-dialog', + templateUrl: 'payload-info-dialog.component.html', + styleUrls: ['../base-info-dialog.component.less'] +}) +export class PayloadInfoDialogComponent { + isPreview = true; + + onSave = new EventEmitter(); + + constructor(@Inject(MAT_DIALOG_DATA) public payload: PayloadModel) { } + + updatePayload() { + this.onSave.emit(this.payload); + } +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/result-info-dialog.html b/bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.html similarity index 86% rename from bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/result-info-dialog.html rename to bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.html index 6ce43c2..50580d9 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/result-info-dialog.html +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.html @@ -39,13 +39,7 @@
-

- - Result - - -

+
diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/result_info.component.ts b/bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.ts similarity index 74% rename from bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/result_info.component.ts rename to bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.ts index b02fae5..4d9734f 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/result_info.component.ts +++ b/bin/u_panel/src/gui/fe/src/app/components/dialogs/result-info-dialog/result-info-dialog.component.ts @@ -1,11 +1,11 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA } from '@angular/material/dialog'; -import { ResultModel } from '../../models/result.model'; +import { ResultModel } from '../../../models/result.model'; @Component({ selector: 'result-info-dialog', - templateUrl: 'result-info-dialog.html', - styleUrls: ['info-dialog.component.less'] + templateUrl: 'result-info-dialog.component.html', + styleUrls: ['../base-info-dialog.component.less'] }) export class ResultInfoDialogComponent { decodedResult: string; diff --git a/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.html b/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.html new file mode 100644 index 0000000..e69de29 diff --git a/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.less b/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.less new file mode 100644 index 0000000..e69de29 diff --git a/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.ts b/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.ts new file mode 100644 index 0000000..24e4327 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/global-error/global-error.component.ts @@ -0,0 +1,34 @@ +import { Component, OnInit } from '@angular/core'; +import { MatSnackBar, MatSnackBarConfig } from '@angular/material/snack-bar'; +import { ErrorService } from 'src/app/services/error.service'; + +@Component({ + selector: 'global-error', + templateUrl: './global-error.component.html', + styleUrls: ['./global-error.component.less'] +}) +export class GlobalErrorComponent implements OnInit { + + constructor( + private snackBar: MatSnackBar, + private errorService: ErrorService + ) { } + + ngOnInit() { + this.errorService.error$.subscribe(err => { + const _config = (duration: number): MatSnackBarConfig => { + return { + horizontalPosition: 'right', + verticalPosition: 'bottom', + duration + } + } + const error = true; + const cfg = error ? _config(0) : _config(2000) + + if (err != '') { + this.snackBar.open(err, 'Ok', cfg) + } + }) + } +} diff --git a/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html new file mode 100644 index 0000000..4e4fc7c --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.html @@ -0,0 +1,6 @@ + + Payload data + + + \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.less b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.less new file mode 100644 index 0000000..e69de29 diff --git a/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts new file mode 100644 index 0000000..e961bb2 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/payload-overview/payload-overview.component.ts @@ -0,0 +1,21 @@ +import { Component, Input, OnInit } from '@angular/core'; + +@Component({ + selector: 'payload-overview', + templateUrl: './payload-overview.component.html', + styleUrls: ['./payload-overview.component.less'] +}) +export class PayloadOverviewComponent implements OnInit { + @Input() payload: number[] | null = null; + @Input("preview") isPreview = true; + isTooBigPayload = false; + decodedPayload = ""; + + ngOnInit() { + if (this.payload !== null) { + this.decodedPayload = new TextDecoder().decode(new Uint8Array(this.payload)) + } else { + this.isTooBigPayload = true + } + } +} diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.html b/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.html similarity index 97% rename from bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.html rename to bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.html index fbaaba9..78aa311 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.html +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.html @@ -6,7 +6,7 @@ Filter - + diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts new file mode 100644 index 0000000..89e0b4b --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/agent-table/agent-table.component.ts @@ -0,0 +1,42 @@ +import { Component, OnInit } from '@angular/core'; +import { TableComponent } from '../base-table/base-table.component'; +import { AgentModel, Area } from '../../../models'; +import { AssignJobDialogComponent, AgentInfoDialogComponent } from '../../dialogs'; + +@Component({ + selector: 'agent-table', + templateUrl: './agent-table.component.html', + styleUrls: ['../base-table/base-table.component.less'], +}) +export class AgentComponent extends TableComponent implements OnInit { + area = 'agents' as Area + displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions'] + + showItemDialog(id: string) { + this.dataSource.getAgent(id).subscribe(resp => { + const dialog = this.infoDialog.open(AgentInfoDialogComponent, { + data: resp, + width: '1000px', + }); + + const saveSub = dialog.componentInstance.onSave.subscribe(result => { + this.dataSource.updateAgent(result).subscribe(_ => { + alert('Saved') + this.loadTableData() + }) + }) + + dialog.afterClosed().subscribe(result => { + saveSub.unsubscribe() + this.router.navigate(['.'], { relativeTo: this.route }) + }) + }) + } + + assignJobs(id: string) { + const dialog = this.infoDialog.open(AssignJobDialogComponent, { + data: id, + width: '1000px', + }); + } +} diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/table.component.less b/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.less similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/tables/table.component.less rename to bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.less diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts new file mode 100644 index 0000000..42a9d93 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/base-table/base-table.component.ts @@ -0,0 +1,59 @@ +import { OnInit, Directive, Component } from '@angular/core'; +import { ApiTableService } from '../../..'; +import { MatTableDataSource } from '@angular/material/table'; +import { MatDialog } from '@angular/material/dialog'; +import { ApiModel, Area } from '../../../models'; +import { ActivatedRoute, Router } from '@angular/router'; + +@Directive() +export abstract class TableComponent implements OnInit { + abstract area: Area; + table_data: MatTableDataSource = new MatTableDataSource; + isLoadingResults = true; + + constructor( + public dataSource: ApiTableService, + public infoDialog: MatDialog, + public route: ActivatedRoute, + public router: Router, + ) { } + + ngOnInit() { + this.loadTableData(); + this.route.queryParams.subscribe(params => { + const id = params['id'] + const new_item = params['new'] + if (id) { + this.showItemDialog(id); + } + if (new_item) { + this.showItemDialog(null); + } + }) + //interval(10000).subscribe(_ => this.loadTableData()); + } + + loadTableData() { + this.isLoadingResults = true; + + this.dataSource.getMany(this.area).subscribe(resp => { + this.isLoadingResults = false; + this.table_data.data = resp; + }) + } + + applyFilter(event: Event) { + const filterValue = (event.target as HTMLInputElement).value; + this.table_data.filter = filterValue.trim().toLowerCase(); + } + + deleteItem(id: string) { + if (confirm(`Delete ${id}?`)) { + this.dataSource.delete(id, this.area).subscribe(_ => { }) + this.loadTableData() + } + } + + abstract displayedColumns: string[]; + abstract showItemDialog(id: string | null): void; +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/index.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/index.ts new file mode 100644 index 0000000..cf8d91d --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/index.ts @@ -0,0 +1,5 @@ +export * from './agent-table/agent-table.component'; +export * from './base-table/base-table.component'; +export * from './job-table/job-table.component'; +export * from './payload-table/payload-table.component'; +export * from './result-table/result-table.component'; \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/job.component.html b/bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.html similarity index 96% rename from bin/u_panel/src/gui/fe/src/app/core/tables/job.component.html rename to bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.html index bfd99b2..fc53ad0 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/job.component.html +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/job-table/job-table.component.html @@ -6,7 +6,7 @@ Filter - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Name + {{row.name}} + MIME-type + {{row.mime_type}} + Size + {{row.size}} + + + | + +
No data
+ + + + \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts new file mode 100644 index 0000000..fd6461c --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/payload-table/payload-table.component.ts @@ -0,0 +1,67 @@ +import { Component } from '@angular/core'; +import { Area } from 'src/app/models'; +import { NewPayloadModel, PayloadModel } from 'src/app/models/payload.model'; +import { PayloadInfoDialogComponent } from '../../dialogs'; +import { NewPayloadDialogComponent } from '../../dialogs/new-payload-dialog/new-payload-dialog.component'; +import { TableComponent } from '../base-table/base-table.component'; + +@Component({ + selector: 'payload-table', + templateUrl: './payload-table.component.html', + styleUrls: ['../base-table/base-table.component.less'], + providers: [{ provide: 'area', useValue: 'payloads' }] +}) +export class PayloadComponent extends TableComponent { + area = 'payloads' as Area + displayedColumns = ["name", "mime_type", "size", 'actions']; + + showItemDialog(id: string | null) { + if (id === null) { + const payload: NewPayloadModel = { + name: "", + data: [] + } + const dialog = this.infoDialog.open(NewPayloadDialogComponent, { + data: payload, + width: '1000px', + }); + + dialog.componentInstance.onSave.subscribe(result => { + this.dataSource.createPayload(result) + .subscribe(_ => { + alert("Created") + this.loadTableData() + }) + + dialog.close() + }) + + dialog.afterClosed().subscribe(_ => { + this.router.navigate(['.'], { relativeTo: this.route }) + }) + + } else { + this.dataSource.getPayload(id as string).subscribe(resp => { + const dialog = this.infoDialog.open(PayloadInfoDialogComponent, { + data: resp, + width: '1000px', + }); + + const saveSub = dialog.componentInstance.onSave.subscribe(result => { + this.dataSource.updatePayload(result) + .subscribe(_ => { + alert("Updated") + this.loadTableData() + }) + dialog.close() + }) + + dialog.afterClosed().subscribe(_ => { + saveSub.unsubscribe() + this.router.navigate(['.'], { relativeTo: this.route }) + }) + }) + } + } + +} diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/result.component.html b/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.html similarity index 95% rename from bin/u_panel/src/gui/fe/src/app/core/tables/result.component.html rename to bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.html index afda1cb..08296e6 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/result.component.html +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.html @@ -6,7 +6,7 @@ Filter - + @@ -49,7 +49,7 @@ - ID + Last updated {{row.updated.secs_since_epoch * 1000| date:'long'}} diff --git a/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts b/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts new file mode 100644 index 0000000..511c62e --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/components/tables/result-table/result-table.component.ts @@ -0,0 +1,36 @@ +import { Component, OnInit } from '@angular/core'; +import { TableComponent } from '../base-table/base-table.component'; +import { Area, ResultModel } from '../../../models'; +import { ResultInfoDialogComponent } from '../../dialogs'; + +@Component({ + selector: 'results-table', + templateUrl: './result-table.component.html', + styleUrls: ['../base-table/base-table.component.less'], + providers: [{ provide: 'area', useValue: 'map' }] +}) +export class ResultComponent extends TableComponent { + area = 'map' as Area + displayedColumns = [ + 'id', + 'alias', + 'agent_id', + 'job_id', + 'state', + 'last_updated', + 'actions' + ]; + + showItemDialog(id: string) { + this.dataSource.getResult(id).subscribe(resp => { + const dialog = this.infoDialog.open(ResultInfoDialogComponent, { + data: resp, + width: '1000px', + }); + + dialog.afterClosed().subscribe(_ => { + this.router.navigate(['.'], { relativeTo: this.route }) + }) + }) + } +} diff --git a/bin/u_panel/src/gui/fe/src/app/core/models/index.ts b/bin/u_panel/src/gui/fe/src/app/core/models/index.ts deleted file mode 100644 index dad077a..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/models/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -export * from './agent.model'; -export * from './result.model'; -export * from './job.model'; - -export interface UTCDate { - secs_since_epoch: number, - nanos_since_epoch: number -} - -export abstract class ApiModel { } - -export interface Empty extends ApiModel { } - -export type Area = "agents" | "jobs" | "map"; \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/models/job.model.ts b/bin/u_panel/src/gui/fe/src/app/core/models/job.model.ts deleted file mode 100644 index 2b84d70..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/models/job.model.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { ApiModel } from "."; - -export interface JobModel extends ApiModel { - alias: string, - argv: string, - id: string, - exec_type: string, - platform: string, - payload: number[] | null, - schedule: string | null, -} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/services/api.service.ts b/bin/u_panel/src/gui/fe/src/app/core/services/api.service.ts deleted file mode 100644 index 84348e2..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/services/api.service.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { environment } from 'src/environments/environment'; -import { HttpClient } from '@angular/common/http'; -import { firstValueFrom } from 'rxjs'; -import { ApiModel, Empty, Area } from '../models'; - -interface ServerResponse { - status: "ok" | "err", - data: T | string -} - -export class ApiTableService { - area: Area; - - constructor(private http: HttpClient, area: Area) { - this.area = area; - } - - requestUrl = `${environment.server}/cmd/`; - - async req(cmd: string): Promise> { - return await firstValueFrom(this.http.post>(this.requestUrl, cmd)) - } - - async getOne(id: string, area: string = this.area): Promise> { - const resp = await this.req(`${area} read ${id}`) - if (resp.data.length === 0) { - return { - status: 'err', - data: `${id} not found in ${area}` - } - } - return { - status: resp.status, - data: resp.data[0] - } - } - - async getMany(): Promise> { - return await this.req(`${this.area} read`) - } - - async update(item: T): Promise> { - return await this.req(`${this.area} update '${JSON.stringify(item)}'`) - } - - async delete(id: string): Promise> { - return await this.req(`${this.area} delete ${id}`) - } - - async create(item: string): Promise> { - return await this.req(`${this.area} create ${item}`) - } -} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.ts deleted file mode 100644 index c72b8dc..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/agent.component.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { Component, OnDestroy, OnInit } from '@angular/core'; -import { TablesComponent } from './table.component'; -import { AgentModel } from '../models'; -import { AgentInfoDialogComponent } from './dialogs/agent_info.component'; -import { HttpErrorResponse } from '@angular/common/http'; -import { AssignJobDialogComponent } from './dialogs'; - -@Component({ - selector: 'agent-table', - templateUrl: './agent.component.html', - styleUrls: ['./table.component.less'] -}) -export class AgentComponent extends TablesComponent implements OnInit { - - //dialogSubscr!: Subscription; - area = 'agents' as const; - - displayedColumns = ['id', 'alias', 'username', 'hostname', 'last_active', 'actions'] - - show_item_dialog(id: string) { - this.data_source!.getOne(id).then(resp => { - if (resp.status === 'ok') { - const dialog = this.infoDialog.open(AgentInfoDialogComponent, { - data: resp.data as AgentModel, - width: '1000px', - }); - - const saveSub = dialog.componentInstance.onSave.subscribe(result => { - this.data_source!.update(result).then(_ => { - this.openSnackBar('Saved', false) - this.loadTableData() - }) - .catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) - }) - - dialog.afterClosed().subscribe(result => { - saveSub.unsubscribe() - this.router.navigate(['.'], { relativeTo: this.route }) - }) - } else { - this.openSnackBar(resp.data) - } - }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) - } - - assignJobs(id: string) { - const dialog = this.infoDialog.open(AssignJobDialogComponent, { - data: id, - width: '1000px', - }); - } -} diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/assign_job.component.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/assign_job.component.ts deleted file mode 100644 index cd278d6..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/assign_job.component.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Component, Inject } from '@angular/core'; -import { MAT_DIALOG_DATA } from '@angular/material/dialog'; -import { HttpClient } from '@angular/common/http'; -import { ApiTableService } from '../../services'; -import { JobModel } from '../../models'; -import { MatListOption } from '@angular/material/list'; - -@Component({ - selector: 'assign-job-dialog', - templateUrl: 'assign-job-dialog.html', - styleUrls: [] -}) -export class AssignJobDialogComponent { - rows: string[] = []; - selected_rows: string[] = []; - - constructor(@Inject(MAT_DIALOG_DATA) public agent_id: string, private http: HttpClient) { - new ApiTableService(http, "jobs").getMany().then(result => { - if (result.status == "ok") { - const jobs = result.data as JobModel[] - this.rows = jobs.map(j => `${j.id} ${j.alias}`) - } else { - alert(result.data as string) - } - }).catch(err => alert(err)) - } - - assignSelectedJobs() { - const job_ids = this.selected_rows.map(row => row.split(' ', 1)[0]).join(' '); - const request = `${this.agent_id} ${job_ids}` - new ApiTableService(this.http, "map").create(request).catch(err => alert(err)) - } -} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/index.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/index.ts deleted file mode 100644 index 4bdb1aa..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './agent_info.component'; -export * from './result_info.component'; -export * from './job_info.component'; -export * from './assign_job.component'; \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job-info-dialog.html b/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job-info-dialog.html deleted file mode 100644 index 7bb97f6..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job-info-dialog.html +++ /dev/null @@ -1,44 +0,0 @@ -

Job info

-

Editing job info

- -
- - ID - - - - Alias - - - - Args - - -
-
- - Type - - - - Platform - - - - Schedule - - -
-
- - Payload - - -
-
- - - - - \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job_info.component.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job_info.component.ts deleted file mode 100644 index 3433cbf..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/dialogs/job_info.component.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { Component, Inject } from '@angular/core'; -import { MAT_DIALOG_DATA } from '@angular/material/dialog'; -import { EventEmitter } from '@angular/core'; -import { JobModel } from '../../models/job.model'; - -@Component({ - selector: 'job-info-dialog', - templateUrl: 'job-info-dialog.html', - styleUrls: ['info-dialog.component.less'] -}) -export class JobInfoDialogComponent { - is_preview = true; - decodedPayload: string; - onSave = new EventEmitter(); - - constructor(@Inject(MAT_DIALOG_DATA) public data: JobModel) { - if (data.payload !== null) { - this.decodedPayload = new TextDecoder().decode(new Uint8Array(data.payload)) - } else { - this.decodedPayload = "" - } - } - - updateJob() { - if (this.decodedPayload.length > 0) { - this.data.payload = Array.from(new TextEncoder().encode(this.decodedPayload)) - } - this.onSave.emit(this.data); - } -} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/index.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/index.ts deleted file mode 100644 index 11dfaf2..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './agent.component'; -export * from './job.component'; -export * from './result.component'; \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/job.component.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/job.component.ts deleted file mode 100644 index 5b474c7..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/job.component.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { Component, OnInit } from '@angular/core'; -import { TablesComponent } from './table.component'; -import { JobModel } from '../models'; -import { JobInfoDialogComponent } from './dialogs'; -import { HttpErrorResponse } from '@angular/common/http'; - -@Component({ - selector: 'job-table', - templateUrl: './job.component.html', - styleUrls: ['./table.component.less'] -}) -export class JobComponent extends TablesComponent { - area = 'jobs' as const; - displayedColumns = ['id', 'alias', 'platform', 'schedule', 'exec_type', 'actions'] - - show_item_dialog(id: string | null) { - const show_dlg = (id: string, edit: boolean) => { - this.data_source!.getOne(id).then(resp => { - if (resp.status === 'ok') { - var dialog = this.infoDialog.open(JobInfoDialogComponent, { - data: resp.data as JobModel, - width: '1000px', - }); - if (edit) { - dialog.componentInstance.is_preview = false - } - - const saveSub = dialog.componentInstance.onSave.subscribe(result => { - this.data_source!.update(result) - .then(_ => { - this.openSnackBar("Saved", false) - this.loadTableData() - }) - .catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) - }) - - dialog.afterClosed().subscribe(result => { - saveSub.unsubscribe() - this.router.navigate(['.'], { relativeTo: this.route }) - }) - } else { - this.openSnackBar(resp.data) - } - }).catch((err: any) => this.openSnackBar(err)) - } - - if (id) { - show_dlg(id, false) - } else { - this.data_source!.create('"{}"').then(resp => { - if (resp.status === 'ok') { - show_dlg(resp.data[0], true) - } else { - this.openSnackBar(resp.data) - } - }).catch((err: HttpErrorResponse) => this.openSnackBar(err.error)) - } - } -} diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/result.component.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/result.component.ts deleted file mode 100644 index ec98ab7..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/result.component.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Component, OnInit } from '@angular/core'; -import { TablesComponent } from './table.component'; -import { ResultModel } from '../models'; -import { ResultInfoDialogComponent } from './dialogs'; -import { HttpErrorResponse } from '@angular/common/http'; - -@Component({ - selector: 'results-table', - templateUrl: './result.component.html', - styleUrls: ['./table.component.less'] -}) -export class ResultComponent extends TablesComponent { - area = 'map' as const; - - displayedColumns = [ - 'id', - 'alias', - 'agent_id', - 'job_id', - 'state', - 'last_updated', - 'actions' - ]; - - show_item_dialog(id: string) { - this.data_source!.getOne(id).then(resp => { - if (resp.status === 'ok') { - const dialog = this.infoDialog.open(ResultInfoDialogComponent, { - data: resp.data as ResultModel, - width: '1000px', - }); - - dialog.afterClosed().subscribe(result => { - this.router.navigate(['.'], { relativeTo: this.route }) - }) - } else { - this.openSnackBar(resp.data) - } - }).catch((err: HttpErrorResponse) => this.openSnackBar(err.message)) - } -} diff --git a/bin/u_panel/src/gui/fe/src/app/core/tables/table.component.ts b/bin/u_panel/src/gui/fe/src/app/core/tables/table.component.ts deleted file mode 100644 index 5a7bd23..0000000 --- a/bin/u_panel/src/gui/fe/src/app/core/tables/table.component.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { OnInit, Directive } from '@angular/core'; -import { HttpClient } from '@angular/common/http'; -import { ApiTableService } from '../'; -import { MatTableDataSource } from '@angular/material/table'; -import { MatDialog } from '@angular/material/dialog'; -import { ApiModel, Area } from '../models'; -import { ActivatedRoute, Router } from '@angular/router'; -import { interval } from 'rxjs'; -import { MatSnackBar, MatSnackBarConfig } from '@angular/material/snack-bar'; - -@Directive() -export abstract class TablesComponent implements OnInit { - abstract area: Area; - data_source!: ApiTableService; - table_data!: MatTableDataSource; - - isLoadingResults = true; - - constructor( - public httpClient: HttpClient, - public infoDialog: MatDialog, - public route: ActivatedRoute, - public router: Router, - public snackBar: MatSnackBar - ) { - this.table_data = new MatTableDataSource; - } - - ngOnInit() { - this.data_source = new ApiTableService(this.httpClient, this.area); - this.loadTableData(); - this.route.queryParams.subscribe(params => { - const id = params['id'] - const new_agent = params['new'] - if (id) { - this.show_item_dialog(id); - } - if (new_agent) { - this.show_item_dialog(null); - } - }) - //interval(10000).subscribe(_ => this.loadTableData()); - } - - async loadTableData() { - this.isLoadingResults = true; - //possibly needs try/catch - const data = await this.data_source!.getMany(); - this.isLoadingResults = false; - - if (typeof data.data !== 'string') { - this.table_data.data = data.data - } else { - alert(`Error: ${data}`) - }; - } - - apply_filter(event: Event) { - const filterValue = (event.target as HTMLInputElement).value; - this.table_data.filter = filterValue.trim().toLowerCase(); - } - - deleteItem(id: string) { - if (confirm(`Delete ${id}?`)) { - this.data_source!.delete(id).catch(this.openSnackBar) - } - } - - openSnackBar(message: any, error: boolean = true) { - const msg = JSON.stringify(message) - const _config = (duration: number): MatSnackBarConfig => { - return { - horizontalPosition: 'right', - verticalPosition: 'bottom', - duration - } - } - const cfg = error ? _config(0) : _config(2000) - this.snackBar.open(msg, 'Ok', cfg); - } - - abstract displayedColumns: string[]; - abstract show_item_dialog(id: string | null): void; -} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/index.ts b/bin/u_panel/src/gui/fe/src/app/index.ts similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/index.ts rename to bin/u_panel/src/gui/fe/src/app/index.ts diff --git a/bin/u_panel/src/gui/fe/src/app/core/models/agent.model.ts b/bin/u_panel/src/gui/fe/src/app/models/agent.model.ts similarity index 77% rename from bin/u_panel/src/gui/fe/src/app/core/models/agent.model.ts rename to bin/u_panel/src/gui/fe/src/app/models/agent.model.ts index d798f0e..cd8ef76 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/models/agent.model.ts +++ b/bin/u_panel/src/gui/fe/src/app/models/agent.model.ts @@ -1,6 +1,6 @@ -import { UTCDate, ApiModel } from "."; +import { UTCDate } from "."; -export interface AgentModel extends ApiModel { +export interface AgentModel { alias: string | null, hostname: string, host_info: string, diff --git a/bin/u_panel/src/gui/fe/src/app/models/index.ts b/bin/u_panel/src/gui/fe/src/app/models/index.ts new file mode 100644 index 0000000..dbe93ca --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/models/index.ts @@ -0,0 +1,20 @@ +import { AgentModel } from './agent.model'; +import { JobModel } from './job.model'; +import { PayloadModel } from './payload.model'; +import { ResultModel } from './result.model'; + +export * from './agent.model'; +export * from './result.model'; +export * from './job.model'; +export * from './payload.model'; + +export interface UTCDate { + secs_since_epoch: number, + nanos_since_epoch: number +} + +export type Area = "agents" | "jobs" | "map" | "payloads"; + +export type ApiModel = AgentModel | JobModel | ResultModel | PayloadModel | Empty; + +export interface Empty { } diff --git a/bin/u_panel/src/gui/fe/src/app/models/job.model.ts b/bin/u_panel/src/gui/fe/src/app/models/job.model.ts new file mode 100644 index 0000000..96d97b7 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/models/job.model.ts @@ -0,0 +1,16 @@ +import { PayloadModel } from './' + +export interface JobModel { + alias: string | null, + argv: string, + id?: string, + exec_type: string, + target_platforms: string, + payload_id: string | null, + schedule: string | null, +} + +export interface Job { + meta: JobModel, + payload: PayloadModel | null, +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts b/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts new file mode 100644 index 0000000..368c56c --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/models/payload.model.ts @@ -0,0 +1,12 @@ +export interface PayloadModel { + id: string, + mime_type: string, + name: string, + size: number, + data: number[] | null +} + +export interface NewPayloadModel { + name: string, + data: number[] +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/core/models/result.model.ts b/bin/u_panel/src/gui/fe/src/app/models/result.model.ts similarity index 52% rename from bin/u_panel/src/gui/fe/src/app/core/models/result.model.ts rename to bin/u_panel/src/gui/fe/src/app/models/result.model.ts index c699787..eb474a6 100644 --- a/bin/u_panel/src/gui/fe/src/app/core/models/result.model.ts +++ b/bin/u_panel/src/gui/fe/src/app/models/result.model.ts @@ -1,13 +1,18 @@ -import { UTCDate, ApiModel } from "."; +import { UTCDate } from "."; -export interface ResultModel extends ApiModel { +export interface ResultModel { agent_id: string, alias: string, created: UTCDate, id: string, job_id: string, - result: number[], + result: number[] | null, state: "Queued" | "Running" | "Finished", retcode: number | null, updated: UTCDate, +} + +export interface AssignedJobByIdModel { + job_id: string, + agent_id: string } \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/services/api.service.ts b/bin/u_panel/src/gui/fe/src/app/services/api.service.ts new file mode 100644 index 0000000..b410f05 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/services/api.service.ts @@ -0,0 +1,142 @@ +import { environment } from 'src/environments/environment'; +import { HttpClient, HttpErrorResponse } from '@angular/common/http'; +import { Observable, map, catchError, throwError } from 'rxjs'; +import { ApiModel, PayloadModel, Empty, Area, AgentModel, JobModel, ResultModel, Job, NewPayloadModel, AssignedJobByIdModel } from '../models'; +import { Injectable, Inject } from '@angular/core'; +import { ErrorService } from './error.service'; + +type Status = "ok" | "err"; + +interface ServerResponse { + status: Status, + data: T | string +} + +@Injectable({ + providedIn: 'root' +}) +export class ApiTableService { + + constructor( + private http: HttpClient, + private errorService: ErrorService + ) { + } + + requestUrl = `${environment.server}/cmd/`; + + req(cmd: string): Observable> { + return this.http.post>(this.requestUrl, cmd) + } + + getOne(id: string, area: Area, brief: 'yes' | 'no' | 'auto' | null = null): Observable { + const request = `${area} read ${id}` + (brief !== null ? `-b=${brief}` : '') + const resp = this.req(request).pipe( + map(resp => { + if (resp.data.length === 0) { + return { + status: 'err' as Status, + data: `${id} not found in ${area}` + } + } + return { + status: resp.status, + data: resp.data[0] + } + })); + return this.filterErrStatus(resp) + } + + getAgent(id: string): Observable { + return this.getOne(id, 'agents') + } + + getJob(id: string): Observable { + return this.getOne(id, 'jobs') + } + + getResult(id: string): Observable { + return this.getOne(id, 'map') + } + + getPayload(id: string): Observable { + return this.getOne(id, 'payloads') + } + + getMany(area: Area): Observable { + return this.filterErrStatus(this.req(`${area} read`)) + } + + getAgents(): Observable { + return this.getMany('agents') + } + + getJobs(): Observable { + return this.getMany('jobs') + } + + getResults(): Observable { + return this.getMany('map') + } + + getPayloads(): Observable { + return this.getMany('payloads') + } + + update(item: T, area: Area): Observable { + return this.filterErrStatus(this.req(`${area} update '${JSON.stringify(item)}'`)) + } + + updateAgent(item: AgentModel): Observable { + return this.update(item, 'agents') + } + + updateJob(item: JobModel): Observable { + return this.update(item, 'jobs') + } + + updateResult(item: ResultModel): Observable { + return this.update(item, 'map') + } + + updatePayload(item: PayloadModel): Observable { + return this.update(item, 'payloads') + } + + delete(id: string, area: Area): Observable { + return this.filterErrStatus(this.req(`${area} delete ${id}`)) + } + + create(item: T | null, area: Area): Observable { + var serialized = '"{}"' + if (item) { + serialized = JSON.stringify(item); + } + return this.filterErrStatus(this.req(`${area} create '${serialized}'`)) + } + + createResult(item: AssignedJobByIdModel[]): Observable { + return this.create(item, 'map') + } + + createPayload(item: NewPayloadModel): Observable { + return this.create(item, 'payloads') + } + + filterErrStatus(obs: Observable>): Observable { + return obs.pipe( + map(r => { + if (r.status == 'err') { + throw new Error(r.data as string) + } + return r.data as R + }), + catchError(this.errorHandler.bind(this))) + } + + errorHandler(err: HttpErrorResponse, caught: any) { + var error = err.error.data !== undefined ? JSON.stringify(err.error.data) : err.message; + this.errorService.handle(error); + return throwError(() => new Error()); + } +} \ No newline at end of file diff --git a/bin/u_panel/src/gui/fe/src/app/services/error.service.ts b/bin/u_panel/src/gui/fe/src/app/services/error.service.ts new file mode 100644 index 0000000..a4d1d13 --- /dev/null +++ b/bin/u_panel/src/gui/fe/src/app/services/error.service.ts @@ -0,0 +1,17 @@ +import { Injectable } from '@angular/core'; +import { Subject } from 'rxjs'; + +@Injectable({ + providedIn: 'root' +}) +export class ErrorService { + error$ = new Subject(); + + handle(msg: string) { + this.error$.next(msg) + } + + clear() { + this.handle('') + } +} diff --git a/bin/u_panel/src/gui/fe/src/app/core/services/index.ts b/bin/u_panel/src/gui/fe/src/app/services/index.ts similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/services/index.ts rename to bin/u_panel/src/gui/fe/src/app/services/index.ts diff --git a/bin/u_panel/src/gui/fe/src/app/core/utils.ts b/bin/u_panel/src/gui/fe/src/app/utils.ts similarity index 100% rename from bin/u_panel/src/gui/fe/src/app/core/utils.ts rename to bin/u_panel/src/gui/fe/src/app/utils.ts diff --git a/bin/u_panel/src/gui/mod.rs b/bin/u_panel/src/gui/mod.rs index 5be4487..c3de15f 100644 --- a/bin/u_panel/src/gui/mod.rs +++ b/bin/u_panel/src/gui/mod.rs @@ -8,7 +8,7 @@ use futures_util::StreamExt; use rust_embed::RustEmbed; use std::borrow::Cow; use structopt::StructOpt; -use u_lib::{api::ClientHandler, unwrap_enum}; +use u_lib::{api::HttpClient, unwrap_enum}; #[derive(RustEmbed)] #[folder = "./src/gui/fe/dist/fe/"] @@ -42,7 +42,7 @@ async fn resources_adapter(path: web::Path<(String,)>) -> impl Responder { #[post("/cmd/")] async fn send_cmd( mut body: web::Payload, - client: web::Data, + client: web::Data, ) -> Result { let mut bytes = web::BytesMut::new(); @@ -65,16 +65,14 @@ async fn send_cmd( let response = if result.is_ok() { HttpResponse::Ok().body(result_string) - } else if result.is_err() { - HttpResponse::BadRequest().body(result_string) } else { - unreachable!() + HttpResponse::BadRequest().body(result_string) }; Ok(response) } -pub async fn serve(client: ClientHandler) -> anyhow::Result<()> { +pub async fn serve(client: HttpClient) -> anyhow::Result<()> { info!("Connecting to u_server..."); client.ping().await?; diff --git a/bin/u_panel/src/main.rs b/bin/u_panel/src/main.rs index f707aef..9e7234c 100644 --- a/bin/u_panel/src/main.rs +++ b/bin/u_panel/src/main.rs @@ -7,19 +7,19 @@ extern crate tracing; use anyhow::Result as AnyResult; use argparse::{process_cmd, Args}; use structopt::StructOpt; -use u_lib::api::ClientHandler; +use u_lib::api::HttpClient; use u_lib::config::AccessEnv; use u_lib::logging::init_logger; #[actix_web::main] async fn main() -> AnyResult<()> { + init_logger(None); + let env = AccessEnv::load()?; - let client = ClientHandler::new(&env.u_server, Some(env.admin_auth_token)).await?; + let client = HttpClient::new(&env.u_server, Some(env.admin_auth_token)).await?; let args = Args::from_args(); - - init_logger(None::<&str>); - let result = process_cmd(client, args).await.to_string(); + println!("{result}"); Ok(()) } diff --git a/bin/u_server/Cargo.toml b/bin/u_server/Cargo.toml index ee1e2a8..210f7a6 100644 --- a/bin/u_server/Cargo.toml +++ b/bin/u_server/Cargo.toml @@ -9,6 +9,7 @@ anyhow = { workspace = true } diesel = { workspace = true } deadpool-diesel = { workspace = true } hyper = "0.14" +mime_guess = { workspace = true } once_cell = "1.7.2" openssl = { workspace = true } serde = { workspace = true } @@ -19,6 +20,7 @@ tokio = { workspace = true, features = ["macros"] } uuid = { workspace = true, features = ["serde", "v4"] } u_lib = { path = "../../lib/u_lib", features = ["server"] } warp = { version = "0.3.1", features = ["tls"] } +serde_qs = { version = "0.12.0", features = ["warp"] } [dev-dependencies] rstest = "0.12" diff --git a/bin/u_server/src/db.rs b/bin/u_server/src/db.rs index 23b1dc1..21b6c61 100644 --- a/bin/u_server/src/db.rs +++ b/bin/u_server/src/db.rs @@ -1,9 +1,14 @@ use crate::error::Error; use diesel::{pg::PgConnection, prelude::*, result::Error as DslError, Connection}; -use u_lib::db::PgAsyncPool; -use u_lib::models::{schema, Agent, AssignedJob, JobState, ThinJobMeta}; -use u_lib::platform::Platform; -use u_lib::types::Id; +use std::collections::{HashMap, HashSet}; +use std::mem::drop; +use u_lib::{ + db::PgAsyncPool, + models::{schema, Agent, AssignedJob, AssignedJobById, Job, JobMeta, JobState, Payload}, + platform::Platform, + types::Id, +}; +use uuid::Uuid; type Result = std::result::Result; @@ -47,27 +52,40 @@ pub struct UDB<'c> { } impl UDB<'_> { - pub fn insert_jobs(&mut self, job_metas: &[ThinJobMeta]) -> Result> { + pub fn insert_jobs(&mut self, jobs: &[JobMeta]) -> Result<()> { use schema::jobs; diesel::insert_into(jobs::table) - .values(job_metas) - .get_results(self.conn) - .map(|rows| rows.iter().map(|job: &ThinJobMeta| job.id).collect()) + .values(jobs) + .execute(self.conn) + .map(drop) .map_err(with_err_ctx("Can't insert jobs")) } - pub fn get_job(&mut self, id: Id) -> Result> { - use schema::jobs; + pub fn insert_payload(&mut self, payload: &Payload) -> Result<()> { + use schema::payloads; - jobs::table + diesel::insert_into(payloads::table) + .values(payload) + .execute(self.conn) + .map(drop) + .map_err(with_err_ctx(format!("Can't insert payload {payload:?}"))) + } + + pub fn get_job(&mut self, id: Id) -> Result> { + use schema::{jobs, payloads}; + + let maybe_job_with_payload = jobs::table + .left_join(payloads::table) .filter(jobs::id.eq(id)) - .first(self.conn) + .first::<(JobMeta, Option)>(self.conn) .optional() - .map_err(with_err_ctx(format!("Can't get job {id}"))) + .map_err(with_err_ctx(format!("Can't get job {id}")))?; + + Ok(maybe_job_with_payload.map(|(job, payload)| Job { meta: job, payload })) } - pub fn get_jobs(&mut self) -> Result> { + pub fn get_jobs(&mut self) -> Result> { use schema::jobs; jobs::table @@ -75,27 +93,59 @@ impl UDB<'_> { .map_err(with_err_ctx("Can't get jobs")) } - pub fn find_job_by_alias(&mut self, alias: &str) -> Result> { - use schema::jobs; + pub fn get_payload(&mut self, id: Id) -> Result> { + use schema::payloads; - jobs::table - .filter(jobs::alias.eq(alias)) + payloads::table + .filter(payloads::id.eq(id)) .first(self.conn) .optional() - .map_err(with_err_ctx(format!("Can't find job by alias {alias}"))) + .map_err(with_err_ctx(format!("Can't get payload {id}"))) } - pub fn insert_agent(&mut self, agent: &Agent) -> Result<()> { - use schema::agents; + pub fn get_payload_by_name(&mut self, name: String) -> Result> { + use schema::payloads; - diesel::insert_into(agents::table) - .values(agent) - .on_conflict(agents::id) - .do_update() - .set(agent) - .execute(self.conn) - .map_err(with_err_ctx(format!("Can't insert agent {agent:?}")))?; - Ok(()) + payloads::table + .filter(payloads::name.eq(&name)) + .first(self.conn) + .optional() + .map_err(with_err_ctx(format!("Can't get payload by name {name}"))) + } + + pub fn get_payloads(&mut self) -> Result> { + use schema::payloads; + + payloads::table + .load(self.conn) + .map_err(with_err_ctx("Can't get payloads")) + } + + pub fn payload_exists(&mut self, payload_id: Id) -> Result { + use schema::payloads; + + payloads::table + .find(payload_id) + .first::(self.conn) + .optional() + .map(|r| r.is_some()) + .map_err(with_err_ctx("Can't check payload {payload_id}")) + } + + pub fn get_job_by_alias(&mut self, alias: &str) -> Result> { + use schema::{jobs, payloads}; + + let maybe_job_with_payload = jobs::table + .left_join(payloads::table) + .filter(jobs::alias.eq(alias)) + .first::<(JobMeta, Option)>(self.conn) + .optional() + .map_err(with_err_ctx(format!("Can't get job by alias {alias}")))?; + + Ok(maybe_job_with_payload.map(|(job, payload_meta)| Job { + meta: job, + payload: payload_meta, + })) } pub fn insert_result(&mut self, result: &AssignedJob) -> Result<()> { @@ -138,7 +188,11 @@ impl UDB<'_> { } //TODO: filters possibly could work in a wrong way, check - pub fn get_exact_jobs(&mut self, id: Option, personal: bool) -> Result> { + pub fn get_assigned_jobs( + &mut self, + id: Option, + personal: bool, + ) -> Result> { use schema::results; let mut q = results::table.into_boxed(); @@ -163,39 +217,72 @@ impl UDB<'_> { Ok(result) } - pub fn set_jobs_for_agent(&mut self, agent_id: Id, job_ids: &[Id]) -> Result> { + // todo: move to handlers + pub fn assign_jobs(&mut self, assigned_jobs: &[AssignedJobById]) -> Result<()> { use schema::{jobs, results}; - let agent_platform = match self.get_agent(agent_id)? { - Some(agent) => Platform::new(&agent.platform), - None => { - return Err(Error::ProcessingError(format!( - "Agent {agent_id} not found" - ))) - } - }; + struct JobBriefMeta { + alias: Option, + target_platform: String, + } + + let assigned_job_ids = HashSet::::from_iter(assigned_jobs.iter().map(|a| a.job_id)); + + let jobs_meta = HashMap::::from_iter( + jobs::table + .select((jobs::id, jobs::alias, jobs::target_platforms)) + .filter(jobs::id.eq_any(&assigned_job_ids)) + .load::<(Id, Option, String)>(self.conn) + .map_err(with_err_ctx(format!( + "Can't find jobs {:?}", + assigned_job_ids + )))? + .into_iter() + .map(|(id, alias, target_platform)| { + ( + id, + JobBriefMeta { + alias, + target_platform, + }, + ) + }), + ); - let jobs_meta = jobs::table - .select((jobs::id, jobs::alias, jobs::platform)) - .filter(jobs::id.eq_any(job_ids)) - .load::<(Id, Option, String)>(self.conn) - .map_err(with_err_ctx(format!("Can't find jobs {job_ids:?}")))?; + let existing_job_ids = HashSet::from_iter(jobs_meta.keys().copied()); - for meta in &jobs_meta { - if !agent_platform.matches(&meta.2) { + if assigned_job_ids != existing_job_ids { + return Err(Error::ProcessingError(format!( + "Jobs not found: {:?}", + assigned_job_ids.difference(&existing_job_ids), + ))); + } + + for ajob in assigned_jobs { + let meta = &jobs_meta[&ajob.job_id]; + let agent_platform = match self.get_agent(ajob.agent_id)? { + Some(agent) => Platform::new(&agent.platform), + None => { + return Err(Error::ProcessingError(format!( + "Agent {} not found", + ajob.agent_id + ))) + } + }; + if !agent_platform.matches(&meta.target_platform) { return Err(Error::InsuitablePlatform( agent_platform.into_string(), - meta.2.clone(), + meta.target_platform.clone(), )); } } - let job_requests = jobs_meta + let job_requests = assigned_jobs .into_iter() - .map(|(job_id, alias, _)| AssignedJob { - job_id, - agent_id, - alias, + .map(|a| AssignedJob { + job_id: a.job_id, + agent_id: a.agent_id, + alias: jobs_meta[&a.job_id].alias.clone(), ..Default::default() }) .collect::>(); @@ -203,68 +290,76 @@ impl UDB<'_> { diesel::insert_into(results::table) .values(&job_requests) .execute(self.conn) - .map_err(with_err_ctx(format!( - "Can't setup jobs {job_ids:?} for agent {agent_id:?}" - )))?; - - Ok(job_requests.iter().map(|aj| aj.id).collect()) + .map(drop) + .map_err(with_err_ctx("Can't assign jobs")) } - pub fn del_jobs(&mut self, ids: &[Id]) -> Result { + pub fn del_jobs(&mut self, ids: &[Id]) -> Result<()> { use schema::jobs; - let mut affected = 0; - for id in ids { - let deleted = diesel::delete(jobs::table) - .filter(jobs::id.eq(id)) - .execute(self.conn) - .map_err(with_err_ctx("Can't delete jobs"))?; - affected += deleted; - } - Ok(affected) + diesel::delete(jobs::table) + .filter(jobs::id.eq_any(ids)) + .execute(self.conn) + .map(drop) + .map_err(with_err_ctx("Can't delete jobs")) } - pub fn del_results(&mut self, ids: &[Id]) -> Result { + pub fn del_results(&mut self, ids: &[Id]) -> Result<()> { use schema::results; - let mut affected = 0; - for id in ids { - let deleted = diesel::delete(results::table) - .filter(results::id.eq(id)) - .execute(self.conn) - .map_err(with_err_ctx("Can't delete results"))?; - affected += deleted; - } - Ok(affected) + diesel::delete(results::table) + .filter(results::id.eq_any(ids)) + .execute(self.conn) + .map(drop) + .map_err(with_err_ctx("Can't delete results")) } - pub fn del_agents(&mut self, ids: &[Id]) -> Result { + pub fn del_agents(&mut self, ids: &[Id]) -> Result<()> { use schema::agents; - let mut affected = 0; - for id in ids { - let deleted = diesel::delete(agents::table) - .filter(agents::id.eq(id)) - .execute(self.conn) - .map_err(with_err_ctx("Can't delete agents"))?; - affected += deleted; - } - Ok(affected) + diesel::delete(agents::table) + .filter(agents::id.eq_any(ids)) + .execute(self.conn) + .map(drop) + .map_err(with_err_ctx("Can't delete agents")) } - pub fn update_agent(&mut self, agent: &Agent) -> Result<()> { - agent - .save_changes::(self.conn) - .map_err(with_err_ctx(format!("Can't update agent {agent:?}")))?; + pub fn del_payloads(&mut self, ids: &[Id]) -> Result<()> { + use schema::payloads; + + diesel::delete(payloads::table) + .filter(payloads::id.eq_any(ids)) + .execute(self.conn) + .map(drop) + .map_err(with_err_ctx("Can't delete payloads")) + } + + pub fn upsert_agent(&mut self, agent: &Agent) -> Result<()> { + use schema::agents; + + diesel::insert_into(agents::table) + .values(agent) + .on_conflict(agents::id) + .do_update() + .set(agent) + .execute(self.conn) + .map_err(with_err_ctx(format!("Can't insert agent {agent:?}")))?; Ok(()) } - pub fn update_job(&mut self, job: &ThinJobMeta) -> Result<()> { - job.save_changes::(self.conn) + pub fn update_job(&mut self, job: &JobMeta) -> Result<()> { + job.save_changes::(self.conn) .map_err(with_err_ctx(format!("Can't update job {job:?}")))?; Ok(()) } + pub fn update_payload(&mut self, payload: &Payload) -> Result<()> { + payload + .save_changes::(self.conn) + .map_err(with_err_ctx(format!("Can't update payload {payload:?}")))?; + Ok(()) + } + pub fn update_result(&mut self, result: &AssignedJob) -> Result<()> { debug!( "updating result: id = {}, job_id = {}, agent_id = {}", diff --git a/bin/u_server/src/error.rs b/bin/u_server/src/error.rs index 0d4a1d4..5fab3f7 100644 --- a/bin/u_server/src/error.rs +++ b/bin/u_server/src/error.rs @@ -1,6 +1,6 @@ use diesel::result::Error as DslError; use thiserror::Error; -use u_lib::ufs; +use u_lib::{ufs, UError}; use warp::{ http::StatusCode, reject::Reject, @@ -13,7 +13,7 @@ pub enum Error { #[error("Configs error: {0}")] ConfigError(#[from] u_lib::config::Error), - #[error("Error processing {0}")] + #[error("Processing error: {0}")] ProcessingError(String), #[error(transparent)] @@ -30,6 +30,15 @@ pub enum Error { #[error("Job cannot be ran on this platform. Expected: {0}, got: {1}")] InsuitablePlatform(String, String), + + #[error("{0}\nContext: {1}")] + Contexted(Box, String), + + #[error(transparent)] + UError(#[from] UError), + + #[error("Runtime error: {0}")] + Runtime(String), } impl Reject for Error {} @@ -56,7 +65,7 @@ impl RejResponse { pub fn internal() -> Self { Self { - message: "INTERNAL_SERVER_ERROR".to_string(), + message: "INTERNAL SERVER ERROR".to_string(), status: StatusCode::INTERNAL_SERVER_ERROR, } } @@ -67,3 +76,23 @@ impl Reply for RejResponse { with_status(self.message, self.status).into_response() } } + +impl From for Error { + fn from(e: anyhow::Error) -> Self { + let ctx = e + .chain() + .rev() + .skip(1) + .map(|cause| format!("ctx: {}", cause)) + .collect::>() + .join("\n"); + + match e.downcast::() { + Ok(err) => Error::Contexted(Box::new(err), ctx), + Err(err) => match err.downcast::() { + Ok(err) => Error::Contexted(Box::new(Error::FSError(err)), ctx), + Err(err) => Error::Runtime(err.to_string()), + }, + } + } +} diff --git a/bin/u_server/src/handlers.rs b/bin/u_server/src/handlers.rs index c1f037d..e1a8ec7 100644 --- a/bin/u_server/src/handlers.rs +++ b/bin/u_server/src/handlers.rs @@ -2,23 +2,22 @@ use std::sync::Arc; use crate::db::{PgRepo, UDB}; use crate::error::Error; -use crate::ValidJobMeta; -use u_lib::{ - jobs::{fat_meta_to_thin, thin_meta_to_fat}, - messaging::{AsMsg, Reportable}, - misc::OneOrVec, - models::*, - types::Id -}; +use serde::Deserialize; +use u_lib::{api::api_types, messaging::Reportable, models::*, types::Id}; use warp::reject::not_found; use warp::Rejection; type EndpResult = Result; +#[derive(Deserialize)] +pub struct PayloadFlags { + brief: Brief, +} + pub struct Endpoints; impl Endpoints { - pub async fn get_agents(repo: Arc, id: Option) -> EndpResult> { + pub async fn get_agents(repo: Arc, id: Option) -> EndpResult { repo.interact(move |mut db| { Ok(match id { Some(id) => { @@ -35,58 +34,125 @@ impl Endpoints { .map_err(From::from) } - pub async fn get_job(repo: Arc, id: Id) -> EndpResult { - let Some(job) = repo.interact(move |mut db| db.get_job(id)).await? else { - return Err(not_found()) + pub async fn get_job( + repo: Arc, + id: Id, + params: Option, + ) -> EndpResult { + let Some(mut job) = repo.interact(move |mut db| db.get_job(id)).await? else { + return Err(not_found()); }; - let fat_meta = thin_meta_to_fat(job).map_err(Error::from)?; - Ok(fat_meta) + Ok(match params.map(|p| p.brief) { + Some(Brief::Yes) => job, + Some(Brief::Auto) | None => { + if let Some(payload) = &mut job.payload { + payload.maybe_join_payload().map_err(Error::from)?; + } + job + } + Some(Brief::No) => { + if let Some(payload) = &mut job.payload { + payload.join_payload().map_err(Error::from)?; + } + job + } + }) } - pub async fn get_jobs(repo: Arc) -> EndpResult> { + pub async fn get_jobs(repo: Arc) -> EndpResult { repo.interact(move |mut db| db.get_jobs()) .await .map_err(From::from) } - pub async fn get_agent_jobs( + pub async fn get_assigned_jobs( repo: Arc, id: Option, - ) -> EndpResult> { - repo.interact(move |mut db| db.get_exact_jobs(id, false)) + ) -> EndpResult { + repo.interact(move |mut db| db.get_assigned_jobs(id, false)) .await .map_err(From::from) } - pub async fn get_personal_jobs(repo: Arc, id: Id) -> EndpResult> { + pub async fn get_payloads(repo: Arc) -> EndpResult { + repo.interact(move |mut db| db.get_payloads()) + .await + .map_err(From::from) + } + + pub async fn get_payload( + repo: Arc, + name_or_id: String, + params: Option, + ) -> EndpResult { + let mut payload = match repo + .interact(move |mut db| match Id::parse_str(&name_or_id) { + Ok(id) => db.get_payload(id), + Err(_) => db.get_payload_by_name(name_or_id), + }) + .await? + { + Some(p) => p, + None => return Err(not_found()), + }; + + Ok(match params.map(|p| p.brief) { + Some(Brief::Yes) => { + payload.data = None; + payload + } + None | Some(Brief::Auto) => { + payload.maybe_join_payload().map_err(Error::from)?; + payload + } + _ => { + payload.join_payload().map_err(Error::from)?; + payload + } + }) + } + + pub async fn get_personal_jobs( + repo: Arc, + agent_id: Id, + ) -> EndpResult { repo.transaction(move |mut db| { - let agent = db.get_agent(id)?; + let agent = db.get_agent(agent_id)?; match agent { Some(mut agent) => { agent.touch(); - db.update_agent(&agent)?; + db.upsert_agent(&agent)?; } None => { - let new_agent = Agent::with_id(id); + let mut new_agent = Agent::empty(); + new_agent.id = agent_id; - db.insert_agent(&new_agent)?; + db.upsert_agent(&new_agent)?; let job = db - .find_job_by_alias("agent_hello")? + .get_job_by_alias("agent_hello")? .expect("agent_hello job not found"); - db.set_jobs_for_agent(id, &[job.id])?; + let assigned_job = AssignedJobById { + agent_id, + job_id: job.meta.id, + ..Default::default() + }; + db.assign_jobs(&[assigned_job])?; } } - let assigned_jobs = db.get_exact_jobs(Some(id), true)?; + let assigned_jobs = db.get_assigned_jobs(Some(agent_id), true)?; for job in &assigned_jobs { db.update_job_status(job.id, JobState::Running)?; } - Ok(assigned_jobs) + Ok(assigned_jobs + .into_iter() + .map(|j| AssignedJobById::from(&j)) + .collect()) }) .await .map_err(From::from) @@ -94,69 +160,91 @@ impl Endpoints { pub async fn upload_jobs( repo: Arc, - msg: Vec, - ) -> EndpResult> { - let jobs = msg + jobs: Vec, + ) -> EndpResult { + let mut checked_jobs = vec![]; + for mut job in jobs { + if let Some(payload) = &mut job.payload { + payload.maybe_split_payload().map_err(Error::from)?; + } else if let Some(pld_id) = job.meta.payload_id { + if !repo + .interact(move |mut db| db.payload_exists(pld_id)) + .await? + { + Err(Error::ProcessingError(format!( + "Payload {pld_id} not found" + )))? + } + } + checked_jobs.push(job) + } + + let (jobs, payloads_opt): (Vec<_>, Vec<_>) = checked_jobs + .into_iter() + .map(|j| (j.meta, j.payload)) + .unzip(); + + let payloads = payloads_opt .into_iter() - .map(|meta| Ok(fat_meta_to_thin(meta)?)) - .collect::, Error>>()?; + .filter_map(|p| p) + .collect::>(); - repo.interact(move |mut db| db.insert_jobs(&jobs)) + repo.transaction(move |mut db| { + for payload in payloads { + db.insert_payload(&payload)?; + } + db.insert_jobs(&jobs) + }) + .await + .map_err(From::from) + } + + pub async fn upload_payload( + repo: Arc, + raw_payload: RawPayload, + ) -> EndpResult { + let payloads = raw_payload.into_payload().map_err(Error::from)?; + + repo.interact(move |mut db| db.insert_payload(&payloads)) .await .map_err(From::from) } - pub async fn del(repo: Arc, id: Id) -> EndpResult { + pub async fn del(repo: Arc, id: Id) -> EndpResult<()> { repo.transaction(move |mut db| { - let del_fns = &[UDB::del_agents, UDB::del_jobs, UDB::del_results]; - for del_fn in del_fns { - let affected = del_fn(&mut db, &[id])?; - if affected > 0 { - return Ok(affected); - } - } - Ok(0) + [ + UDB::del_agents, + UDB::del_jobs, + UDB::del_results, + UDB::del_payloads, + ] + .iter() + .map(|f| f(&mut db, &[id])) + .collect::>() }) .await .map_err(From::from) } - pub async fn set_jobs( + pub async fn assign_jobs( repo: Arc, - agent_id: Id, - job_idents: Vec, - ) -> EndpResult> { + assigned_jobs: Vec, + ) -> EndpResult<()> { repo.transaction(move |mut db| { - job_idents - .into_iter() - .map(|ident| { - Id::parse_str(&ident).or_else(|_| { - let job_from_db = db.find_job_by_alias(&ident); - match job_from_db { - Ok(job) => match job { - Some(j) => Ok(j.id), - None => { - Err(Error::ProcessingError(format!("unknown ident {ident}"))) - } - }, - Err(e) => Err(e), - } - }) - }) - .collect::, Error>>() - .and_then(|j| db.set_jobs_for_agent(agent_id, &j)) + db.assign_jobs(&assigned_jobs)?; + Ok(()) }) .await .map_err(From::from) } - pub async fn report + AsMsg + Send + Sync + 'static>( + pub async fn report( repo: Arc, - msg: Data, - agent_id: Id - ) -> EndpResult<()> { + msg: Vec, + agent_id: Id, + ) -> EndpResult { repo.transaction(move |mut db| { - for entry in msg.into_vec() { + for entry in msg { match entry { Reportable::Assigned(mut result) => { let result_agent_id = &result.agent_id; @@ -165,13 +253,13 @@ impl Endpoints { continue; } result.touch(); - + info!("agent {agent_id} updated job {}", result.id); match result.exec_type { JobType::Init => { result.state = JobState::Finished; - + match &result.result { Some(rbytes) => { let mut agent: Agent = match serde_json::from_slice(&rbytes) { @@ -182,7 +270,7 @@ impl Endpoints { } }; agent.state = AgentState::Active; - db.insert_agent(&agent)?; + db.upsert_agent(&agent)?; } None => error!("Empty agent data"), }}, @@ -209,32 +297,46 @@ impl Endpoints { pub async fn update_agent( repo: Arc, agent: Agent, - ) -> EndpResult<()> { - repo.interact(move |mut db| db.update_agent(&agent)) - .await?; - Ok(()) + ) -> EndpResult { + repo.interact(move |mut db| db.upsert_agent(&agent)) + .await + .map_err(From::from) } - pub async fn update_job( - repo: Arc, - job: ValidJobMeta, - ) -> EndpResult<()> { - let thin_meta = fat_meta_to_thin(job).map_err(Error::from)?; - repo.interact(move |mut db| db.update_job(&thin_meta)) - .await?; - Ok(()) + pub async fn update_job(repo: Arc, job: JobMeta) -> EndpResult { + repo.interact(move |mut db| db.update_job(&job.validate()?)) + .await + .map_err(From::from) } pub async fn update_assigned_job( repo: Arc, assigned: AssignedJob, - ) -> EndpResult<()> { + ) -> EndpResult { repo.interact(move |mut db| db.update_result(&assigned)) - .await?; - Ok(()) + .await + .map_err(From::from) } - pub async fn download(_file_id: String) -> EndpResult> { - todo!() + pub async fn update_payload( + repo: Arc, + payload: Payload, + ) -> EndpResult { + debug!("update payload: {payload:?}"); + match payload.data { + Some(data) => { + let mut well_formed_payload = + Payload::from_data(data, Some(payload.name)).map_err(Error::from)?; + well_formed_payload.id = payload.id; + + repo.interact(move |mut db| db.update_payload(&well_formed_payload)) + .await + .map_err(From::from) + } + None => repo + .interact(move |mut db| db.update_payload(&payload)) + .await + .map_err(From::from), + } } } diff --git a/bin/u_server/src/u_server.rs b/bin/u_server/src/u_server.rs index 34b37d4..0018463 100644 --- a/bin/u_server/src/u_server.rs +++ b/bin/u_server/src/u_server.rs @@ -1,21 +1,17 @@ #[macro_use] extern crate tracing; -#[cfg(test)] -#[macro_use] -extern crate rstest; - mod db; mod error; mod handlers; +use crate::handlers::{Endpoints, PayloadFlags}; use db::PgRepo; use error::{Error as ServerError, RejResponse}; use std::{convert::Infallible, sync::Arc}; use u_lib::{ config, db::async_pool, - jobs::fat_meta_to_thin, messaging::{AsMsg, Reportable}, models::*, types::Id, @@ -23,13 +19,11 @@ use u_lib::{ use warp::{ body, log::{custom, Info}, - reply::{json, reply, Json, Response}, + reply::{json, Json, Response}, Filter, Rejection, Reply, }; -use crate::handlers::Endpoints; - -type ValidJobMeta = FatJobMeta; +const DEFAULT_RESPONSE: &str = "null"; fn into_message(msg: M) -> Json { json(&msg) @@ -39,8 +33,15 @@ pub fn init_endpoints( auth_token: &str, db: PgRepo, ) -> impl Filter + Clone { + fn make_optional( + f: impl Filter + Clone, + ) -> impl Filter,), Error = Infallible> + Clone { + f.map(Some) + .or_else(|_| async { Ok::<(Option,), Infallible>((None,)) }) + } + let path = |p: &'static str| warp::post().and(warp::path(p)); - let infallible_none = |_| async { Result::<(Option,), Infallible>::Ok((None,)) }; + let create_qs_cfg = || serde_qs::Config::new(1, true); let with_db = { let adb = Arc::new(db); @@ -49,19 +50,22 @@ pub fn init_endpoints( let get_agents = path("get_agents") .and(with_db.clone()) - .and(warp::path::param::().map(Some).or_else(infallible_none)) + .and(make_optional(warp::path::param::())) .and_then(Endpoints::get_agents) .map(into_message); let upload_jobs = path("upload_jobs") .and(with_db.clone()) - .and(body::json::>()) + .and(body::json::>()) .and_then(Endpoints::upload_jobs) .map(into_message); let get_job = path("get_job") .and(with_db.clone()) .and(warp::path::param::()) + .and(make_optional(serde_qs::warp::query::( + create_qs_cfg(), + ))) .and_then(Endpoints::get_job) .map(into_message); @@ -70,10 +74,10 @@ pub fn init_endpoints( .and_then(Endpoints::get_jobs) .map(into_message); - let get_agent_jobs = path("get_agent_jobs") + let get_assigned_jobs = path("get_assigned_jobs") .and(with_db.clone()) - .and(warp::path::param::().map(Some).or_else(infallible_none)) - .and_then(Endpoints::get_agent_jobs) + .and(make_optional(warp::path::param::())) + .and_then(Endpoints::get_assigned_jobs) .map(into_message); let get_personal_jobs = path("get_personal_jobs") @@ -88,11 +92,10 @@ pub fn init_endpoints( .and_then(Endpoints::del) .map(ok); - let set_jobs = path("set_jobs") + let assign_jobs = path("assign_jobs") .and(with_db.clone()) - .and(warp::path::param::()) - .and(body::json::>()) - .and_then(Endpoints::set_jobs) + .and(body::json::>()) + .and_then(Endpoints::assign_jobs) .map(into_message); let report = path("report") @@ -110,7 +113,7 @@ pub fn init_endpoints( let update_job = path("update_job") .and(with_db.clone()) - .and(body::json::()) + .and(body::json::()) .and_then(Endpoints::update_job) .map(ok); @@ -120,12 +123,33 @@ pub fn init_endpoints( .and_then(Endpoints::update_assigned_job) .map(ok); - let download = path("download") + let get_payloads = path("get_payloads") + .and(with_db.clone()) + .and_then(Endpoints::get_payloads) + .map(into_message); + + let get_payload = path("get_payload") + .and(with_db.clone()) .and(warp::path::param::()) - .and_then(Endpoints::download) + .and(make_optional(serde_qs::warp::query::( + create_qs_cfg(), + ))) + .and_then(Endpoints::get_payload) + .map(into_message); + + let upload_payload = path("upload_payload") + .and(with_db.clone()) + .and(body::json::()) + .and_then(Endpoints::upload_payload) + .map(ok); + + let update_payload = path("update_payload") + .and(with_db.clone()) + .and(body::json::()) + .and_then(Endpoints::update_payload) .map(ok); - let ping = path("ping").map(reply); + let ping = path("ping").map(|| DEFAULT_RESPONSE); let auth_token = format!("Bearer {auth_token}",).into_boxed_str(); let auth_header = warp::header::exact("authorization", Box::leak(auth_token)); @@ -133,20 +157,21 @@ pub fn init_endpoints( let auth_zone = (get_agents .or(get_job.clone()) .or(get_jobs.clone()) + .or(get_payloads) + .or(get_payload) .or(upload_jobs) + .or(upload_payload) .or(del) - .or(set_jobs) - .or(get_agent_jobs) - .or(update_agent.or(update_job).or(update_assigned_job)) - .or(download) + .or(assign_jobs) + .or(get_assigned_jobs) + .or(update_agent) + .or(update_job) + .or(update_assigned_job) + .or(update_payload) .or(ping)) .and(auth_header); - let agent_zone = get_job - .or(get_jobs) - .or(get_personal_jobs) - .or(report) - .or(download); + let agent_zone = get_job.or(get_jobs).or(get_personal_jobs).or(report); auth_zone.or(agent_zone) } @@ -154,16 +179,14 @@ pub fn init_endpoints( pub async fn preload_jobs(repo: &PgRepo) -> Result<(), ServerError> { repo.interact(|mut db| { let job_alias = "agent_hello"; - let if_job_exists = db.find_job_by_alias(job_alias)?; + let if_job_exists = db.get_job_by_alias(job_alias)?; if if_job_exists.is_none() { - let agent_hello = fat_meta_to_thin( - FatJobMeta::builder() - .with_type(JobType::Init) - .with_alias(job_alias) - .build() - .unwrap(), - )?; - db.insert_jobs(&[agent_hello])?; + let agent_hello = RawJob::default() + .with_type(JobType::Init) + .with_alias(job_alias) + .try_into_job() + .unwrap(); + db.insert_jobs(&[agent_hello.meta])?; } Ok(()) }) @@ -219,13 +242,13 @@ fn logger(info: Info<'_>) { .take(2) .collect::() ) - .unwrap_or_else(|| "NO_AGENT".to_string()), + .unwrap_or_else(|| "NO_AGENT_UID".to_string()), status = info.status() ); } fn ok(_: T) -> impl Reply { - reply() + DEFAULT_RESPONSE } /* diff --git a/deploy/podman-compose.yml b/deploy/podman-compose.yml index 5e27513..d8edd93 100644 --- a/deploy/podman-compose.yml +++ b/deploy/podman-compose.yml @@ -18,14 +18,14 @@ services: u_db: condition: service_healthy ports: - - 63714:63714 + - 9990:9990 env_file: - ./.env - ./.env.private environment: RUST_LOG: warp=info,u_server_lib=debug healthcheck: - test: ss -tlpn | grep 63714 + test: ss -tlpn | grep 9990 interval: 5s timeout: 2s retries: 2 diff --git a/images/tests_runner.Dockerfile b/images/tests_runner.Dockerfile index 35355c5..7819ddf 100644 --- a/images/tests_runner.Dockerfile +++ b/images/tests_runner.Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.67 +FROM rust:1.72 RUN rustup target add x86_64-unknown-linux-musl RUN mkdir -p /tests && chmod 777 /tests diff --git a/images/u_server.Dockerfile b/images/u_server.Dockerfile index 122ef4d..d9f21d6 100644 --- a/images/u_server.Dockerfile +++ b/images/u_server.Dockerfile @@ -1,3 +1,3 @@ FROM alpine:3.17 -RUN apk add iproute2 bash \ No newline at end of file +RUN apk add iproute2 bash file \ No newline at end of file diff --git a/integration/Cargo.lock b/integration-tests/Cargo.lock similarity index 100% rename from integration/Cargo.lock rename to integration-tests/Cargo.lock diff --git a/integration/Cargo.toml b/integration-tests/Cargo.toml similarity index 78% rename from integration/Cargo.toml rename to integration-tests/Cargo.toml index 706d97d..eedeb02 100644 --- a/integration/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "integration" +name = "integration-tests" version = "0.1.0" authors = ["plazmoid "] edition = "2021" @@ -7,18 +7,19 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +ctor = "0.2.0" once_cell = "1.10.0" reqwest = { workspace = true } -rstest = "0.12" +rstest = "0.17" serde = { workspace = true } serde_json = { workspace = true } shlex = "1.0.0" tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "time"] } tracing = { workspace = true } uuid = { workspace = true, features = ["serde", "v4"] } -u_lib = { path = "../lib/u_lib", features = ["panel"] } +u_lib = { path = "../lib/u_lib", features = ["panel", "server"] } [[test]] -name = "integration" +name = "integration-tests" path = "tests/lib.rs" diff --git a/integration/docker-compose.yml b/integration-tests/docker-compose.yml similarity index 78% rename from integration/docker-compose.yml rename to integration-tests/docker-compose.yml index 8d82795..093312d 100644 --- a/integration/docker-compose.yml +++ b/integration-tests/docker-compose.yml @@ -23,17 +23,17 @@ services: u_db: condition: service_healthy ports: - - 63714:63714 + - 9990:9990 env_file: - ../.env - ../.env.private environment: - RUST_LOG: warp=info,u_server_lib=debug + RUST_LOG: warp=info,u_server_lib=debug,u_lib=debug healthcheck: - test: ss -tlpn | grep 63714 - interval: 5s - timeout: 2s - retries: 2 + test: ss -tlpn | grep 9990 + interval: 5s + timeout: 2s + retries: 2 u_db: image: localhost/unki/u_db @@ -51,11 +51,11 @@ services: target: /u_db_entrypoint.sh command: /u_db_entrypoint.sh healthcheck: - # test if db's port is open and db is created - test: ss -tlpn | grep 5432 && psql -lqt -U $${POSTGRES_USER} | grep -qw $${POSTGRES_DATABASE} - interval: 5s - timeout: 5s - retries: 3 + # test if db's port is open and db is created + test: ss -tlpn | grep 5432 && psql -lqt -U $${POSTGRES_USER} | grep -qw $${POSTGRES_DATABASE} + interval: 5s + timeout: 5s + retries: 3 u_agent: user: *user @@ -83,13 +83,13 @@ services: volumes: - ${HOME}/.cargo/registry/:/usr/local/cargo/registry/ - ../__Cargo_integration.toml:/tests/Cargo.toml - - ./:/tests/integration/ + - ./:/tests/integration-tests/ - ../certs:/tests/certs - ../target/x86_64-unknown-linux-musl/${PROFILE:-debug}/u_panel:/u_panel - ../lib/u_lib:/tests/lib/u_lib - - ../logs:/tests/integration/logs:rw + - ../logs:/tests/integration-tests/logs:rw working_dir: - /tests/integration/ + /tests/integration-tests/ depends_on: u_agent: condition: service_started @@ -100,4 +100,5 @@ services: - ../.env.private environment: RUST_BACKTRACE: 1 + RUST_LOG: hyper=info,reqwest=info U_SERVER: u_server \ No newline at end of file diff --git a/integration/docker.py b/integration-tests/docker.py similarity index 98% rename from integration/docker.py rename to integration-tests/docker.py index 2103103..06a4fa6 100644 --- a/integration/docker.py +++ b/integration-tests/docker.py @@ -90,7 +90,7 @@ class Compose: ] def __init__(self): - self.container_tpl = 'integration-%s-%d' + self.container_tpl = 'integration-tests-%s-%d' self.cmd_container = self.container_tpl % ('tests_runner', 1) self.ALL_CONTAINERS = [self.container_tpl % (c, 1) for c in self.ALL_IMAGES] diff --git a/integration/integration_tests.py b/integration-tests/integration_tests.py similarity index 75% rename from integration/integration_tests.py rename to integration-tests/integration_tests.py index 4b1ab63..bc98e33 100644 --- a/integration/integration_tests.py +++ b/integration-tests/integration_tests.py @@ -6,7 +6,7 @@ from docker import rebuild_images_if_needed, Compose from pathlib import Path from utils import * -CARGO_INTEGRATION_TOML = Path('../__Cargo_integration.toml') +CARGO_INTEGRATION_TESTS_TOML = Path('../__Cargo_integration.toml') CLUSTER = Compose() @@ -22,12 +22,12 @@ def usage_exit(): fail(usage) -def create_integration_workspace(): - if CARGO_INTEGRATION_TOML.exists(): - CARGO_INTEGRATION_TOML.unlink() +def create_integration_tests_workspace(): + if CARGO_INTEGRATION_TESTS_TOML.exists(): + CARGO_INTEGRATION_TESTS_TOML.unlink() workspace = toml.load('../Cargo.toml') - workspace['workspace']['members'] = ['integration'] - with open(CARGO_INTEGRATION_TOML, 'w') as fo: + workspace['workspace']['members'] = ['integration-tests'] + with open(CARGO_INTEGRATION_TESTS_TOML, 'w') as fo: toml.dump(workspace, fo) @@ -44,7 +44,7 @@ def run_tests(): def _cleanup(): if not preserve_containers and not only_setup_cluster: CLUSTER.down() - CARGO_INTEGRATION_TOML.unlink(missing_ok=True) + CARGO_INTEGRATION_TESTS_TOML.unlink(missing_ok=True) def abort_handler(s, _): warn(f'Received signal: {s}, gracefully stopping...') @@ -53,16 +53,16 @@ def run_tests(): if down_cluster: _cleanup() return - + for s in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): signal.signal(s, abort_handler) rebuild_images_if_needed(force_rebuild) - create_integration_workspace() + create_integration_tests_workspace() try: CLUSTER.up() CLUSTER.is_alive() if not only_setup_cluster: - CLUSTER.run('cargo test --test integration') + CLUSTER.run('cargo test --test integration-tests') except Exception as e: #CLUSTER.print_containers_logs() fail(e) diff --git a/integration/integration_tests.sh b/integration-tests/integration_tests.sh similarity index 86% rename from integration/integration_tests.sh rename to integration-tests/integration_tests.sh index b8efc7e..adf3097 100755 --- a/integration/integration_tests.sh +++ b/integration-tests/integration_tests.sh @@ -2,5 +2,7 @@ set -e export DOCKER_UID=$(id -u) export DOCKER_GID=$(id -g) + +rm ../logs/u_agent* || true [[ "$@" =~ "--release" ]] && export PROFILE=release || export PROFILE=debug python integration_tests.py $@ diff --git a/integration/src/main.rs b/integration-tests/src/main.rs similarity index 100% rename from integration/src/main.rs rename to integration-tests/src/main.rs diff --git a/integration-tests/tests/fixtures/agent.rs b/integration-tests/tests/fixtures/agent.rs new file mode 100644 index 0000000..db67502 --- /dev/null +++ b/integration-tests/tests/fixtures/agent.rs @@ -0,0 +1,38 @@ +use super::connections::*; +use super::run_async; +use u_lib::{api::HttpClient, messaging::Reportable, models::*, types::Id}; + +pub struct RegisteredAgent { + pub id: Id, +} + +#[fixture] +#[once] +pub fn registered_agent(client: &HttpClient) -> RegisteredAgent { + run_async(async { + let agent = Agent::with_current_platform(); + let agent_id = agent.id; + println!("registering agent {agent_id}"); + debug!("registering agent1 {agent_id}"); + let resp = client + .get_personal_jobs(agent_id) + .await + .unwrap() + .pop() + .unwrap(); + let job_id = resp.job_id; + let job = client.get_job(job_id, Brief::No).await.unwrap(); + + assert_eq!(job.meta.alias, Some("agent_hello".to_string())); + + let mut agent_data = AssignedJob::from((&job.meta, resp)); + agent_data.set_result(&agent); + + client + .report([Reportable::Assigned(agent_data)]) + .await + .unwrap(); + + RegisteredAgent { id: agent_id } + }) +} diff --git a/integration-tests/tests/fixtures/connections.rs b/integration-tests/tests/fixtures/connections.rs new file mode 100644 index 0000000..87ae099 --- /dev/null +++ b/integration-tests/tests/fixtures/connections.rs @@ -0,0 +1,28 @@ +use super::env::*; +use super::run_async; + +pub use u_lib::api::HttpClient; +use u_lib::db::unpooled; +pub use u_lib::db::PgConnection; + +#[fixture] +#[once] +pub fn client(env_default: EndpointsEnv) -> HttpClient { + run_async(HttpClient::new(&env_default.u_server, None)).unwrap() +} + +#[fixture] +#[once] +pub fn client_panel(env_access: AccessEnv) -> HttpClient { + run_async(HttpClient::new( + &env_access.u_server, + Some(env_access.admin_auth_token), + )) + .unwrap() +} + +#[fixture] +#[once] +pub fn db(env_db: DBEnv) -> PgConnection { + unpooled(&env_db) +} diff --git a/integration-tests/tests/fixtures/env.rs b/integration-tests/tests/fixtures/env.rs new file mode 100644 index 0000000..4738400 --- /dev/null +++ b/integration-tests/tests/fixtures/env.rs @@ -0,0 +1,16 @@ +pub use u_lib::config::{AccessEnv, DBEnv, EndpointsEnv}; + +#[fixture] +pub fn env_default() -> EndpointsEnv { + EndpointsEnv::load() +} + +#[fixture] +pub fn env_access() -> AccessEnv { + AccessEnv::load().unwrap() +} + +#[fixture] +pub fn env_db() -> DBEnv { + DBEnv::load().unwrap() +} diff --git a/integration-tests/tests/fixtures/mod.rs b/integration-tests/tests/fixtures/mod.rs new file mode 100644 index 0000000..5372b09 --- /dev/null +++ b/integration-tests/tests/fixtures/mod.rs @@ -0,0 +1,17 @@ +pub mod agent; +pub mod connections; +pub mod env; + +use std::future::Future; +use std::thread; +use tokio::runtime::Runtime; + +// tokio runtime cannot be created inside another runtime, +// so i create a separate non-'static thread not to interfere +fn run_async(fut: impl Future + Send) -> R { + thread::scope(|s| { + s.spawn(|| Runtime::new().unwrap().block_on(fut)) + .join() + .expect("async task failed") + }) +} diff --git a/integration/tests/helpers/jobs.rs b/integration-tests/tests/helpers/jobs.rs similarity index 100% rename from integration/tests/helpers/jobs.rs rename to integration-tests/tests/helpers/jobs.rs diff --git a/integration-tests/tests/helpers/mod.rs b/integration-tests/tests/helpers/mod.rs new file mode 100644 index 0000000..19ca72b --- /dev/null +++ b/integration-tests/tests/helpers/mod.rs @@ -0,0 +1,4 @@ +pub mod jobs; +pub mod panel; + +pub use panel::Panel; diff --git a/integration/tests/helpers/panel.rs b/integration-tests/tests/helpers/panel.rs similarity index 86% rename from integration/tests/helpers/panel.rs rename to integration-tests/tests/helpers/panel.rs index 865e16e..f33085b 100644 --- a/integration/tests/helpers/panel.rs +++ b/integration-tests/tests/helpers/panel.rs @@ -10,13 +10,22 @@ pub struct Panel; impl Panel { fn run(args: &[&str]) -> Output { - Command::new(PANEL_BINARY).args(args).output().unwrap() + Command::new(PANEL_BINARY) + .env("RUST_LOG", "u_lib=debug") + .args(args) + .output() + .unwrap() } pub fn output_argv(argv: &[&str]) -> PanelResult { let result = Self::run(argv); let output = ProcOutput::from_output(&result); + let stderr = output.get_stderr(); + if !stderr.is_empty() { + println!("\n{}\n", String::from_utf8_lossy(stderr)); + } + match from_slice(output.get_stdout()) { Ok(r) => r, Err(e) => { @@ -40,8 +49,8 @@ impl Panel { .as_ref(), ); match &result { - PanelResult::Ok(r) => eprintln!("+<< {r:?}"), - PanelResult::Err(e) => eprintln!("!<< {e:?}"), + PanelResult::Ok(r) => eprintln!("+<< {r:#?}"), + PanelResult::Err(e) => eprintln!("!<< {e:#?}"), } result } diff --git a/integration/tests/integration/behaviour.rs b/integration-tests/tests/integration_tests/behaviour.rs similarity index 52% rename from integration/tests/integration/behaviour.rs rename to integration-tests/tests/integration_tests/behaviour.rs index 8faa9b3..cea4f85 100644 --- a/integration/tests/integration/behaviour.rs +++ b/integration-tests/tests/integration_tests/behaviour.rs @@ -2,19 +2,17 @@ use crate::fixtures::agent::*; use crate::helpers::{jobs::retry_with_interval, Panel}; use rstest::rstest; -use serde_json::{json, to_string}; +use serde_json::to_string; use u_lib::config::AGENT_ITERATION_INTERVAL; use u_lib::models::*; -use uuid::Uuid; #[rstest] #[tokio::test] -async fn registration(#[future] register_agent: RegisteredAgent) { - let agent = register_agent.await; +async fn registration(registered_agent: &RegisteredAgent) { let agents: Vec = Panel::check_output("agents read"); - let found = agents.iter().find(|v| v.id == agent.id); + let found = agents.iter().find(|v| v.id == registered_agent.id); assert!(found.is_some()); - Panel::check_status(format!("agents delete {}", agent.id)); + Panel::check_status(format!("agents delete {}", registered_agent.id)); } #[tokio::test] @@ -22,19 +20,28 @@ async fn setup_tasks() { let agents: Vec = Panel::check_output("agents read"); let agent_id = agents[0].id; let job_alias = "passwd_contents"; - let job = json!( - {"alias": job_alias, "payload": b"cat /etc/passwd", "argv": "/bin/bash {}" } - ); + let job = RawJob::default() + .with_alias(job_alias) + .with_raw_payload("cat /etc/passwd") + .with_shell("/bin/bash {}") + .with_target_platforms("*linux*") + .try_into_job() + .unwrap(); + let job_id = job.meta.id; + + Panel::check_status(["jobs", "create", &to_string(&RawJob::from(job)).unwrap()]); - Panel::check_status(["jobs", "create", &to_string(&job).unwrap()]); + let assigned = AssignedJobById { + agent_id, + job_id, + ..Default::default() + }; - let cmd = format!("map create {} {}", agent_id, job_alias); - let assigned_ids: Vec = Panel::check_output(cmd); + Panel::check_status(["map", "create", &to_string(&[assigned]).unwrap()]); retry_with_interval(5, AGENT_ITERATION_INTERVAL, || { let result = - Panel::check_output::>(format!("map read {}", assigned_ids[0])) - .remove(0); + Panel::check_output::>(format!("map read {}", job_id)).remove(0); if result.state == JobState::Finished { eprintln!("{}", result.to_str_result()); assert!(result.to_str_result().contains("root:x:0:0")); @@ -48,26 +55,30 @@ async fn setup_tasks() { #[tokio::test] async fn large_payload() { - let agent_id = Panel::check_output::>("agents read")[0].id; - + let agent = &Panel::check_output::>("agents read")[0]; + let agent_id = agent.id; let job_alias = "large_payload"; - - let job = FatJobMeta::builder() + let job = RawJob::default() .with_alias(job_alias) .with_payload_path("./tests/bin/echoer") .with_shell("{} type echo") - .build() + .with_target_platforms(&agent.platform) + .try_into_job() .unwrap(); + let job_id = job.meta.id; - Panel::check_status(["jobs", "create", &to_string(&job).unwrap()]); + Panel::check_status(["jobs", "create", &to_string(&RawJob::from(job)).unwrap()]); - let cmd = format!("map create {agent_id} {job_alias}"); - let assigned_ids: Vec = Panel::check_output(cmd); + let assigned = AssignedJobById { + agent_id, + job_id, + ..Default::default() + }; + Panel::check_status(["map", "create", &to_string(&[assigned]).unwrap()]); retry_with_interval(5, AGENT_ITERATION_INTERVAL, || { let result = - Panel::check_output::>(format!("map read {}", assigned_ids[0])) - .remove(0); + Panel::check_output::>(format!("map read {}", job_id)).remove(0); if result.state == JobState::Finished { assert_eq!(result.to_str_result(), "type echo\n"); Ok(()) diff --git a/integration/tests/integration/connection.rs b/integration-tests/tests/integration_tests/connection.rs similarity index 73% rename from integration/tests/integration/connection.rs rename to integration-tests/tests/integration_tests/connection.rs index 156a6a1..2822f86 100644 --- a/integration/tests/integration/connection.rs +++ b/integration-tests/tests/integration_tests/connection.rs @@ -1,14 +1,15 @@ -use crate::helpers::ENV; +use crate::fixtures::env::*; use u_lib::config::MASTER_PORT; +#[rstest] #[tokio::test] -async fn non_auth_connection_dropped() { +async fn non_auth_connection_dropped(env_default: EndpointsEnv) { let client = reqwest::ClientBuilder::new() .danger_accept_invalid_certs(true) .build() .unwrap(); match client - .get(format!("https://{}:{}", &ENV.u_server, MASTER_PORT)) + .get(format!("https://{}:{}", &env_default.u_server, MASTER_PORT)) .send() .await { diff --git a/integration-tests/tests/integration_tests/endpoints.rs b/integration-tests/tests/integration_tests/endpoints.rs new file mode 100644 index 0000000..e575d77 --- /dev/null +++ b/integration-tests/tests/integration_tests/endpoints.rs @@ -0,0 +1,97 @@ +// get_personal_jobs(&self, url_param: Id) +// report(&self, payload: impl OneOrVec) +// dl(&self, file: String) +// get_job(&self, job: Id) +// get_jobs(&self) +// get_agents(&self, agent: Option) +// update_agent(&self, agent: Agent) +// update_job(&self, job: FatJob) +// update_result(&self, result: AssignedJob) +// upload_jobs(&self, payload: impl OneOrVec) +// del(&self, item: Id) +// assign_jobs(&self, agent: Id, job_idents: impl OneOrVec) +// get_agent_jobs(&self, agent: Option) +// ping(&self) + +use crate::fixtures::connections::*; +use std::iter::repeat; +use u_lib::models::{Brief, RawJob, RawPayload, MAX_READABLE_PAYLOAD_SIZE}; + +#[rstest] +#[tokio::test] +async fn jobs_upload_update_get_del(client_panel: &HttpClient) { + let job_alias = "henlo"; + let mut job = RawJob::default() + .with_shell("/bin/bash {}") + .with_raw_payload("echo henlo") + .with_alias(job_alias) + .try_into_job() + .unwrap(); + + let job_id = job.meta.id; + + client_panel.upload_jobs([&job]).await.unwrap(); + + let fetched_job = client_panel.get_full_job(job_id).await.unwrap(); + assert_eq!(job, fetched_job); + + let new_alias = "henlo2".to_string(); + job.meta.alias = Some(new_alias.clone()); + client_panel.update_job(&job.meta).await.unwrap(); + + let fetched_job = client_panel.get_full_job(job_id).await.unwrap(); + assert_eq!( + fetched_job.payload.as_ref().unwrap().data.as_ref().unwrap(), + b"echo henlo" + ); + assert_eq!(fetched_job.meta.alias, Some(new_alias)); + + client_panel.del(job_id).await.unwrap(); + + let not_found_err = client_panel.get_brief_job(job_id).await.unwrap_err(); + assert!(not_found_err.to_string().contains("404 Not Found")) +} + +#[rstest] +#[tokio::test] +async fn payloads_upload_update_get_del(client_panel: &HttpClient) { + let name = "test1".to_string(); + let data = b"qweasdzxc".to_vec(); + let payload = RawPayload { + name: Some(name.clone()), + data: data.clone(), + }; + + client_panel.upload_payload(&payload).await.unwrap(); + + let mut fetched_payload = client_panel.get_payload(&name, Brief::No).await.unwrap(); + let fetched_payload_auto = client_panel.get_payload(&name, Brief::Auto).await.unwrap(); + + assert_eq!(fetched_payload, fetched_payload_auto); + assert_eq!(fetched_payload.data.unwrap(), data); + + let new_size = MAX_READABLE_PAYLOAD_SIZE + 1; + let big_data = repeat(1u8).take(new_size as usize).collect::>(); + + fetched_payload.data = Some(big_data.clone()); + client_panel.update_payload(&fetched_payload).await.unwrap(); + + let fetched_big_payload = client_panel.get_payload(&name, Brief::Yes).await.unwrap(); + let fetched_big_payload_auto = client_panel.get_payload(&name, Brief::Auto).await.unwrap(); + + assert_eq!(fetched_big_payload, fetched_big_payload_auto); + assert_eq!(fetched_big_payload.size, new_size); + assert!(fetched_big_payload.data.is_none()); + + let fetched_big_payload_full = client_panel.get_payload(&name, Brief::No).await.unwrap(); + + assert_eq!(fetched_big_payload_full.data.unwrap(), big_data); + + client_panel.del(fetched_big_payload_full.id).await.unwrap(); + + let not_found_err = client_panel + .get_payload(&name, Brief::Yes) + .await + .unwrap_err(); + assert!(not_found_err.to_string().contains("404 Not Found")) +} diff --git a/integration/tests/integration/mod.rs b/integration-tests/tests/integration_tests/mod.rs similarity index 67% rename from integration/tests/integration/mod.rs rename to integration-tests/tests/integration_tests/mod.rs index 0b76512..bbc23d4 100644 --- a/integration/tests/integration/mod.rs +++ b/integration-tests/tests/integration_tests/mod.rs @@ -1,2 +1,3 @@ mod behaviour; mod connection; +mod endpoints; diff --git a/integration-tests/tests/lib.rs b/integration-tests/tests/lib.rs new file mode 100644 index 0000000..0f10c3a --- /dev/null +++ b/integration-tests/tests/lib.rs @@ -0,0 +1,14 @@ +mod fixtures; +mod helpers; +mod integration_tests; + +#[macro_use] +extern crate rstest; + +#[macro_use] +extern crate tracing; + +#[ctor::ctor] +fn __init() { + u_lib::logging::init_logger(None); +} diff --git a/integration/utils.py b/integration-tests/utils.py similarity index 100% rename from integration/utils.py rename to integration-tests/utils.py diff --git a/integration/tests/fixtures/agent.rs b/integration/tests/fixtures/agent.rs deleted file mode 100644 index af023f3..0000000 --- a/integration/tests/fixtures/agent.rs +++ /dev/null @@ -1,36 +0,0 @@ -use crate::helpers::ENV; -use u_lib::{ - api::ClientHandler, config::get_self_id, jobs::fat_meta_to_thin, messaging::Reportable, - models::*, types::Id, -}; - -pub struct RegisteredAgent { - pub id: Id, -} - -impl RegisteredAgent { - pub async fn unregister(self) { - let cli = ClientHandler::new(&ENV.u_server, None).await.unwrap(); - cli.del(self.id).await.unwrap(); - } -} - -#[fixture] -pub async fn register_agent() -> RegisteredAgent { - let cli = ClientHandler::new(&ENV.u_server, None).await.unwrap(); - let agent_id = get_self_id(); - println!("registering agent {agent_id}"); - let resp = cli - .get_personal_jobs(agent_id) - .await - .unwrap() - .pop() - .unwrap(); - let job_id = resp.job_id; - let job = cli.get_job(job_id).await.unwrap(); - assert_eq!(job.alias, Some("agent_hello".to_string())); - let mut agent_data = AssignedJob::from((&fat_meta_to_thin(job).unwrap(), resp)); - agent_data.set_result(&Agent::with_id(agent_id)); - cli.report(Reportable::Assigned(agent_data)).await.unwrap(); - RegisteredAgent { id: agent_id } -} diff --git a/integration/tests/fixtures/mod.rs b/integration/tests/fixtures/mod.rs deleted file mode 100644 index f17bc55..0000000 --- a/integration/tests/fixtures/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod agent; diff --git a/integration/tests/helpers/mod.rs b/integration/tests/helpers/mod.rs deleted file mode 100644 index 3ac8a05..0000000 --- a/integration/tests/helpers/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -pub mod jobs; -pub mod panel; - -pub use panel::Panel; - -use once_cell::sync::Lazy; -use u_lib::config::EndpointsEnv; - -pub static ENV: Lazy = Lazy::new(|| EndpointsEnv::load()); diff --git a/integration/tests/lib.rs b/integration/tests/lib.rs deleted file mode 100644 index 826b82d..0000000 --- a/integration/tests/lib.rs +++ /dev/null @@ -1,6 +0,0 @@ -mod fixtures; -mod helpers; -mod integration; - -#[macro_use] -extern crate rstest; diff --git a/lib/u_lib/Cargo.toml b/lib/u_lib/Cargo.toml index f702142..8cf77d7 100644 --- a/lib/u_lib/Cargo.toml +++ b/lib/u_lib/Cargo.toml @@ -14,7 +14,6 @@ diesel-derive-enum = { version = "2.0.0", features = ["postgres"], optional = tr deadpool-diesel = { workspace = true, optional = true } dotenv = "0.15.0" envy = "0.4.2" -futures = "0.3.5" guess_host_triple = "0.1.2" libc = "^0.2" lazy_static = "1.4.0" @@ -32,12 +31,15 @@ tracing-appender = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } uuid = { workspace = true, features = ["serde", "v4"] } parking_lot = "0.12.1" +bincode = "1.3.3" +sha3 = "0.10.7" [target.'cfg(unix)'.dependencies] -daemonize = "0.4.1" +daemonize = "0.5" nix = "0.17" [features] +agent = [] panel = [] server = ["dep:diesel", "dep:diesel-derive-enum", "dep:deadpool-diesel"] diff --git a/lib/u_lib/src/api.rs b/lib/u_lib/src/api.rs index 9d306a9..e0e9771 100644 --- a/lib/u_lib/src/api.rs +++ b/lib/u_lib/src/api.rs @@ -1,8 +1,8 @@ -use std::collections::HashMap; use std::fmt::Debug; use std::net::SocketAddr; +use std::{collections::HashMap, time::Duration}; -use anyhow::{Context, Result}; +use anyhow::{anyhow, Context, Result}; use reqwest::{header, header::HeaderMap, Certificate, Client, Identity, Method, Url}; use serde::de::DeserializeOwned; use serde_json::{from_str, Value}; @@ -11,7 +11,6 @@ use crate::{ config::{get_self_id, MASTER_PORT}, conv::opt_to_string, messaging::{self, AsMsg}, - misc::OneOrVec, models::*, types::Id, UError, UResult, @@ -20,13 +19,36 @@ use crate::{ const AGENT_IDENTITY: &[u8] = include_bytes!("../../../certs/alice.p12"); const ROOT_CA_CERT: &[u8] = include_bytes!("../../../certs/ca.crt"); +pub mod api_types { + use super::*; + + pub type GetPersonalJobs = Vec; + pub type Report = (); + pub type GetJob = Job; + pub type GetBriefJob = Job; + pub type GetJobs = Vec; + pub type GetAgents = Vec; + pub type UpdateAgent = (); + pub type UpdateJob = (); + pub type UpdateResult = (); + pub type UpdatePayload = (); + pub type UploadJobs = (); + pub type UploadPayloads = (); + pub type Del = (); + pub type SetJobs = (); + pub type GetAgentJobs = Vec; + pub type Ping = (); + pub type GetPayloads = Vec; + pub type GetPayload = Payload; +} + #[derive(Clone, Debug)] -pub struct ClientHandler { +pub struct HttpClient { base_url: Url, client: Client, } -impl ClientHandler { +impl HttpClient { pub async fn new(server: &str, password: Option) -> UResult { let identity = Identity::from_pkcs12_der(AGENT_IDENTITY, "").unwrap(); let mut default_headers = @@ -36,17 +58,17 @@ impl ClientHandler { default_headers.insert(header::AUTHORIZATION, format!("Bearer {pwd}")); } - // todo: don't rely only on dns resolve - let client = { - let client = Client::builder() - .identity(identity) - .default_headers(HeaderMap::try_from(&default_headers).unwrap()) - .add_root_certificate(Certificate::from_pem(ROOT_CA_CERT).unwrap()); + let client = Client::builder() + .identity(identity) + .default_headers(HeaderMap::try_from(&default_headers).unwrap()) + .add_root_certificate(Certificate::from_pem(ROOT_CA_CERT).unwrap()) + .timeout(Duration::from_secs(20)); + async fn resolve(domain_name: &str) -> Result { let dns_response = Client::new() .request( Method::GET, - format!("https://1.1.1.1/dns-query?name={server}&type=A"), + format!("https://1.1.1.1/dns-query?name={domain_name}&type=A"), ) .header(header::ACCEPT, "application/dns-json") .send() @@ -54,16 +76,22 @@ impl ClientHandler { .text() .await?; - match from_str::(&dns_response).unwrap()["Answer"] - .get(0) + let ip = from_str::(&dns_response)? + .get("Answer") + .and_then(|a| a.get(0)) .and_then(|a| a.get("data")) - { - Some(ip) => { - let raw_addr = format!("{}:{MASTER_PORT}", ip.as_str().unwrap()); - let addr: SocketAddr = raw_addr.parse().unwrap(); - client.resolve(server, addr) - } - None => client, + .map(|ip| ip.to_owned()) + .ok_or_else(|| anyhow!("can't extract dns answer"))?; + + let raw_addr = format!("{}:{MASTER_PORT}", ip.as_str().unwrap()); + Ok(raw_addr.parse().unwrap()) + } + + let client = match resolve(server).await { + Ok(addr) => client.resolve(server, addr), + Err(e) => { + warn!("DNS error: {e}"); + client } } .build() @@ -75,120 +103,164 @@ impl ClientHandler { }) } - async fn req(&self, url: impl AsRef) -> Result { - self.req_with_payload(url, ()).await + async fn req(&self, url: impl AsRef) -> Result { + self.req_with_payload(url, &()).await } - async fn req_with_payload( + async fn req_with_payload( &self, url: impl AsRef, - payload: P, + payload: &P, ) -> Result { + let url = url.as_ref(); let request = self .client - .post(self.base_url.join(url.as_ref()).unwrap()) - .json(&payload); + .post(self.base_url.join(url).unwrap()) + .json(payload); let response = request .send() .await .context("error while sending request")?; - let content_len = response.content_length(); + let is_success = match response.error_for_status_ref() { Ok(_) => Ok(()), Err(e) => Err(UError::from(e)), }; + let resp = response.text().await.context("resp")?; let result = match is_success { - Ok(_) => from_str::(&resp).or_else(|e| match content_len { - Some(0) => Ok(Default::default()), - _ => Err(UError::NetError(e.to_string(), resp)), - }), - Err(UError::NetError(err, _)) => Err(UError::NetError(err, resp)), + Ok(_) => { + from_str::(&resp).map_err(|e| UError::DeserializeError(e.to_string(), resp)) + } + Err(UError::NetError(err, url, _)) => Err(UError::NetError(err, url, resp)), _ => unreachable!(), } .map_err(From::from); - debug!("url = {}, resp = {:?}", url.as_ref(), result); + debug!("url = {url}, response = {result:?}"); result } - // get jobs for client - pub async fn get_personal_jobs(&self, url_param: Id) -> Result> { - self.req(format!("get_personal_jobs/{}", url_param)).await + /// get jobs for agent + pub async fn get_personal_jobs(&self, agent_id: Id) -> Result { + self.req(format!("get_personal_jobs/{}", agent_id)).await } - // send something to server - pub async fn report(&self, payload: impl OneOrVec) -> Result<()> { - self.req_with_payload("report", payload.into_vec()).await + /// send something to server + pub async fn report( + &self, + payload: impl IntoIterator, + ) -> Result { + self.req_with_payload("report", &payload.into_iter().collect::>()) + .await } - // download file - pub async fn dl(&self, file: String) -> Result> { + /// download payload + pub async fn _dl(&self, file: &str) -> Result> { self.req(format!("dl/{file}")).await } /// get exact job - pub async fn get_job(&self, job: Id) -> Result> { - self.req(format!("get_job/{job}")).await + pub async fn get_job(&self, job: Id, brief: Brief) -> Result { + self.req(format!("get_job/{job}?brief={brief}")).await + } + + pub async fn get_full_job(&self, job: Id) -> Result { + self.get_job(job, Brief::No).await + } + + pub async fn get_brief_job(&self, job: Id) -> Result { + self.get_job(job, Brief::Yes).await } /// get all available jobs - pub async fn get_jobs(&self) -> Result> { + pub async fn get_jobs(&self) -> Result { self.req("get_jobs").await } } //##########// Admin area //##########// #[cfg(feature = "panel")] -impl ClientHandler { +impl HttpClient { /// agent listing - pub async fn get_agents(&self, agent: Option) -> Result> { + pub async fn get_agents(&self, agent: Option) -> Result { self.req(format!("get_agents/{}", opt_to_string(agent))) .await } /// update agent - pub async fn update_agent(&self, agent: Agent) -> Result<()> { + pub async fn update_agent(&self, agent: &Agent) -> Result { self.req_with_payload("update_agent", agent).await } /// update job - pub async fn update_job(&self, job: FatJobMeta) -> Result<()> { + pub async fn update_job(&self, job: &JobMeta) -> Result { self.req_with_payload("update_job", job).await } /// update result - pub async fn update_result(&self, result: AssignedJob) -> Result<()> { + pub async fn update_result(&self, result: &AssignedJob) -> Result { self.req_with_payload("update_result", result).await } + pub async fn update_payload(&self, payload: &Payload) -> Result { + self.req_with_payload("update_payload", payload).await + } + /// create and upload job - pub async fn upload_jobs(&self, payload: impl OneOrVec>) -> Result> { - self.req_with_payload("upload_jobs", payload.into_vec()) + pub async fn upload_jobs( + &self, + jobs: impl IntoIterator, + ) -> Result { + self.req_with_payload("upload_jobs", &jobs.into_iter().collect::>()) .await } + pub async fn upload_payload(&self, payload: &RawPayload) -> Result { + self.req_with_payload("upload_payload", payload).await + } + /// delete something - pub async fn del(&self, item: Id) -> Result { + pub async fn del(&self, item: Id) -> Result { self.req(format!("del/{item}")).await } /// set jobs for any agent - pub async fn set_jobs(&self, agent: Id, job_idents: impl OneOrVec) -> Result> { - self.req_with_payload(format!("set_jobs/{agent}"), job_idents.into_vec()) + pub async fn assign_jobs( + &self, + assigned: impl IntoIterator, + ) -> Result { + self.req_with_payload( + format!("assign_jobs"), + &assigned.into_iter().collect::>(), + ) + .await + } + + /// get jobs for any agent by job_id, agent_id or result_id + pub async fn get_assigned_jobs(&self, id: Option) -> Result { + self.req(format!("get_assigned_jobs/{}", opt_to_string(id))) .await } - /// get jobs for any agent - pub async fn get_agent_jobs(&self, agent: Option) -> Result> { - self.req(format!("get_agent_jobs/{}", opt_to_string(agent))) + pub async fn get_payloads(&self) -> Result { + self.req("get_payloads").await + } + + pub async fn get_payload( + &self, + payload: impl AsRef, + brief: Brief, + ) -> Result { + let payload = payload.as_ref(); + self.req(format!("get_payload/{payload}?brief={brief}")) .await } - pub async fn ping(&self) -> Result<()> { + pub async fn ping(&self) -> Result { self.req("ping").await } } diff --git a/lib/u_lib/src/cache.rs b/lib/u_lib/src/cache.rs index 17ccfaa..3031b0f 100644 --- a/lib/u_lib/src/cache.rs +++ b/lib/u_lib/src/cache.rs @@ -1,10 +1,11 @@ -use crate::models::ThinJobMeta; +use crate::models::Job; use crate::types::Id; use lazy_static::lazy_static; use parking_lot::{RwLock, RwLockReadGuard}; use std::{collections::HashMap, ops::Deref}; -type Cache = HashMap; +type Val = Job; +type Cache = HashMap; lazy_static! { static ref JOB_CACHE: RwLock = RwLock::new(HashMap::new()); @@ -13,8 +14,8 @@ lazy_static! { pub struct JobCache; impl JobCache { - pub fn insert(job_meta: ThinJobMeta) { - JOB_CACHE.write().insert(job_meta.id, job_meta); + pub fn insert(job: Val) { + JOB_CACHE.write().insert(job.meta.id, job); } pub fn contains(id: Id) -> bool { @@ -37,7 +38,7 @@ impl JobCache { pub struct JobCacheHolder<'jh>(pub RwLockReadGuard<'jh, Cache>, pub Id); impl<'jh> Deref for JobCacheHolder<'jh> { - type Target = ThinJobMeta; + type Target = Val; fn deref(&self) -> &Self::Target { self.0.get(&self.1).unwrap() diff --git a/lib/u_lib/src/combined_result.rs b/lib/u_lib/src/combined_result.rs index 2c1c4bb..6aee4a9 100644 --- a/lib/u_lib/src/combined_result.rs +++ b/lib/u_lib/src/combined_result.rs @@ -1,6 +1,5 @@ use std::fmt::Debug; -use crate::misc::OneOrVec; use anyhow::Error; pub struct CombinedResult { @@ -16,17 +15,12 @@ impl CombinedResult { } } - pub fn ok(&mut self, result: impl OneOrVec) { - self.ok.extend(result.into_vec()); + pub fn push_ok(&mut self, result: T) { + self.ok.push(result); } - pub fn err>(&mut self, err: impl OneOrVec) { - self.err.extend( - err.into_vec() - .into_iter() - .map(Into::into) - .collect::>(), - ); + pub fn push_err(&mut self, err: impl Into) { + self.err.push(err.into()); } pub fn unwrap(self) -> Vec { diff --git a/lib/u_lib/src/config.rs b/lib/u_lib/src/config.rs index 5eb1c7e..5890996 100644 --- a/lib/u_lib/src/config.rs +++ b/lib/u_lib/src/config.rs @@ -1,3 +1,4 @@ +use crate::types::Id; use envy::{from_env, prefixed, Result as EnvResult}; use lazy_static::lazy_static; use serde::Deserialize; @@ -5,9 +6,7 @@ use std::time::Duration; pub use envy::Error; -use crate::types::Id; - -pub const MASTER_PORT: u16 = 63714; +pub const MASTER_PORT: u16 = 9990; pub const AGENT_ITERATION_INTERVAL: Duration = Duration::from_secs(5); diff --git a/lib/u_lib/src/db.rs b/lib/u_lib/src/db.rs index 3e874d2..0c5d8ef 100644 --- a/lib/u_lib/src/db.rs +++ b/lib/u_lib/src/db.rs @@ -1,5 +1,6 @@ use deadpool_diesel::{Manager as DManager, Pool as DPool, Runtime}; -use diesel::pg::PgConnection; +pub use diesel::pg::PgConnection; +use diesel::Connection; use std::time::Duration; use crate::config::DBEnv; @@ -24,3 +25,9 @@ pub fn async_pool(config: &DBEnv) -> PgAsyncPool { .build() .unwrap() } + +pub fn unpooled(config: &DBEnv) -> PgConnection { + let db_url = generate_postgres_url(config); + + PgConnection::establish(&db_url).unwrap() +} diff --git a/lib/u_lib/src/error/mod.rs b/lib/u_lib/src/error/mod.rs index 0a63f1d..929d7fd 100644 --- a/lib/u_lib/src/error/mod.rs +++ b/lib/u_lib/src/error/mod.rs @@ -3,8 +3,8 @@ mod chan; pub use chan::*; use crate::ufs; -use reqwest::Error as ReqError; use serde::{Deserialize, Serialize}; +use std::io; use thiserror::Error; use uuid::Uuid; @@ -15,8 +15,8 @@ pub enum UError { #[error("Runtime error: {0}")] Runtime(String), - #[error("Connection error: {0}. Body: {1}")] - NetError(String, String), + #[error("Connection error: {0}; url: {1}; body: '''{2}'''")] + NetError(String, String, String), #[error("Parse error")] ParseError, @@ -33,6 +33,9 @@ pub enum UError { #[error(transparent)] FSError(#[from] ufs::Error), + #[error("I/O error: {0}")] + IOError(String), + #[error("Wrong auth token")] WrongToken, @@ -42,27 +45,48 @@ pub enum UError { #[error("Panel error: {0}")] PanelError(String), - #[error("Deserialize from json error: {0}")] - DeserializeError(String), + #[error("Deserialize from json error: {0}, body: {1}")] + DeserializeError(String, String), + + #[error("{0}\nContext: {1}")] + Contexted(Box, String), } -impl From for UError { - fn from(e: ReqError) -> Self { - UError::NetError(e.to_string(), String::new()) +impl From for UError { + fn from(e: reqwest::Error) -> Self { + UError::NetError( + e.to_string(), + e.url().map(|u| u.to_string()).unwrap_or_default(), + String::new(), + ) } } -impl From for UError { - fn from(e: serde_json::Error) -> Self { - UError::DeserializeError(e.to_string()) +impl From for UError { + fn from(err: io::Error) -> Self { + UError::IOError(err.to_string()) } } impl From for UError { fn from(e: anyhow::Error) -> Self { + let ctx = e + .chain() + .rev() + .skip(1) + .map(|cause| format!("ctx: {}", cause)) + .collect::>() + .join("\n"); + match e.downcast::() { - Ok(err) => err, - Err(err) => UError::Runtime(err.to_string()), + Ok(err) => UError::Contexted(Box::new(err), ctx), + Err(err) => match err.downcast::() { + Ok(err) => UError::Contexted(Box::new(UError::FSError(err)), ctx), + Err(err) => match err.downcast::() { + Ok(err) => UError::Contexted(Box::new(UError::from(err)), ctx), + Err(err) => UError::Runtime(err.to_string()), + }, + }, } } } diff --git a/lib/u_lib/src/executor.rs b/lib/u_lib/src/executor.rs index 169002f..929011b 100644 --- a/lib/u_lib/src/executor.rs +++ b/lib/u_lib/src/executor.rs @@ -1,16 +1,18 @@ use crate::{models::AssignedJob, UResult}; -use futures::{future::BoxFuture, lock::Mutex}; use lazy_static::lazy_static; use std::collections::HashMap; use std::future::Future; +use std::pin::Pin; use tokio::{ runtime::Handle, sync::mpsc::{channel, Receiver, Sender}, + sync::Mutex, task::{spawn, spawn_blocking, JoinHandle}, }; use uuid::Uuid; pub type ExecResult = UResult; +type BoxFuture<'a, T> = Pin + Send + 'a>>; lazy_static! { static ref FUT_RESULTS: Mutex> = Mutex::new(HashMap::new()); @@ -94,10 +96,9 @@ impl Waiter { async fn init_receiver() { while let Some(fid) = FUT_CHANNEL.1.lock().await.recv().await { - if let Some(mut lock) = FUT_RESULTS.try_lock() { - if let Some(j) = lock.get_mut(&fid) { - j.completed = true; - } + let mut lock = FUT_RESULTS.lock().await; + if let Some(j) = lock.get_mut(&fid) { + j.completed = true; } } } diff --git a/lib/u_lib/src/jobs.rs b/lib/u_lib/src/jobs.rs index ea0efd3..899aa67 100644 --- a/lib/u_lib/src/jobs.rs +++ b/lib/u_lib/src/jobs.rs @@ -1,10 +1,8 @@ use crate::{ combined_result::CombinedResult, executor::{ExecResult, Waiter}, - misc::OneOrVec, - models::{Agent, AssignedJob, AssignedJobById, FatJobMeta, JobType, Payload, ThinJobMeta}, + models::{Agent, AssignedJob, AssignedJobById, Job, JobType, RawJob}, proc_output::ProcOutput, - ufs, }; use std::collections::HashMap; use std::process::exit; @@ -16,11 +14,10 @@ pub struct AnonymousJobBatch { } impl AnonymousJobBatch { - pub fn from_meta_with_id(jobs: impl OneOrVec<(ThinJobMeta, AssignedJobById)>) -> Self { - let jobs = jobs.into_vec(); + pub fn from_meta_with_id(jobs: impl IntoIterator) -> Self { let mut waiter = Waiter::new(); - for (meta, job) in jobs { - waiter.push(run_assigned_job(meta, job)); + for (job, ids) in jobs { + waiter.push(run_assigned_job(job, ids)); } Self { waiter, @@ -28,14 +25,13 @@ impl AnonymousJobBatch { } } - pub fn from_meta(metas: impl OneOrVec) -> Self { - let jobs: Vec<_> = metas - .into_vec() + pub fn from_meta(jobs: impl IntoIterator) -> Self { + let jobs_ids: Vec<_> = jobs .into_iter() - .map(|meta| { - let job_id = meta.id; + .map(|job| { + let job_id = job.meta.id; ( - meta, + job, AssignedJobById { job_id, ..Default::default() @@ -43,11 +39,12 @@ impl AnonymousJobBatch { ) }) .collect(); - AnonymousJobBatch::from_meta_with_id(jobs) + AnonymousJobBatch::from_meta_with_id(jobs_ids) } /// Spawn jobs pub async fn spawn(mut self) -> Self { + debug!("spawning jobs"); self.waiter = self.waiter.spawn().await; self.is_running = true; self @@ -77,45 +74,35 @@ pub struct NamedJobBatch { } impl NamedJobBatch { - pub fn from_shell( - named_jobs: impl OneOrVec<(&'static str, &'static str)>, - ) -> CombinedResult { + pub fn from_shell(named_jobs: Vec<(&'static str, &'static str)>) -> CombinedResult { let mut result = CombinedResult::new(); let jobs: Vec<_> = named_jobs - .into_vec() .into_iter() .filter_map(|(alias, cmd)| { - match FatJobMeta::builder() + match RawJob::default() .with_shell(cmd) .with_alias(alias) - .build() + .try_into_job() { - Ok(fat_meta) => match fat_meta_to_thin(fat_meta) { - Ok(thin_meta) => Some(thin_meta), - Err(e) => { - result.err(e); - None - } - }, + Ok(jpm) => Some(jpm), Err(e) => { - result.err(e); + result.push_err(e); None } } }) .collect(); - result.ok(Self::from_meta(jobs)); + result.push_ok(Self::from_meta(jobs)); result } - pub fn from_meta(named_jobs: impl OneOrVec) -> Self { - let (job_names, job_metas): (Vec<_>, Vec<_>) = named_jobs - .into_vec() + pub fn from_meta(named_jobs: Vec) -> Self { + let (job_names, jobs): (Vec<_>, Vec<_>) = named_jobs .into_iter() - .map(|meta| (meta.alias.clone().unwrap(), meta)) + .map(|job| (job.meta.alias.clone().unwrap(), job)) .unzip(); Self { - runner: Some(AnonymousJobBatch::from_meta(job_metas)), + runner: Some(AnonymousJobBatch::from_meta(jobs)), job_names, results: HashMap::new(), } @@ -145,13 +132,14 @@ impl NamedJobBatch { } } -pub async fn run_assigned_job(meta: ThinJobMeta, ids: AssignedJobById) -> ExecResult { - let mut job = AssignedJob::from((&meta, ids)); +pub async fn run_assigned_job(job: Job, ids: AssignedJobById) -> ExecResult { + let Job { meta, payload } = job; + let mut result = AssignedJob::from((&meta, ids)); match meta.exec_type { JobType::Shell => { let (argv, _prepared_payload) = { - if let Some(ref payload) = meta.payload { - let (prep_exec, prep_exec_path) = ufs::prepare_executable(payload)?; + if let Some(payload) = payload { + let (prep_exec, prep_exec_path) = payload.prepare_executable()?; let argv_with_exec = meta.argv.replace("{}", &prep_exec_path); (argv_with_exec, Some(prep_exec)) } else { @@ -175,69 +163,27 @@ pub async fn run_assigned_job(meta: ThinJobMeta, ids: AssignedJobById) -> ExecRe None, ), }; - job.result = Some(data); - job.retcode = retcode; + result.result = Some(data); + result.retcode = retcode; } JobType::Init => { - job.set_result(&Agent::run().await); - job.retcode = Some(0); + let agent = Agent::gather().await; + debug!("gathered info from agent: {agent:?}"); + result.set_result(&agent); + result.retcode = Some(0); } JobType::Service => todo!(), JobType::Update => todo!(), JobType::Terminate => exit(0), }; - Ok(job) -} - -pub fn fat_meta_to_thin(meta: FatJobMeta) -> Result { - let payload_ident = if let Some(mut payload) = meta.payload { - let job_name = match &meta.alias { - Some(a) => a.to_string(), - None => meta.id.simple().to_string(), - }; - payload.write_self_into(&job_name)?; - Some(job_name) - } else { - None - }; - - Ok(ThinJobMeta { - alias: meta.alias, - argv: meta.argv, - id: meta.id, - exec_type: meta.exec_type, - platform: meta.platform, - payload: payload_ident, - schedule: meta.schedule, - }) -} - -pub fn thin_meta_to_fat(meta: ThinJobMeta) -> Result, ufs::Error> { - let payload = if let Some(payload) = meta.payload { - let mut fat_payload = Payload::Ident(payload); - fat_payload.read_into_self()?; - Some(fat_payload) - } else { - None - }; - - Ok(FatJobMeta { - alias: meta.alias, - argv: meta.argv, - id: meta.id, - exec_type: meta.exec_type, - platform: meta.platform, - payload, - schedule: meta.schedule, - }) + Ok(result) } #[cfg(test)] mod tests { - use super::*; use crate::{ jobs::{AnonymousJobBatch, NamedJobBatch}, - models::{misc::JobType, FatJobMeta}, + models::{misc::JobType, RawJob}, unwrap_enum, UError, }; use std::time::SystemTime; @@ -247,10 +193,8 @@ mod tests { #[tokio::test] async fn test_is_really_async() { const SLEEP_SECS: u64 = 1; - let job = FatJobMeta::from_shell(format!("sleep {SLEEP_SECS}")).unwrap(); - let sleep_jobs = (0..50) - .map(|_| fat_meta_to_thin(job.clone()).unwrap()) - .collect::>(); + let job = RawJob::from_shell(format!("sleep {SLEEP_SECS}")).unwrap(); + let sleep_jobs = (0..50).map(|_| job.clone()).collect::>(); let now = SystemTime::now(); AnonymousJobBatch::from_meta(sleep_jobs).wait().await; @@ -260,12 +204,12 @@ mod tests { #[rstest] #[case::sh_payload( "/bin/sh {}", - Some(b"echo test01 > /tmp/asd; cat /tmp/asd".as_slice()), + Some(b"echo test01 > /tmp/asd; cat /tmp/asd".as_slice()), "test01" )] #[case::python_cmd(r#"/usr/bin/python3 -c 'print("test02")'"#, None, "test02")] #[case::sh_multiline_payload( - "/{}", + "/{}", Some( br#"#!/bin/sh TMPPATH=/tmp/lol @@ -278,7 +222,7 @@ mod tests { )] #[case::standalone_binary_with_args( "/{} 'some msg as arg'", - Some(include_bytes!("../tests/fixtures/echoer").as_slice()), + Some(include_bytes!("../tests/fixtures/echoer").as_slice()), "some msg as arg" )] #[tokio::test] @@ -287,12 +231,15 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] expected_result: &str, ) -> TestResult { - let mut job = FatJobMeta::builder().with_shell(cmd); + let mut raw_job = RawJob::default().with_shell(cmd); if let Some(p) = payload { - job = job.with_payload(p); + raw_job = raw_job.with_raw_payload(p); } - let job = fat_meta_to_thin(job.build().unwrap()).unwrap(); - let result = AnonymousJobBatch::from_meta(job).wait_one().await.unwrap(); + let job = raw_job.try_into_job().unwrap(); + let result = AnonymousJobBatch::from_meta([job]) + .wait_one() + .await + .unwrap(); let result = result.to_str_result(); assert_eq!(result.trim(), expected_result); Ok(()) @@ -302,23 +249,19 @@ mod tests { async fn test_complex_load() -> TestResult { const SLEEP_SECS: u64 = 1; let now = SystemTime::now(); - let longest_job = FatJobMeta::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); - let longest_job = AnonymousJobBatch::from_meta(fat_meta_to_thin(longest_job).unwrap()) - .spawn() - .await; - let ls = AnonymousJobBatch::from_meta( - fat_meta_to_thin(FatJobMeta::from_shell("ls").unwrap()).unwrap(), - ) - .wait_one() - .await - .unwrap(); + let longest_job = RawJob::from_shell(format!("sleep {}", SLEEP_SECS)).unwrap(); + let longest_job = AnonymousJobBatch::from_meta([longest_job]).spawn().await; + let ls = AnonymousJobBatch::from_meta([RawJob::from_shell("ls").unwrap()]) + .wait_one() + .await + .unwrap(); assert_eq!(ls.retcode.unwrap(), 0); let folders = ls.to_str_result(); let subfolders_jobs = folders .lines() - .map(|f| fat_meta_to_thin(FatJobMeta::from_shell(format!("ls {f}")).unwrap()).unwrap()) + .map(|f| RawJob::from_shell(format!("ls {f}")).unwrap()) .collect::>(); let ls_subfolders = AnonymousJobBatch::from_meta(subfolders_jobs).wait().await; @@ -351,8 +294,11 @@ mod tests { */ #[tokio::test] async fn test_failing_shell_job() -> TestResult { - let job = fat_meta_to_thin(FatJobMeta::from_shell("lol_kek_puk").unwrap()).unwrap(); - let job_result = AnonymousJobBatch::from_meta(job).wait_one().await.unwrap(); + let job = RawJob::from_shell("lol_kek_puk").unwrap(); + let job_result = AnonymousJobBatch::from_meta([job]) + .wait_one() + .await + .unwrap(); let output = job_result.to_str_result(); assert!(output.contains("No such file")); assert!(job_result.retcode.is_none()); @@ -368,11 +314,11 @@ mod tests { #[case] payload: Option<&[u8]>, #[case] err_str: &str, ) -> TestResult { - let mut job = FatJobMeta::builder().with_shell(cmd); + let mut raw_job = RawJob::default().with_shell(cmd); if let Some(p) = payload { - job = job.with_payload(p); + raw_job = raw_job.with_raw_payload(p); } - let err = job.build().unwrap_err(); + let err = raw_job.try_into_job().unwrap_err(); let err_msg = unwrap_enum!(err, UError::JobBuildError); assert!(err_msg.contains(err_str)); Ok(()) @@ -380,23 +326,18 @@ mod tests { #[tokio::test] async fn test_different_job_types() -> TestResult { - let mut jobs = NamedJobBatch::from_meta( - [ - FatJobMeta::builder() - .with_shell("sleep 3") - .with_alias("sleeper") - .build() - .unwrap(), - FatJobMeta::builder() - .with_type(JobType::Init) - .with_alias("gatherer") - .build() - .unwrap(), - ] - .into_iter() - .map(|meta| fat_meta_to_thin(meta).unwrap()) - .collect::>(), - ) + let mut jobs = NamedJobBatch::from_meta(vec![ + RawJob::default() + .with_shell("sleep 3") + .with_alias("sleeper") + .try_into_job() + .unwrap(), + RawJob::default() + .with_type(JobType::Init) + .with_alias("gatherer") + .try_into_job() + .unwrap(), + ]) .wait() .await; let gathered = jobs.pop("gatherer").unwrap(); diff --git a/lib/u_lib/src/logging.rs b/lib/u_lib/src/logging.rs index 9c4f1db..e552f18 100644 --- a/lib/u_lib/src/logging.rs +++ b/lib/u_lib/src/logging.rs @@ -1,28 +1,37 @@ use std::env; +use std::io::stderr; use std::path::Path; use tracing_appender::rolling; use tracing_subscriber::{fmt, prelude::*, registry, EnvFilter}; -pub fn init_logger(logfile: Option + Send + Sync + 'static>) { +pub fn init_logger(logfile: Option<&str>) { if env::var("RUST_LOG").is_err() { env::set_var("RUST_LOG", "info") } + let layer = fmt::layer().with_line_number(true); + + let output_layer = if cfg!(test) { + layer.with_test_writer().boxed() + } else { + layer.with_writer(stderr).boxed() + }; + let reg = registry() .with(EnvFilter::from_default_env()) - .with(fmt::layer()); + .with(output_layer); match logfile { - Some(file) => reg - .with( + Some(file) => { + let file_path = Path::new(file).with_extension("log"); + reg.with( fmt::layer() - .with_writer(move || { - rolling::never("logs", file.as_ref().with_extension("log")) - }) + .with_writer(move || rolling::never("logs", &file_path)) .with_ansi(false), ) - .init(), + .init() + } None => reg.init(), }; } diff --git a/lib/u_lib/src/messaging/mod.rs b/lib/u_lib/src/messaging.rs similarity index 71% rename from lib/u_lib/src/messaging/mod.rs rename to lib/u_lib/src/messaging.rs index 1dc699f..b1e1ecf 100644 --- a/lib/u_lib/src/messaging/mod.rs +++ b/lib/u_lib/src/messaging.rs @@ -1,30 +1,30 @@ -mod files; - use crate::models::*; use crate::types::Id; use crate::UError; -pub use files::*; use serde::{Deserialize, Serialize}; use std::fmt::Debug; +/// Represents types that could be used in client-server interaction pub trait AsMsg: Clone + Serialize + Debug {} impl AsMsg for Agent {} impl AsMsg for AssignedJob {} impl AsMsg for AssignedJobById {} -impl AsMsg for DownloadInfo {} -impl AsMsg for FatJobMeta {} +impl AsMsg for JobMeta {} impl AsMsg for Reportable {} -impl AsMsg for String {} -impl AsMsg for ThinJobMeta {} +impl AsMsg for Payload {} +impl AsMsg for RawPayload {} +impl AsMsg for Job {} impl AsMsg for Id {} -impl AsMsg for i32 {} -impl AsMsg for u8 {} +impl AsMsg for String {} +impl AsMsg for Vec {} impl AsMsg for () {} impl AsMsg for Vec {} impl<'msg, M: AsMsg> AsMsg for &'msg [M] {} +impl AsMsg for &M {} + #[derive(Serialize, Deserialize, Clone, Debug)] pub enum Reportable { Assigned(AssignedJob), diff --git a/lib/u_lib/src/messaging/files.rs b/lib/u_lib/src/messaging/files.rs deleted file mode 100644 index 4cdb804..0000000 --- a/lib/u_lib/src/messaging/files.rs +++ /dev/null @@ -1,8 +0,0 @@ -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -#[derive(Serialize, Deserialize, Clone, Debug, Default)] -pub struct DownloadInfo { - hashsum: String, - dl_fid: Uuid, -} diff --git a/lib/u_lib/src/misc.rs b/lib/u_lib/src/misc.rs index bbc6484..2e280cb 100644 --- a/lib/u_lib/src/misc.rs +++ b/lib/u_lib/src/misc.rs @@ -1,26 +1,10 @@ -pub trait OneOrVec { - fn into_vec(self) -> Vec; -} - -impl OneOrVec for T { - fn into_vec(self) -> Vec { - vec![self] - } -} - -impl OneOrVec for Vec { - fn into_vec(self) -> Vec { - self - } -} - #[macro_export] macro_rules! unwrap_enum { ($src:expr, $t:path) => { if let $t(result) = $src { result } else { - panic!("wrong type {}", stringify!($t)) + panic!("wrong type '{}'", stringify!($t)) } }; } diff --git a/lib/u_lib/src/models/agent.rs b/lib/u_lib/src/models/agent.rs index 5a81866..a05e74d 100644 --- a/lib/u_lib/src/models/agent.rs +++ b/lib/u_lib/src/models/agent.rs @@ -14,7 +14,7 @@ use self::server::*; use crate::{ config::get_self_id, conv::systime_to_string, executor::ExecResult, jobs::NamedJobBatch, - platform::Platform, types::Id, + platform, types::Id, }; #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Display)] @@ -54,10 +54,29 @@ pub struct Agent { } impl Agent { - pub fn with_id(id: Id) -> Self { + pub fn empty() -> Self { Self { - id, - ..Default::default() + alias: None, + id: get_self_id(), + hostname: String::new(), + host_info: String::new(), + is_root: false, + is_root_allowed: false, + last_active: SystemTime::now(), + platform: String::new(), + regtime: SystemTime::now(), + state: AgentState::New, + token: None, + username: String::new(), + ip_gray: None, + ip_white: None, + } + } + + pub fn with_current_platform() -> Self { + Self { + platform: platform::current_as_string(), + ..Self::empty() } } @@ -91,33 +110,7 @@ impl Agent { host_info: decoder(builder.pop("host_info")), is_root: &decoder(builder.pop("is_root")) == "0", username: decoder(builder.pop("username")), - platform: Platform::current().into_string(), - ..Default::default() - } - } - - pub async fn run() -> Agent { - Agent::gather().await - } -} - -impl Default for Agent { - fn default() -> Self { - Self { - alias: None, - id: get_self_id(), - hostname: String::new(), - host_info: String::new(), - is_root: false, - is_root_allowed: false, - last_active: SystemTime::now(), - platform: String::new(), - regtime: SystemTime::now(), - state: AgentState::New, - token: None, - username: String::new(), - ip_gray: None, - ip_white: None, + ..Self::with_current_platform() } } } diff --git a/lib/u_lib/src/models/jobs/assigned.rs b/lib/u_lib/src/models/jobs/assigned.rs index 0e302ef..a7caf20 100644 --- a/lib/u_lib/src/models/jobs/assigned.rs +++ b/lib/u_lib/src/models/jobs/assigned.rs @@ -1,4 +1,4 @@ -use super::{JobState, JobType, ThinJobMeta}; +use super::{JobMeta, JobState, JobType}; #[cfg(feature = "server")] use crate::models::schema::*; use crate::{ @@ -56,12 +56,13 @@ impl Debug for AssignedJob { #[derive(Serialize, Deserialize, Clone, Copy, Debug)] pub struct AssignedJobById { pub agent_id: Id, + #[serde(default)] pub id: Id, pub job_id: Id, } -impl From<(&ThinJobMeta, AssignedJobById)> for AssignedJob { - fn from((meta, ids): (&ThinJobMeta, AssignedJobById)) -> Self { +impl From<(&JobMeta, AssignedJobById)> for AssignedJob { + fn from((job, ids): (&JobMeta, AssignedJobById)) -> Self { let AssignedJobById { agent_id, id, @@ -72,13 +73,30 @@ impl From<(&ThinJobMeta, AssignedJobById)> for AssignedJob { id, agent_id, job_id, - alias: meta.alias.clone(), - exec_type: meta.exec_type, + alias: job.alias.clone(), + exec_type: job.exec_type, ..Default::default() } } } +impl From<&AssignedJob> for AssignedJobById { + fn from(j: &AssignedJob) -> Self { + let &AssignedJob { + agent_id, + id, + job_id, + .. + } = j; + + AssignedJobById { + agent_id, + id, + job_id, + } + } +} + impl Default for AssignedJobById { fn default() -> Self { Self { diff --git a/lib/u_lib/src/models/jobs/meta.rs b/lib/u_lib/src/models/jobs/meta.rs index 2feca3d..10a8026 100644 --- a/lib/u_lib/src/models/jobs/meta.rs +++ b/lib/u_lib/src/models/jobs/meta.rs @@ -1,52 +1,25 @@ use std::fmt; use super::JobType; -use crate::models::payload::Payload; #[cfg(feature = "server")] use crate::models::schema::*; -use crate::platform::Platform; +use crate::models::Payload; +use crate::platform; use crate::types::Id; use crate::{UError, UResult}; #[cfg(feature = "server")] use diesel::{Identifiable, Insertable, Queryable}; use serde::{Deserialize, Serialize}; +use std::borrow::Cow; -#[derive(Serialize, Deserialize, Clone)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr( feature = "server", derive(Queryable, Identifiable, Insertable, AsChangeset), - diesel(table_name = jobs) + diesel(table_name = jobs), + diesel(treat_none_as_null = true) )] -pub struct ThinJobMeta { - pub alias: Option, - /// string like `bash -c {} -a 1 --arg2`, - /// where {} is replaced by executable's tmp path - pub argv: String, - pub id: Id, - pub exec_type: JobType, - /// target triple - pub platform: String, - pub payload: Option, - /// cron-like string - pub schedule: Option, -} - -impl fmt::Debug for ThinJobMeta { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ThinJobMeta") - .field("alias", &self.alias) - .field("argv", &self.argv) - .field("id", &self.id.to_string()) - .field("exec_type", &self.exec_type) - .field("platform", &self.platform) - .field("payload", &self.payload) - .field("schedule", &self.schedule) - .finish() - } -} - -#[derive(Serialize, Deserialize, Clone)] -pub struct FatJobMeta { +pub struct JobMeta { #[serde(default)] pub alias: Option, @@ -62,160 +35,221 @@ pub struct FatJobMeta { pub exec_type: JobType, /// target triple - #[serde(default = "Platform::current_as_string")] - pub platform: String, + #[serde(default = "platform::current_as_string")] + pub target_platforms: String, #[serde(default)] - pub payload: Option, + pub payload_id: Option, /// cron-like string #[serde(default)] pub schedule: Option, } -impl fmt::Debug for FatJobMeta { +impl Default for JobMeta { + fn default() -> Self { + Self { + alias: None, + argv: String::new(), + id: Id::new_v4(), + exec_type: JobType::default(), + target_platforms: String::new(), + payload_id: None, + schedule: None, + } + } +} + +impl fmt::Debug for JobMeta { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("FatJobMeta") + f.debug_struct("JobMeta") .field("alias", &self.alias) .field("argv", &self.argv) .field("id", &self.id.to_string()) .field("exec_type", &self.exec_type) - .field("platform", &self.platform) - .field("payload", &self.payload) + .field("target_platforms", &self.target_platforms) + .field("payload_id", &self.payload_id.map(|id| id.to_string())) .field("schedule", &self.schedule) .finish() } } -impl FatJobMeta { - pub fn validated(self) -> UResult> { - JobMetaBuilder { inner: self }.build() - } +impl JobMeta { + pub fn validate(mut self) -> UResult { + fn mk_err(msg: impl Into) -> UError { + UError::JobBuildError(msg.into()) + } - pub fn from_shell(cmd: impl Into) -> UResult> { - Self::builder().with_shell(cmd).build() - } + const ARGV_STR_LEN: usize = 2048; + + if self.argv.is_empty() { + // TODO: fix detecting + self.argv = String::from("echo 'hello, world!'") + } else if self.argv.len() > ARGV_STR_LEN { + return Err(mk_err(format!( + "argv length limit ({ARGV_STR_LEN}) exceeded" + ))); + } + + let argv_parts = shlex::split(&self.argv).ok_or_else(|| mk_err("Shlex failed"))?; + let empty_err = mk_err("Empty argv"); + + if argv_parts.get(0).ok_or(empty_err.clone())?.is_empty() { + return Err(empty_err); + } + + if self.payload_id.is_some() && !self.argv.contains("{}") { + return Err(mk_err("Argv contains no executable placeholder")); + } + + if self.argv.contains("{}") && self.payload_id.is_none() { + return Err(mk_err( + "No payload provided, but argv contains executable placeholder", + )); + } + + if self.target_platforms.is_empty() { + self.target_platforms = "*".to_string(); + } + + if !platform::is_valid_glob(&self.target_platforms) { + return Err(mk_err(format!( + "Unknown platform '{}'", + self.target_platforms + ))); + } - pub fn builder() -> JobMetaBuilder { - JobMetaBuilder::default() + Ok(self) } } -impl Default for FatJobMeta { - fn default() -> Self { - Self { - id: Id::new_v4(), - alias: None, - argv: String::new(), - exec_type: JobType::Shell, - platform: Platform::current_as_string(), - payload: None, - schedule: None, +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Job { + pub meta: JobMeta, + pub payload: Option, +} + +#[derive(Serialize, Deserialize, Clone, Default)] +pub struct RawJob<'p> { + #[serde(default)] + payload_path: Option, + + #[serde(default)] + raw_payload: Option>, + + #[serde(default, flatten)] + meta: JobMeta, +} + +impl From for RawJob<'_> { + fn from(job: Job) -> Self { + let Job { meta, payload } = job; + + RawJob { + payload_path: payload.map(|m| m.name), + raw_payload: None, + meta, } } } -#[derive(Default)] -pub struct JobMetaBuilder { - inner: FatJobMeta, -} +impl<'p> RawJob<'p> { + pub fn try_into_job(self) -> UResult { + Job::try_from(self) + } + + pub fn from_shell(cmd: impl Into) -> UResult { + Self::default().with_shell(cmd).try_into_job() + } -impl JobMetaBuilder { pub fn with_shell(mut self, shell_cmd: impl Into) -> Self { - self.inner.argv = shell_cmd.into(); - self.inner.exec_type = JobType::Shell; + self.meta.argv = shell_cmd.into(); + self.meta.exec_type = JobType::Shell; self } - pub fn with_payload(mut self, payload: impl Into>) -> Self { - self.inner.payload = Some(Payload::from_payload(payload)); + pub fn with_alias(mut self, alias: impl Into) -> Self { + self.meta.alias = Some(alias.into()); self } - pub fn with_payload_path(mut self, path: impl Into) -> Self { - self.inner.payload = Some(Payload::Ident(path.into())); + pub fn with_type(mut self, e_type: JobType) -> Self { + self.meta.exec_type = e_type; self } - pub fn with_alias(mut self, alias: impl Into) -> Self { - self.inner.alias = Some(alias.into()); + pub fn with_target_platforms(mut self, platform: impl Into) -> Self { + self.meta.target_platforms = platform.into(); self } - pub fn with_type(mut self, e_type: JobType) -> Self { - self.inner.exec_type = e_type; + pub fn with_raw_payload(mut self, raw_payload: impl AsPayload<'p>) -> Self { + self.raw_payload = Some(raw_payload.as_payload()); + self.payload_path = None; self } - pub fn build(self) -> UResult> { - let mut inner = self.inner; - let validated = |jmeta: FatJobMeta| FatJobMeta:: { - alias: jmeta.alias, - argv: jmeta.argv, - id: jmeta.id, - exec_type: jmeta.exec_type, - platform: jmeta.platform, - payload: jmeta.payload, - schedule: jmeta.schedule, - }; + pub fn with_payload_path(mut self, path: impl Into) -> Self { + self.payload_path = Some(path.into()); + self.raw_payload = None; + self + } +} - match inner.exec_type { - JobType::Shell => { - const ARGV_STR_LEN: usize = 2048; - - if inner.argv.is_empty() { - // TODO: fix detecting - inner.argv = String::from("echo 'hello, world!'") - } else if inner.argv.len() > ARGV_STR_LEN { - return Err(UError::JobBuildError(format!( - "argv length limit ({ARGV_STR_LEN}) exceeded" - ))); - } - - let argv_parts = shlex::split(&inner.argv) - .ok_or(UError::JobBuildError("Shlex failed".into()))?; - let empty_err = UError::JobBuildError("Empty argv".into()); - - if argv_parts.get(0).ok_or(empty_err.clone())?.is_empty() { - return Err(empty_err.into()); - } - - if let Some(payload) = &mut inner.payload { - payload.add_to_index()?; - } - - match inner.payload.as_ref() { - Some(p) => { - if let Payload::Data(d) = p { - if !d.is_empty() && !inner.argv.contains("{}") { - return Err(UError::JobBuildError( - "Argv contains no executable placeholder".into(), - ) - .into()); - } - } - } - None => { - if inner.argv.contains("{}") { - return Err(UError::JobBuildError( - "No payload provided, but argv contains executable placeholder" - .into(), - ) - .into()); - } - } - }; - - if Platform::new(&inner.platform).find_valid().is_none() { - return Err(UError::JobBuildError(format!( - "Unknown platform {}", - inner.platform - ))); - } - - Ok(validated(inner)) - } - _ => Ok(validated(inner)), +impl TryFrom> for Job { + type Error = UError; + + fn try_from(mut raw: RawJob) -> Result { + if raw.raw_payload.is_some() && raw.payload_path.is_some() { + return Err(UError::JobBuildError( + "Can't use both raw payload with payload path".to_string(), + )); } + + let payload = { + let payload_from_path = raw + .payload_path + .as_ref() + .map(|path| Payload::from_path(path)) + .transpose()?; + + if payload_from_path.is_none() { + raw.raw_payload + .as_ref() + .map(|data| Payload::from_data(data, None)) + .transpose()? + } else { + payload_from_path + } + }; + + raw.meta.payload_id = payload.as_ref().map(|p| p.id).or(raw.meta.payload_id); + + Ok(Job { + meta: raw.meta.validate()?, + payload, + }) + } +} + +pub trait AsPayload<'p> { + fn as_payload(&self) -> Cow<'p, [u8]>; +} + +impl<'p> AsPayload<'p> for &'p str { + fn as_payload(&self) -> Cow<'p, [u8]> { + Cow::Borrowed(self.as_bytes()) + } +} + +impl<'p, const N: usize> AsPayload<'p> for &'p [u8; N] { + fn as_payload(&self) -> Cow<'p, [u8]> { + Cow::Borrowed(*self) + } +} + +impl<'p> AsPayload<'p> for &'p [u8] { + fn as_payload(&self) -> Cow<'p, [u8]> { + Cow::Borrowed(self) } } diff --git a/lib/u_lib/src/models/jobs/misc.rs b/lib/u_lib/src/models/jobs/misc.rs index 6a99f7c..faf108e 100644 --- a/lib/u_lib/src/models/jobs/misc.rs +++ b/lib/u_lib/src/models/jobs/misc.rs @@ -26,7 +26,8 @@ pub enum JobState { Finished, } -#[derive(Default, Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Display)] +#[derive(Default, Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] #[cfg_attr( feature = "server", derive(DbEnum), diff --git a/lib/u_lib/src/models/mod.rs b/lib/u_lib/src/models/mod.rs index 3000f8b..61c9a56 100644 --- a/lib/u_lib/src/models/mod.rs +++ b/lib/u_lib/src/models/mod.rs @@ -5,3 +5,13 @@ mod payload; pub mod schema; pub use crate::models::{agent::*, jobs::*, payload::*}; +use serde::Deserialize; +use strum::{Display as StrumDisplay, EnumString}; + +#[derive(Default, Debug, StrumDisplay, EnumString, Deserialize)] +pub enum Brief { + Yes, + #[default] + Auto, + No, +} diff --git a/lib/u_lib/src/models/payload.rs b/lib/u_lib/src/models/payload.rs index 4543ec3..84db3d7 100644 --- a/lib/u_lib/src/models/payload.rs +++ b/lib/u_lib/src/models/payload.rs @@ -1,76 +1,209 @@ -use crate::{conv::bytes_to_string_truncated, ufs}; +use crate::{ + conv::{bytes_to_string, bytes_to_string_truncated}, + types::Id, + ufs, UError, +}; +use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; -use std::{fmt, path::PathBuf}; - -#[derive(Clone, Deserialize, Serialize)] -#[serde(untagged)] -pub enum Payload { - /// Raw payload data - Data(Vec), - /// Payload identifier in ufs - Ident(String), +use std::{fmt::Debug, fs::File, path::Path, process::Command}; + +pub const MAX_READABLE_PAYLOAD_SIZE: i64 = 1024 * 32; + +#[cfg(feature = "server")] +use crate::models::schema::*; +#[cfg(feature = "server")] +use diesel::Identifiable; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RawPayload { + pub name: Option, + pub data: Vec, +} + +impl RawPayload { + pub fn into_payload(self) -> Result { + Payload::from_data(self.data, self.name) + } +} + +#[cfg_attr( + feature = "server", + derive(Insertable, Queryable, Identifiable, AsChangeset), + diesel(table_name = payloads), + diesel(treat_none_as_null = true) +)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Payload { + pub id: Id, + pub mime_type: String, + pub name: String, + pub size: i64, + pub data: Option>, // when None, payload data is stored in ufs +} + +impl Debug for Payload { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Payload") + .field("id", &self.id) + .field("mime_type", &self.mime_type) + .field("name", &self.name) + .field("size", &self.size) + .field( + "data", + &self + .data + .as_ref() + .map(|data| bytes_to_string_truncated(data, 256)), + ) + .finish() + } } impl Payload { - pub fn read_into_self(&mut self) -> Result<(), ufs::Error> { - match self { - Payload::Data(_) => Ok(()), - Payload::Ident(ident) => { - let data = ufs::read(ident)?; - - *self = Payload::Data(data); - Ok(()) - } - } + pub fn is_human_readable(&self) -> bool { + self.size < MAX_READABLE_PAYLOAD_SIZE && self.mime_type.starts_with("text/") } - pub fn write_self_into(&mut self, name: impl AsRef) -> Result<(), ufs::Error> { - match self { - Payload::Ident(_) => Ok(()), - Payload::Data(data) => { - ufs::put(&name, data)?; + pub fn maybe_split_payload(&mut self) -> Result<()> { + if self.is_human_readable() { + return Ok(()); + } - *self = Payload::Ident(name.as_ref().to_string()); - Ok(()) + if let Some(data) = self.data.take() { + if ufs::exists_in_index(&self.name) { + ufs::edit(&self.name, data)?; + } else { + ufs::put(&self.name, data)?; } } + Ok(()) } - pub fn from_payload(data: impl Into>) -> Self { - Payload::Data(data.into()) + pub fn join_payload(&mut self) -> Result<()> { + if self.data.is_none() { + self.data = Some(ufs::read(&self.name)?); + } + Ok(()) } - pub fn from_path(path: impl Into) -> Result { - let path: PathBuf = path.into(); - - if !path.exists() || path.is_dir() { - return Err(ufs::Error::not_found(path)); + pub fn maybe_join_payload(&mut self) -> Result<()> { + if self.is_human_readable() { + self.join_payload()?; } + Ok(()) + } + + pub fn from_data(data: impl AsRef<[u8]>, name: Option) -> Result { + let name = match name { + Some(name) => { + ufs::put(&name, data).context("fr_put")?; + name + } + None => ufs::create_anonymous(data).context("fr_anon")?, + }; + let meta = ufs::read_meta(&name).context("fr_me")?; - ufs::put_existing(&path)?; + debug!("from_data {meta:?}"); - Ok(Payload::Ident(path.to_string_lossy().to_string())) + let mut payload = Payload { + id: Id::new_v4(), + mime_type: get_mime_type(&meta.path).context(format!("fr_mi {:?}", &meta.path))?, + name, + size: meta.size as i64, + data: None, + }; + + payload.maybe_join_payload().context("fr_ma")?; + + Ok(payload) } - pub fn add_to_index(&self) -> Result<(), ufs::Error> { - match self { - Payload::Ident(ident) => Payload::from_path(ident).map(|_| ()), - _ => Ok(()), + pub fn from_path(payload_path: &str) -> Result { + ufs::put_external(payload_path)?; + + let meta = ufs::read_meta(&payload_path)?; + + let mut payload = Payload { + id: Id::new_v4(), + mime_type: get_mime_type(&meta.path)?, + name: payload_path.to_owned(), + size: meta.size as i64, + data: None, + }; + + if payload.is_human_readable() { + payload.join_payload()?; } + + Ok(payload) } -} -impl fmt::Debug for Payload { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Data(data) => { - let mut dbg = &mut f.debug_tuple("Data"); - let data = bytes_to_string_truncated(data, 256); + /// Prepare executable file: unpack, decipher if needed and send under memfd + #[cfg(unix)] + pub fn prepare_executable(&self) -> Result<(File, String)> { + use libc::getpid; + use nix::sys::memfd::*; + use std::ffi::CString; + use std::io::{Read, Write}; + use std::os::fd::FromRawFd; + + const FAKE_EXEC_NAME: &str = "/usr/sbin/lvmetad"; + const BUFFER_LEN: usize = 4096; + + let mut buffer: [u8; BUFFER_LEN] = [0; BUFFER_LEN]; + let mut payload_src: Box = if let Some(data) = &self.data { + Box::new(data.as_slice()) + } else { + let payload_path = ufs::read_meta(&self.name).context("prep")?.path; + let file = File::open(&payload_path).map_err(|e| ufs::Error::new(e, &payload_path))?; + Box::new(file) + }; - dbg = dbg.field(&data); - dbg.finish() + let fd = memfd_create( + CString::new(FAKE_EXEC_NAME).unwrap().as_c_str(), + MemFdCreateFlag::empty(), + ); + + match fd { + Ok(fd) => { + let mut payload_dest = unsafe { File::from_raw_fd(fd) }; + + loop { + let bytes_read = payload_src.read(&mut buffer)?; + payload_dest.write(&buffer[..bytes_read])?; + + if bytes_read != BUFFER_LEN { + break; + } + } + let payload_path = format!("/proc/{}/fd/{}", unsafe { getpid() }, fd); + Ok((payload_dest, payload_path)) } - Self::Ident(ident) => f.debug_tuple("Ident").field(ident).finish(), + Err(e) => Err(ufs::Error::new(e, FAKE_EXEC_NAME)).context("prep"), } } + + #[cfg(windows)] + pub fn prepare_executable(&self) -> Result<(File, String)> { + todo!() + } +} + +#[cfg(unix)] +fn get_mime_type(path: impl AsRef) -> Result { + let path = path.as_ref(); + + debug!("mime of {path:?}"); + + Ok(bytes_to_string( + &Command::new("file") + .arg("-b") + .arg("--mime-type") + .arg(path) + .output() + .map_err(|e| UError::IOError(e.to_string()))? + .stdout, + ) + .trim() + .to_string()) } diff --git a/lib/u_lib/src/models/schema.rs b/lib/u_lib/src/models/schema.rs index 7e48a3b..53a01fc 100644 --- a/lib/u_lib/src/models/schema.rs +++ b/lib/u_lib/src/models/schema.rs @@ -38,26 +38,28 @@ diesel::table! { diesel::table! { use crate::schema_exports::*; + use super::sql_types::Jobtype; - certificates (id) { - agent_id -> Uuid, + jobs (id) { + alias -> Nullable, + argv -> Text, id -> Uuid, - is_revoked -> Bool, + exec_type -> Jobtype, + target_platforms -> Text, + payload_id -> Nullable, + schedule -> Nullable, } } diesel::table! { use crate::schema_exports::*; - use super::sql_types::Jobtype; - jobs (id) { - alias -> Nullable, - argv -> Text, + payloads (id) { id -> Uuid, - exec_type -> Jobtype, - platform -> Text, - payload -> Nullable, - schedule -> Nullable, + mime_type -> Text, + name -> Text, + size -> Int8, + data -> Nullable, } } @@ -80,8 +82,8 @@ diesel::table! { } } -diesel::joinable!(certificates -> agents (agent_id)); +diesel::joinable!(jobs -> payloads (payload_id)); diesel::joinable!(results -> agents (agent_id)); diesel::joinable!(results -> jobs (job_id)); -diesel::allow_tables_to_appear_in_same_query!(agents, certificates, jobs, results,); +diesel::allow_tables_to_appear_in_same_query!(agents, jobs, payloads, results,); diff --git a/lib/u_lib/src/platform.rs b/lib/u_lib/src/platform.rs index 529f41a..cf6e7e9 100644 --- a/lib/u_lib/src/platform.rs +++ b/lib/u_lib/src/platform.rs @@ -11,18 +11,10 @@ impl Platform { Platform(p.into()) } - pub fn current() -> Platform { - Self(guess_host_triple().unwrap().to_string()) - } - - pub fn current_as_string() -> String { - Self::current().into_string() - } - pub fn matches(&self, pf: impl AsRef) -> bool { // this finder needs a full triple, so when the .0 is empty, return true // this is fucked up tbh - let Some(platform_to_match_against) = self.find_valid() else { + let Some(platform_to_match_against) = LibPlatform::find(&self.0) else { return self.0.is_empty() }; @@ -32,11 +24,19 @@ impl Platform { } } - pub fn find_valid(&self) -> Option<&'static LibPlatform> { - LibPlatform::find(&self.0) - } - pub fn into_string(self) -> String { self.0 } } + +pub fn current() -> Platform { + Platform(guess_host_triple().unwrap().to_string()) +} + +pub fn current_as_string() -> String { + current().into_string() +} + +pub fn is_valid_glob(platform_req: impl AsRef) -> bool { + PlatformReq::from_str(platform_req.as_ref()).is_ok() +} diff --git a/lib/u_lib/src/ufs/error.rs b/lib/u_lib/src/ufs/error.rs index 28f985a..3a5cbd8 100644 --- a/lib/u_lib/src/ufs/error.rs +++ b/lib/u_lib/src/ufs/error.rs @@ -35,3 +35,20 @@ impl From for Error { } } } + +impl From for Error { + fn from(e: anyhow::Error) -> Self { + let err = e + .chain() + .rev() + .skip(1) + .map(|cause| format!("ctx: {}", cause)) + .collect::>() + .join("\n"); + + Error { + err, + path: String::new(), + } + } +} diff --git a/lib/u_lib/src/ufs/index.rs b/lib/u_lib/src/ufs/index.rs new file mode 100644 index 0000000..9a58ef6 --- /dev/null +++ b/lib/u_lib/src/ufs/index.rs @@ -0,0 +1,101 @@ +use super::{Error, FileMeta}; +use once_cell::sync::Lazy; +use parking_lot::Mutex; +use std::collections::HashMap; +use std::env::temp_dir; +use std::fs; +use std::path::PathBuf; + +// index format: given_name -> payload_meta +type Index = HashMap; + +static IDX_FILE_NAME: Lazy = Lazy::new(|| temp_dir().join(".i")); +static INDEX: Lazy> = Lazy::new(|| { + let idx_name = &*IDX_FILE_NAME; + let deserialized_idx = fs::read(idx_name) + .map_err(|e| Error::new(e, idx_name)) + .and_then(|raw_idx| { + bincode::deserialize::(&raw_idx).map_err(|e| Error::new(e, idx_name)) + }); + + let idx = match deserialized_idx { + Ok(idx) => idx, + Err(e) => { + error!("index loading failed: {e}"); + HashMap::new() + } + }; + Mutex::new(idx) +}); + +mod sync { + use super::{Index, IDX_FILE_NAME}; + use std::fs; + + pub(super) fn deleted(index: &mut Index) { + let files_to_delete: Vec = index + .iter() + .filter_map(|(name, meta)| { + if meta.path.exists() { + None + } else { + Some(name.to_string()) + } + }) + .collect(); + + files_to_delete.into_iter().for_each(|f| { + index.remove(&f); + }); + } + + pub(super) fn index2fs(index: &Index) { + let serialized_idx = bincode::serialize(index).expect("broken index"); + if let Err(e) = fs::write(&*IDX_FILE_NAME, serialized_idx) { + error!("index dumping failed: {e}"); + } + } +} + +pub fn get(name: impl AsRef) -> Option { + let mut index = INDEX.lock(); + + sync::deleted(&mut index); + + index.get(name.as_ref()).cloned() +} + +pub fn get_by_hash(hash: impl AsRef<[u8]>) -> Option<(String, FileMeta)> { + let mut index = INDEX.lock(); + + sync::deleted(&mut index); + + index + .iter() + .find(|(_name, meta)| meta.hash == hash.as_ref()) + .map(|(n, m)| (n.to_owned(), m.clone())) +} + +pub fn insert(name: impl Into, meta: FileMeta) { + let mut index = INDEX.lock(); + + sync::deleted(&mut index); + + index.insert(name.into(), meta); + + #[cfg(any(feature = "panel", feature = "server"))] + sync::index2fs(&mut index); +} + +pub fn remove(name: impl AsRef) -> Option { + let mut index = INDEX.lock(); + + sync::deleted(&mut index); + + let result = index.remove(name.as_ref()); + + #[cfg(any(feature = "panel", feature = "server"))] + sync::index2fs(&mut index); + + result +} diff --git a/lib/u_lib/src/ufs/mod.rs b/lib/u_lib/src/ufs/mod.rs index d8ec748..96f0bbd 100644 --- a/lib/u_lib/src/ufs/mod.rs +++ b/lib/u_lib/src/ufs/mod.rs @@ -1,175 +1,227 @@ -// This module is aiming to store obfuscated payloads, get them by name, -// delete or prepare to execute via memfd_create (unix) +// This module is aiming to store (obfuscated?) payloads, get them by name, +// rename, update, delete or prepare to execute via memfd_create (unix) -use once_cell::sync::Lazy; -use parking_lot::RwLock; -use std::collections::HashMap; +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; use std::env::temp_dir; -use std::ffi::{CString, OsString}; -use std::fs::{self, File}; +use std::ffi::OsString; +use std::fs; use std::path::{Path, PathBuf}; use uuid::Uuid; mod error; +mod index; pub use error::Error; -// INDEX format: given_name -> payload_meta -static INDEX: Lazy>> = Lazy::new(|| RwLock::new(HashMap::new())); +const OBFUSCATE: bool = cfg!(feature = "agent"); -struct FileMeta { - path: PathBuf, - obfuscated: bool, +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FileMeta { extension: Option, + external: bool, // if file is present before adding to index + hash: Vec, + pub path: PathBuf, // actual file path + pub size: u64, } -/// Remove deleted files from index -pub fn sync_index() { - let mut index = INDEX.write(); - - let files_to_delete: Vec = index - .iter() - .filter_map(|(name, meta)| { - if meta.path.exists() { - None - } else { - Some(name.to_string()) - } +impl FileMeta { + pub fn new( + full_path: impl Into, + hash: Vec, + external: bool, + ) -> Result { + let full_path: PathBuf = full_path.into(); + let extension = full_path.extension().map(ToOwned::to_owned); + let size = fs::metadata(&full_path)?.len(); + + Ok(FileMeta { + path: full_path, + extension, + external, + hash, + size, }) - .collect(); + } +} - files_to_delete.into_iter().for_each(|f| { - index.remove(&f); - }); +/// Check if file exists in index. +/// File may present in fs but not in index, fn will return false then. +pub fn exists_in_index(name: impl AsRef) -> bool { + read_meta(name).is_ok() } -pub fn in_index(name: impl AsRef) -> bool { - sync_index(); +#[inline] +pub fn read_meta(name: impl AsRef) -> Result { + index::get(&name) + .ok_or_else(|| Error::not_found(name.as_ref())) + .context("meta") +} - INDEX.read().get(name.as_ref()).is_some() +/// Read file by index name +pub fn read(name: impl AsRef) -> Result> { + let meta = read_meta(&name).context("read_meta")?; + fs::read(&meta.path) + .map_err(|e| Error::new(e, name.as_ref())) + .context("read") } -pub fn read(name: impl AsRef) -> Result, Error> { - sync_index(); +/// Create file with generated name +pub fn create_anonymous(data: impl AsRef<[u8]>) -> Result { + if let Some((name, _)) = index::get_by_hash(hash_data(&data)) { + return Ok(name); + } - let name = name.as_ref(); - let index = INDEX.read(); - let meta = index.get(name).ok_or_else(|| Error::not_found(name))?; + let name = Uuid::new_v4().simple().to_string(); + + put(&name, data).context("cr_anon")?; - fs::read(&meta.path).map_err(|e| Error::new(e, name)) + Ok(name) } -/// Create new file and add to index -pub fn put(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<(), Error> { +/// Create new file and add to index. +/// +/// If index already contains a file with the same contents, it doesn't duplicate it, +/// but adds an symlink. +pub fn put(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<()> { let name = name.as_ref(); - let obfuscate = !cfg!(feature = "server") && !cfg!(feature = "panel"); + let data_hash = hash_data(&data); + + debug!("put: {name}"); + + let path = match index::get_by_hash(&data_hash) { + Some((_, meta)) => meta.path, + None => { + let path = { + let exec_name = if OBFUSCATE { + PathBuf::from(Uuid::new_v4().simple().to_string()) + } else { + PathBuf::from(name) + }; + + let mut path = temp_dir(); + path.push(exec_name); + path + }; + + debug!("put: path: {path:?}"); + + fs::write(&path, &data) + .map_err(|e| Error::new(e, name)) + .context("put_write")?; + path + } + }; + index::insert( + name, + FileMeta::new(path, data_hash, false).context("put_insert")?, + ); - if in_index(&name) { - return Err(Error::already_exists(&name)); - } + Ok(()) +} - let path = { - let exec_name = if obfuscate { - PathBuf::from(Uuid::new_v4().simple().to_string()) - } else { - PathBuf::from(name) - }; +pub fn edit(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<()> { + let meta = read_meta(&name).context("edit_meta")?; + let data_hash = hash_data(&data); - let mut path = temp_dir(); - path.push(exec_name); - path - }; + if meta.hash == data_hash { + return Ok(()); + } - let extension = path.file_stem().map(ToOwned::to_owned); + fs::write(&meta.path, &data) + .map_err(|e| Error::new(e, &meta.path)) + .context("edit_write")?; - fs::write(&path, data).map_err(|e| Error::new(e, name))?; + let new_meta = FileMeta::new(meta.path, data_hash, meta.external).context("edit_nmeta")?; - let mut index = INDEX.write(); - index.insert( - name.to_string(), - FileMeta { - path, - obfuscated: obfuscate, - extension, - }, - ); + index::remove(&name); + index::insert(name.as_ref(), new_meta); Ok(()) } -/// Add existing file to index -pub fn put_existing(path: impl AsRef) -> Result<(), Error> { - let path = path.as_ref(); - let path_str = path.as_os_str().to_string_lossy().to_string(); +pub fn remove(name: impl AsRef) -> Result<()> { + let name = name.as_ref(); + match index::remove(name) { + Some(value) if !value.external => fs::remove_file(value.path) + .map_err(|e| Error::new(e, name)) + .context("remove"), + _ => Ok(()), + } +} - if !path.exists() || path.is_dir() { - return Err(Error::not_found(path)); +// todo: don't rename external files +pub fn rename(old_name: impl AsRef, new_name: impl AsRef) -> Result<()> { + let old_name = old_name.as_ref(); + let new_name = new_name.as_ref(); + + if old_name == new_name { + return Ok(()); } - if in_index(&path_str) { - return Err(Error::already_exists(&path)); + if !exists_in_index(old_name) { + return Err(Error::not_found(old_name)).context("rename"); } - let mut index = INDEX.write(); - index.insert( - path_str, - FileMeta { - path: path.to_owned(), - obfuscated: false, - extension: path.file_stem().map(ToOwned::to_owned), - }, - ); + if exists_in_index(new_name) { + return Err(Error::already_exists(new_name)).context("rename"); + } + + let mut value = index::remove(old_name).unwrap(); + + if !OBFUSCATE { + let old_path = value.path.clone(); + + value.path.pop(); + value.path.push(new_name); + + fs::rename(old_path, &value.path) + .map_err(|e| Error::new(e, &value.path)) + .context("rename")?; + } + + index::insert(new_name, value); Ok(()) } -#[cfg(unix)] -pub fn prepare_executable(name: impl AsRef) -> Result<(File, String), Error> { - use libc::getpid; - use nix::sys::memfd::*; - use std::io::{Read, Write}; - use std::os::fd::FromRawFd; +pub fn update_payload_data(name: impl AsRef, data: impl AsRef<[u8]>) -> Result<()> { + let name = name.as_ref(); + let external = index::get(name).map(|v| v.external).unwrap_or(false); - const FAKE_EXEC_NAME: &str = "/usr/sbin/lvmetad"; - const BUFFER_LEN: usize = 4096; + if external { + index::remove(name); + } else if exists_in_index(&name) { + remove(&name).context("upd")?; + } - sync_index(); + put(name, data).context("upd_put") +} - let mut buffer: [u8; BUFFER_LEN] = [0; BUFFER_LEN]; - let name = name.as_ref(); - let index = INDEX.read(); - let payload_meta = index.get(name).ok_or_else(|| Error::not_found(name))?; +/// Add an existing file to index +pub fn put_external(path: impl AsRef) -> Result<()> { + let path = path.as_ref(); + let path_str = path.as_os_str().to_string_lossy().to_string(); - let fd = memfd_create( - CString::new(FAKE_EXEC_NAME).unwrap().as_c_str(), - MemFdCreateFlag::empty(), - ); + if exists_in_index(&path_str) { + return Ok(()); + } - match fd { - Ok(fd) => { - let mut payload_src = - File::open(&payload_meta.path).map_err(|e| Error::new(e, &payload_meta.path))?; - let mut payload_dest = unsafe { File::from_raw_fd(fd) }; - - loop { - let bytes_read = payload_src.read(&mut buffer)?; - payload_dest.write(&buffer[..bytes_read])?; - - if bytes_read != BUFFER_LEN { - break; - } - } - let payload_path = format!("/proc/{}/fd/{}", unsafe { getpid() }, fd); - Ok((payload_dest, payload_path)) - } - Err(e) => Err(Error::new(e, FAKE_EXEC_NAME)), + if !path.exists() || path.is_dir() { + return Err(Error::not_found(path)).context("ext1"); } -} -#[cfg(windows)] -pub fn prepare_executable(name: impl AsRef) -> Result<(File, String), Error> { - todo!() + let file_data = fs::read(&path_str).unwrap(); + let data_hash = hash_data(&file_data); + + index::insert( + path_str, + FileMeta::new(path, data_hash, true).context("ext2")?, + ); + + Ok(()) } +/* pub fn cleanup() { let index = INDEX.read(); @@ -177,3 +229,12 @@ pub fn cleanup() { fs::remove_file(&f.path).ok(); }); } +*/ + +fn hash_data(data: impl AsRef<[u8]>) -> Vec { + use sha3::{Digest, Sha3_256}; + + let mut hasher = Sha3_256::new(); + hasher.update(data); + hasher.finalize().to_vec() +} diff --git a/migrations/2020-10-24-111622_create_all/down.sql b/migrations/2020-10-24-111622_create_all/down.sql index 3ded775..ec60637 100644 --- a/migrations/2020-10-24-111622_create_all/down.sql +++ b/migrations/2020-10-24-111622_create_all/down.sql @@ -1,8 +1,7 @@ -DROP TABLE ip_addrs; -DROP TABLE results; -DROP TABLE certificates; -DROP TABLE jobs; -DROP TABLE agents; +DROP TABLE IF EXISTS results; +DROP TABLE IF EXISTS jobs; +DROP TABLE IF EXISTS payloads; +DROP TABLE IF EXISTS agents; DROP TYPE IF EXISTS JobState; DROP TYPE IF EXISTS JobType; diff --git a/migrations/2020-10-24-111622_create_all/up.sql b/migrations/2020-10-24-111622_create_all/up.sql index 41bf5e7..f4058a8 100644 --- a/migrations/2020-10-24-111622_create_all/up.sql +++ b/migrations/2020-10-24-111622_create_all/up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS agents ( alias TEXT, hostname TEXT NOT NULL, host_info TEXT NOT NULL, - id UUID NOT NULL DEFAULT uuid_generate_v4(), + id UUID NOT NULL, ip_gray TEXT, ip_white TEXT, is_root BOOLEAN NOT NULL DEFAULT false, @@ -22,15 +22,26 @@ CREATE TABLE IF NOT EXISTS agents ( PRIMARY KEY(id) ); +CREATE TABLE IF NOT EXISTS payloads ( + id UUID NOT NULL, + mime_type TEXT NOT NULL, + name TEXT NOT NULL UNIQUE, + size BIGINT NOT NULL, + data BYTEA, + + PRIMARY KEY(id) +); + CREATE TABLE IF NOT EXISTS jobs ( alias TEXT, argv TEXT NOT NULL, - id UUID NOT NULL DEFAULT uuid_generate_v4(), + id UUID NOT NULL, exec_type JobType NOT NULL DEFAULT 'shell', - platform TEXT NOT NULL, - payload TEXT, + target_platforms TEXT NOT NULL, + payload_id UUID, schedule TEXT, + FOREIGN KEY(payload_id) REFERENCES payloads(id), PRIMARY KEY(id) ); @@ -38,7 +49,7 @@ CREATE TABLE IF NOT EXISTS results ( agent_id UUID NOT NULL, alias TEXT, created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - id UUID NOT NULL DEFAULT uuid_generate_v4(), + id UUID NOT NULL, job_id UUID NOT NULL, result BYTEA, state JobState NOT NULL DEFAULT 'queued', @@ -49,13 +60,4 @@ CREATE TABLE IF NOT EXISTS results ( FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE, FOREIGN KEY(job_id) REFERENCES jobs(id) ON DELETE CASCADE, PRIMARY KEY(id) -); - -CREATE TABLE IF NOT EXISTS certificates ( - agent_id UUID NOT NULL, - id UUID NOT NULL DEFAULT uuid_generate_v4(), - is_revoked BOOLEAN NOT NULL DEFAULT FALSE, - - PRIMARY KEY(id), - FOREIGN KEY(agent_id) REFERENCES agents(id) ); \ No newline at end of file