diff --git a/Cargo.lock b/Cargo.lock index e38168f..86caeac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -176,9 +176,9 @@ checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" [[package]] name = "argminmax" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52424b59d69d69d5056d508b260553afd91c57e21849579cd1f50ee8b8b88eaa" +checksum = "70f13d10a41ac8d2ec79ee34178d61e6f47a29c2edfe7ef1721c7383b0359e65" dependencies = [ "num-traits", ] @@ -402,7 +402,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", "synstructure", ] @@ -414,7 +414,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -491,7 +491,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -502,7 +502,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -557,7 +557,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" dependencies = [ "base64 0.22.1", - "http 1.1.0", + "http 1.4.0", "log", "url", ] @@ -570,7 +570,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -589,10 +589,10 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.1.0", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.0", + "hyper 1.8.1", "hyper-util", "itoa", "matchit", @@ -607,7 +607,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tower", + "tower 0.5.1", "tower-layer", "tower-service", "tracing", @@ -622,7 +622,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.1.0", + "http 1.4.0", "http-body 1.0.1", "http-body-util", "mime", @@ -745,6 +745,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + [[package]] name = "brotli" version = "7.0.0" @@ -790,22 +801,22 @@ checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" [[package]] name = "bytemuck" -version = "1.19.0" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.8.0" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -917,8 +928,10 @@ checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-targets 0.52.6", ] @@ -1013,7 +1026,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -1259,9 +1272,9 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" dependencies = [ "crossbeam-utils", ] @@ -1413,7 +1426,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -1437,7 +1450,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -1448,7 +1461,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -1498,7 +1511,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ "serde", - "uuid 1.11.0", + "uuid 1.19.0", ] [[package]] @@ -1563,7 +1576,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -1628,7 +1641,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -1695,9 +1708,9 @@ dependencies = [ [[package]] name = "either" -version = "1.13.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "elliptic-curve" @@ -1763,7 +1776,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -1775,7 +1788,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -1786,12 +1799,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1930,7 +1943,7 @@ dependencies = [ "reqwest 0.11.27", "serde", "serde_json", - "syn 2.0.98", + "syn 2.0.111", "toml", "walkdir", ] @@ -1948,7 +1961,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -1974,7 +1987,7 @@ dependencies = [ "serde", "serde_json", "strum", - "syn 2.0.98", + "syn 2.0.111", "tempfile", "thiserror 1.0.65", "tiny-keccak", @@ -2283,9 +2296,15 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "foreign-types" @@ -2414,7 +2433,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -2591,7 +2610,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.1.0", + "http 1.4.0", "indexmap 2.6.0", "slab", "tokio", @@ -2655,7 +2674,18 @@ checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] @@ -2843,12 +2873,11 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", - "fnv", "itoa", ] @@ -2870,7 +2899,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http 1.4.0", ] [[package]] @@ -2881,7 +2910,7 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", + "http 1.4.0", "http-body 1.0.1", "pin-project-lite", ] @@ -2924,20 +2953,22 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ + "atomic-waker", "bytes", "futures-channel", - "futures-util", + "futures-core", "h2 0.4.6", - "http 1.1.0", + "http 1.4.0", "http-body 1.0.1", "httparse", "httpdate", "itoa", "pin-project-lite", + "pin-utils", "smallvec", "tokio", "want", @@ -2964,8 +2995,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", - "http 1.1.0", - "hyper 1.5.0", + "http 1.4.0", + "hyper 1.8.1", "hyper-util", "rustls 0.23.15", "rustls-pki-types", @@ -2974,6 +3005,19 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper 1.8.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.6.0" @@ -2982,7 +3026,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.5.0", + "hyper 1.8.1", "hyper-util", "native-tls", "tokio", @@ -2992,18 +3036,20 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.9" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" dependencies = [ "bytes", "futures-channel", + "futures-core", "futures-util", - "http 1.1.0", + "http 1.4.0", "http-body 1.0.1", - "hyper 1.5.0", + "hyper 1.8.1", + "libc", "pin-project-lite", - "socket2 0.5.7", + "socket2 0.6.1", "tokio", "tower-service", "tracing", @@ -3147,7 +3193,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -3230,9 +3276,9 @@ dependencies = [ "attohttpc", "bytes", "futures", - "http 1.1.0", + "http 1.4.0", "http-body-util", - "hyper 1.5.0", + "hyper 1.8.1", "hyper-util", "log", "rand 0.9.2", @@ -3393,6 +3439,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -3416,13 +3471,25 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" dependencies = [ + "once_cell", "wasm-bindgen", ] +[[package]] +name = "jsonpath_lib_polars_vendor" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4bd9354947622f7471ff713eacaabdb683ccb13bba4edccaab9860abf480b7d" +dependencies = [ + "log", + "serde", + "serde_json", +] + [[package]] name = "jsonwebtoken" version = "8.3.0" @@ -3921,7 +3988,7 @@ source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=c0ed330#c0ed330 dependencies = [ "heck 0.5.0", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -4191,7 +4258,7 @@ dependencies = [ "smallvec", "tagptr", "thiserror 1.0.65", - "uuid 1.11.0", + "uuid 1.19.0", ] [[package]] @@ -4520,7 +4587,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -4611,7 +4678,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -4724,7 +4791,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -4803,7 +4870,7 @@ dependencies = [ "arrow-schema", "arrow-select", "base64 0.22.1", - "brotli", + "brotli 7.0.0", "bytes", "chrono", "flate2", @@ -4827,6 +4894,10 @@ name = "parquet-format-safe" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1131c54b167dd4e4799ce762e1ab01549ebb94d5bdd13e6ec1b467491c378e1f" +dependencies = [ + "async-trait", + "futures", +] [[package]] name = "parse-zoneinfo" @@ -4860,6 +4931,43 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" +[[package]] +name = "pbjson" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e6349fa080353f4a597daffd05cb81572a9c031a6d4fff7e504947496fcc68" +dependencies = [ + "base64 0.21.7", + "serde", +] + +[[package]] +name = "pbjson-build" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eea3058763d6e656105d1403cb04e0a41b7bbac6362d413e7c33be0c32279c9" +dependencies = [ + "heck 0.5.0", + "itertools 0.13.0", + "prost 0.13.5", + "prost-types 0.13.5", +] + +[[package]] +name = "pbjson-types" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e54e5e7bfb1652f95bc361d76f3c780d8e526b134b85417e774166ee941f0887" +dependencies = [ + "bytes", + "chrono", + "pbjson", + "pbjson-build", + "prost 0.13.5", + "prost-build 0.13.5", + "serde", +] + [[package]] name = "pbkdf2" version = "0.11.0" @@ -4967,7 +5075,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -5005,7 +5113,7 @@ checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -5113,6 +5221,7 @@ dependencies = [ "either", "ethnum", "fast-float", + "futures", "getrandom 0.2.15", "hashbrown 0.14.5", "itoa", @@ -5230,10 +5339,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "454ebbebe1cb8cb4768adca44b8fc9431abc3c91d5927f6824e73f916bced911" dependencies = [ "ahash", + "async-trait", "atoi_simd", "bytes", "chrono", "fast-float", + "futures", "glob", "hashbrown 0.14.5", "home", @@ -5247,13 +5358,17 @@ dependencies = [ "polars-core", "polars-error", "polars-json", + "polars-parquet", "polars-schema", "polars-time", "polars-utils", "rayon", "regex", "ryu", + "simd-json", "simdutf8", + "tokio", + "tokio-util", ] [[package]] @@ -5291,6 +5406,7 @@ dependencies = [ "polars-core", "polars-expr", "polars-io", + "polars-json", "polars-mem-engine", "polars-ops", "polars-pipe", @@ -5313,6 +5429,7 @@ dependencies = [ "polars-error", "polars-expr", "polars-io", + "polars-json", "polars-ops", "polars-plan", "polars-time", @@ -5336,16 +5453,19 @@ dependencies = [ "hashbrown 0.14.5", "hex", "indexmap 2.6.0", + "jsonpath_lib_polars_vendor", "memchr", "num-traits", "polars-arrow", "polars-compute", "polars-core", "polars-error", + "polars-json", "polars-schema", "polars-utils", "rayon", "regex", + "serde_json", "unicode-reverse", "version_check", ] @@ -5357,10 +5477,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "491f5af321169259d5b1294c9fe8ed89faaeac34b4dec4abcedc0d1b3d11013a" dependencies = [ "ahash", + "async-stream", "base64 0.22.1", + "brotli 6.0.0", "bytemuck", "ethnum", + "flate2", + "futures", "hashbrown 0.14.5", + "lz4", "num-traits", "parquet-format-safe", "polars-arrow", @@ -5368,7 +5493,9 @@ dependencies = [ "polars-error", "polars-utils", "simdutf8", + "snap", "streaming-decompression", + "zstd 0.13.3", ] [[package]] @@ -5380,6 +5507,7 @@ dependencies = [ "crossbeam-channel", "crossbeam-queue", "enum_dispatch", + "futures", "hashbrown 0.14.5", "num-traits", "polars-arrow", @@ -5392,7 +5520,7 @@ dependencies = [ "polars-row", "polars-utils", "rayon", - "uuid 1.11.0", + "uuid 1.19.0", "version_check", ] @@ -5416,7 +5544,9 @@ dependencies = [ "polars-arrow", "polars-core", "polars-io", + "polars-json", "polars-ops", + "polars-parquet", "polars-time", "polars-utils", "rayon", @@ -5571,12 +5701,12 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "prettyplease" -version = "0.2.24" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "910d41a655dac3b764f1ade94821093d3610248694320cd072303a8eedcf221d" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -5619,7 +5749,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", "version_check", "yansi 1.0.1", ] @@ -5644,7 +5774,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -5675,12 +5805,12 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive 0.13.3", + "prost-derive 0.13.5", ] [[package]] @@ -5698,9 +5828,29 @@ dependencies = [ "petgraph", "prettyplease", "prost 0.12.6", - "prost-types", + "prost-types 0.12.6", + "regex", + "syn 2.0.111", + "tempfile", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck 0.5.0", + "itertools 0.13.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost 0.13.5", + "prost-types 0.13.5", "regex", - "syn 2.0.98", + "syn 2.0.111", "tempfile", ] @@ -5714,20 +5864,20 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] name = "prost-derive" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -5739,6 +5889,15 @@ dependencies = [ "prost 0.12.6", ] +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost 0.13.5", +] + [[package]] name = "psm" version = "0.1.23" @@ -5980,7 +6139,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -6020,7 +6179,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -6067,6 +6226,16 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "regress" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2057b2325e68a893284d1538021ab90279adac1139957ca2a74426c6f118fb48" +dependencies = [ + "hashbrown 0.16.1", + "memchr", +] + [[package]] name = "reqwest" version = "0.11.27" @@ -6121,10 +6290,10 @@ dependencies = [ "futures-core", "futures-util", "h2 0.4.6", - "http 1.1.0", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.0", + "hyper 1.8.1", "hyper-rustls 0.27.3", "hyper-tls", "hyper-util", @@ -6474,6 +6643,30 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "schemars" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.111", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -6652,7 +6845,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df141464944fdf8e2a6f2184eb1d973a20456466f788346b6e3a51791cdaa370" dependencies = [ "axum", - "http 1.1.0", + "http 1.4.0", "pin-project", "sentry-core", "tower-layer", @@ -6686,7 +6879,7 @@ dependencies = [ "thiserror 1.0.65", "time", "url", - "uuid 1.11.0", + "uuid 1.19.0", ] [[package]] @@ -6697,10 +6890,11 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.213" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea7893ff5e2466df8d720bb615088341b295f849602c6956047f8f80f0e9bc1" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] @@ -6710,15 +6904,35 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "794e44574226fc701e3be5c651feb7939038fc67fb73f6f4dd5c4ba90fd3be70" +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + [[package]] name = "serde_derive" -version = "1.0.213" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -6753,6 +6967,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_tokenstream" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64060d864397305347a78851c51588fd283767e7e7589829e8121d65512340f1" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "syn 2.0.111", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -6792,7 +7018,20 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.6.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", ] [[package]] @@ -6977,7 +7216,7 @@ dependencies = [ [[package]] name = "sqd-assignments" version = "0.1.0" -source = "git+https://github.com/subsquid/sqd-network.git?rev=fd4c694#fd4c694e21481fdac1381fe3f94f5c727a4a198c" +source = "git+https://github.com/subsquid/sqd-network.git?rev=2a7cb0e#2a7cb0e5755cd872be93826a49ee2fd93c545d76" dependencies = [ "anyhow", "crypto_box", @@ -7001,7 +7240,7 @@ dependencies = [ [[package]] name = "sqd-contract-client" version = "1.2.1" -source = "git+https://github.com/subsquid/sqd-network.git?rev=fd4c694#fd4c694e21481fdac1381fe3f94f5c727a4a198c" +source = "git+https://github.com/subsquid/sqd-network.git?rev=2a7cb0e#2a7cb0e5755cd872be93826a49ee2fd93c545d76" dependencies = [ "async-trait", "clap", @@ -7020,14 +7259,14 @@ dependencies = [ [[package]] name = "sqd-messages" version = "2.1.0" -source = "git+https://github.com/subsquid/sqd-network.git?rev=fd4c694#fd4c694e21481fdac1381fe3f94f5c727a4a198c" +source = "git+https://github.com/subsquid/sqd-network.git?rev=2a7cb0e#2a7cb0e5755cd872be93826a49ee2fd93c545d76" dependencies = [ "bytemuck", "flate2", "hex", "libp2p-identity", - "prost 0.13.3", - "prost-build", + "prost 0.13.5", + "prost-build 0.12.6", "semver", "serde", "sha3", @@ -7036,7 +7275,7 @@ dependencies = [ [[package]] name = "sqd-network-transport" version = "3.0.0" -source = "git+https://github.com/subsquid/sqd-network.git?rev=fd4c694#fd4c694e21481fdac1381fe3f94f5c727a4a198c" +source = "git+https://github.com/subsquid/sqd-network.git?rev=2a7cb0e#2a7cb0e5755cd872be93826a49ee2fd93c545d76" dependencies = [ "anyhow", "async-trait", @@ -7055,7 +7294,7 @@ dependencies = [ "parking_lot", "pin-project", "prometheus-client", - "prost 0.13.3", + "prost 0.13.5", "rand 0.9.2", "serde", "sqd-contract-client", @@ -7109,7 +7348,7 @@ dependencies = [ [[package]] name = "sqd-worker" -version = "2.6.1" +version = "2.7.0" dependencies = [ "anyhow", "async-compression", @@ -7119,6 +7358,7 @@ dependencies = [ "base64 0.21.7", "bs58", "camino", + "chrono", "clap", "criterion", "crypto_box", @@ -7132,8 +7372,10 @@ dependencies = [ "lazy_static", "mimalloc", "parking_lot", + "polars", + "polars-plan", "prometheus-client", - "prost 0.12.6", + "prost 0.13.5", "rand 0.9.2", "regex", "reqwest 0.12.8", @@ -7152,6 +7394,8 @@ dependencies = [ "sqd-network-transport", "sqd-polars", "sqd-query", + "sql_query_plan", + "substrait", "thiserror 1.0.65", "tokio", "tokio-rusqlite", @@ -7166,6 +7410,24 @@ dependencies = [ "zstd 0.13.3", ] +[[package]] +name = "sql_query_plan" +version = "0.1.0" +source = "git+https://github.com/subsquid/qplan.git?rev=658f88f#658f88f9bdcf52915a2e1ca315853a698d1a60b0" +dependencies = [ + "chrono", + "futures", + "prost 0.13.5", + "serde", + "serde_derive", + "serde_json", + "substrait", + "thiserror 2.0.11", + "tonic", + "tracing", + "uuid 1.19.0", +] + [[package]] name = "sqlparser" version = "0.49.0" @@ -7259,7 +7521,35 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.98", + "syn 2.0.111", +] + +[[package]] +name = "substrait" +version = "0.48.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "441543aa067ca8d745a25c277d66bab78ffa3b94253b5e30c38856440613eeb1" +dependencies = [ + "heck 0.5.0", + "hex", + "pbjson", + "pbjson-build", + "pbjson-types", + "prettyplease", + "prost 0.13.5", + "prost-build 0.13.5", + "prost-types 0.13.5", + "regress", + "schemars", + "semver", + "serde", + "serde_json", + "serde_yaml", + "syn 2.0.111", + "thiserror 2.0.11", + "typify", + "url", + "walkdir", ] [[package]] @@ -7301,9 +7591,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.98" +version = "2.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" dependencies = [ "proc-macro2", "quote", @@ -7333,7 +7623,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -7459,7 +7749,7 @@ checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -7470,7 +7760,7 @@ checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -7571,32 +7861,31 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.0" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" dependencies = [ - "backtrace", "bytes", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.7", + "socket2 0.6.1", "tokio-macros", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -7715,6 +8004,56 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2 0.4.6", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost 0.13.5", + "socket2 0.5.7", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.1" @@ -7740,7 +8079,7 @@ dependencies = [ "bitflags 2.9.1", "bytes", "futures-util", - "http 1.1.0", + "http 1.4.0", "http-body 1.0.1", "http-body-util", "pin-project-lite", @@ -7781,7 +8120,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -7893,6 +8232,53 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "typify" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c644dda9862f0fef3a570d8ddb3c2cfb1d5ac824a1f2ddfa7bc8f071a5ad8a" +dependencies = [ + "typify-impl", + "typify-macro", +] + +[[package]] +name = "typify-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59ab345b6c0d8ae9500b9ff334a4c7c0d316c1c628dc55726b95887eb8dbd11" +dependencies = [ + "heck 0.5.0", + "log", + "proc-macro2", + "quote", + "regress", + "schemars", + "semver", + "serde", + "serde_json", + "syn 2.0.111", + "thiserror 1.0.65", + "unicode-ident", +] + +[[package]] +name = "typify-macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "785e2cdcef0df8160fdd762ed548a637aaec1e83704fdbc14da0df66013ee8d0" +dependencies = [ + "proc-macro2", + "quote", + "schemars", + "semver", + "serde", + "serde_json", + "serde_tokenstream", + "syn 2.0.111", + "typify-impl", +] + [[package]] name = "uint" version = "0.9.5" @@ -7975,6 +8361,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "unsigned-varint" version = "0.7.2" @@ -8066,12 +8458,14 @@ dependencies = [ [[package]] name = "uuid" -version = "1.11.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" dependencies = [ - "getrandom 0.2.15", - "serde", + "getrandom 0.3.1", + "js-sys", + "serde_core", + "wasm-bindgen", ] [[package]] @@ -8140,27 +8534,14 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.95" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.98", "wasm-bindgen-shared", ] @@ -8178,9 +8559,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -8188,22 +8569,25 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.98", - "wasm-bindgen-backend", + "syn 2.0.111", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-streams" @@ -8363,7 +8747,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -8374,7 +8758,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -8385,7 +8769,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -8396,7 +8780,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -8480,6 +8864,15 @@ dependencies = [ "windows-targets 0.53.5", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -8813,7 +9206,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", "synstructure", ] @@ -8844,7 +9237,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -8855,7 +9248,7 @@ checksum = "eea57037071898bf96a6da35fd626f4f27e9cee3ead2a6c703cf09d472b2e700" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] @@ -8875,7 +9268,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", "synstructure", ] @@ -8904,7 +9297,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.111", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a5d490a..8e36533 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "sqd-worker" license = "AGPL-3.0-or-later" -version = "2.6.1" +version = "2.7.0" edition = "2021" [dependencies] @@ -15,6 +15,7 @@ bs58 = "0.5.1" camino = "1.1.6" clap = { version = "4.4.18", features = ["derive", "env"] } criterion = { version = "0.5.1", features = ["async_tokio"] } +chrono = "0.4" crypto_box = "0.9.1" curve25519-dalek = "4.1.3" dotenv = "0.15.0" @@ -27,10 +28,12 @@ lazy_static = "1.4.0" mimalloc = "0.1.43" parking_lot = "0.12.1" prometheus-client = "0.23" -prost = "0.12.3" +polars-plan = "0.43.1" +polars = { version = "0.43.1", features = ["json", "parquet"] } +prost = "0.13.5" rand = "0.9.2" regex = "1.10.2" -reqwest = { version = "0.12.4", features = ["json", "stream"] } +reqwest = { version = "0.12.8", features = ["json", "stream"] } scopeguard = "1.2.0" sentry = { version = "0.32.2", features = ["tracing"] } sentry-tower = { version = "0.32.2", features = ["axum", "http"] } @@ -40,6 +43,7 @@ serde_json = { version = "1.0.111", features = ["preserve_order"] } serde_with = { version = "3.11.0", features = ["base64"] } sha2 = "0.10.8" sha3 = "0.10.8" +substrait = { version = "0.48.0", features = ["serde", "parse"] } thiserror = "1.0.57" tokio = { version = "1.35.1", features = ["full", "tracing", "test-util"] } tokio-rusqlite = "0.5.1" @@ -53,12 +57,13 @@ url = "2.5.2" walkdir = "2.5.0" zstd = "0.13" -sqd-assignments = { git = "https://github.com/subsquid/sqd-network.git", rev = "fd4c694", features = ["reader"] } -sqd-contract-client = { git = "https://github.com/subsquid/sqd-network.git", rev = "fd4c694", version = "1.2.1" } -sqd-messages = { git = "https://github.com/subsquid/sqd-network.git", rev = "fd4c694", version = "2.0.2", features = ["bitstring"] } -sqd-network-transport = { git = "https://github.com/subsquid/sqd-network.git", rev = "fd4c694", version = "3.0.0", features = ["worker", "metrics"] } +sqd-assignments = { git = "https://github.com/subsquid/sqd-network.git", rev = "2a7cb0e", features = ["reader"] } +sqd-contract-client = { git = "https://github.com/subsquid/sqd-network.git", rev = "2a7cb0e", version = "1.2.1" } +sqd-messages = { git = "https://github.com/subsquid/sqd-network.git", rev = "2a7cb0e", version = "2.0.2", features = ["bitstring"] } +sqd-network-transport = { git = "https://github.com/subsquid/sqd-network.git", rev = "2a7cb0e", version = "3.0.0", features = ["worker", "metrics"] } sqd-query = { git = "https://github.com/subsquid/data.git", rev = "4c089d8", features = ["parquet"] } sqd-polars = { git = "https://github.com/subsquid/data.git", rev = "4c089d8" } +sql_query_plan = {git = "https://github.com/subsquid/qplan.git", rev = "658f88f" } [profile.release] debug = true diff --git a/Dockerfile b/Dockerfile index 636b6c0..9f6131d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # See https://www.lpalmieri.com/posts/fast-rust-docker-builds/#cargo-chef for explanation -FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-1.83-slim-bookworm AS chef +FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-1.89-slim AS chef WORKDIR /app diff --git a/src/controller/mod.rs b/src/controller/mod.rs index 9267e79..6b409dd 100644 --- a/src/controller/mod.rs +++ b/src/controller/mod.rs @@ -1,3 +1,5 @@ pub mod assignments; pub mod p2p; +pub mod polars_target; +pub mod sql_request; pub mod worker; diff --git a/src/controller/p2p.rs b/src/controller/p2p.rs index 0c0d4b2..f8f7937 100644 --- a/src/controller/p2p.rs +++ b/src/controller/p2p.rs @@ -23,6 +23,7 @@ use crate::{ self, allocations_checker::{self, AllocationsChecker}, }, + controller::worker::QueryType, logs_storage::LogsStorage, metrics, query::result::{QueryError, QueryResult}, @@ -60,6 +61,9 @@ pub struct P2PController { queries_tx: mpsc::Sender<(PeerId, Query, ResponseChannel)>, queries_rx: UseOnce)>>, + sql_queries_tx: mpsc::Sender<(PeerId, Query, ResponseChannel)>, + sql_queries_rx: + UseOnce)>>, log_requests_tx: mpsc::Sender<(LogsRequest, ResponseChannel)>, log_requests_rx: UseOnce)>>, } @@ -91,6 +95,7 @@ pub async fn create_p2p_controller( let (event_stream, transport_handle) = transport_builder.build_worker(config).await?; let (queries_tx, queries_rx) = mpsc::channel(QUERIES_POOL_SIZE); + let (sql_queries_tx, sql_queries_rx) = mpsc::channel(QUERIES_POOL_SIZE); let (log_requests_tx, log_requests_rx) = mpsc::channel(LOG_REQUESTS_QUEUE_SIZE); Ok(P2PController { @@ -108,6 +113,8 @@ pub async fn create_p2p_controller( assignment_url: args.assignment_url, queries_tx, queries_rx: UseOnce::new(queries_rx), + sql_queries_tx, + sql_queries_rx: UseOnce::new(sql_queries_rx), log_requests_tx, log_requests_rx: UseOnce::new(log_requests_rx), }) @@ -123,6 +130,10 @@ impl + Send + 'static> P2PController + Send + 'static> P2PController + Send + 'static> P2PController + Send + 'static> P2PController, cancellation_token: CancellationToken) { + let sql_queries_rx = self.sql_queries_rx.take().unwrap(); + ReceiverStream::new(sql_queries_rx) + .take_until(cancellation_token.cancelled_owned()) + .for_each_concurrent(CONCURRENT_QUERY_MESSAGES, |(peer_id, query, resp_chan)| { + let this = self.clone(); + tokio::spawn(async move { + this.handle_query(peer_id, query, resp_chan, QueryType::SqlQuery) + .await; + }) + .map(|r| r.unwrap()) + }) + .await; + info!("SQL Query processing task finished"); + } + async fn run_assignments_loop( &self, cancellation_token: CancellationToken, @@ -298,6 +327,24 @@ impl + Send + 'static> P2PController { + if !self.validate_query(&query, peer_id) { + continue; + } + match self.sql_queries_tx.try_send((peer_id, query, resp_chan)) { + Ok(_) => {} + Err(mpsc::error::TrySendError::Full(_)) => { + warn!("SQL Queries queue is full. Dropping query from {peer_id}"); + } + Err(mpsc::error::TrySendError::Closed(_)) => { + break; + } + } + } WorkerEvent::LogsRequest { request, resp_chan } => { match self.log_requests_tx.try_send((request, resp_chan)) { Ok(_) => {} @@ -351,11 +398,12 @@ impl + Send + 'static> P2PController, + query_type: QueryType, ) { let query_id = query.query_id.clone(); let compression = query.compression(); - let (result, retry_after) = self.process_query(peer_id, &query).await; + let (result, retry_after) = self.process_query(peer_id, &query, query_type).await; if let Err(e) = &result { warn!("Query {query_id} by {peer_id} execution failed: {e:?}"); } @@ -394,6 +442,7 @@ impl + Send + 'static> P2PController (QueryResult, Option) { match query.compression { c if c == sqd_messages::Compression::Gzip as i32 @@ -428,6 +477,7 @@ impl + Send + 'static> P2PController(msg: String) -> Result { + Err(PolarsTargetErr::PolarsTarget(msg)) +} + +/// The result type for this specific target plan. +pub type PolarsTargetResult = Result; + +/// Interface to the datastore; we use a parquet reader in production +/// and a mock that just produces predefined data for testing. +pub trait DataStore { + fn get_data_source( + &self, + dataset: &str, + table: &str, + chunks: &[String], + ) -> PolarsTargetResult; +} + +/// Trait implementation for Target Plan. +/// The implementation defines what we essentially need from the original +/// substrait plan to generate this kind of plan, we are, for example, +/// not interested in joins in the polars target. +#[derive(Clone, Debug)] +pub enum PolarsTarget { + Empty, + Relation(RelationType, Vec, Box), + Source(Source), +} + +impl PolarsTarget { + /// Compiles the Target Plan to Polars expressions. + pub fn compile( + &self, + tctx: &TraversalContext, + bucket: &str, + chunk_ids: &[String], + ds: &S, + ) -> PolarsTargetResult> { + let src = self.get_source(); + if src.is_none() { + return Ok(None); + } + let src = src.unwrap(); + match self { + PolarsTarget::Empty => Ok(None), + PolarsTarget::Relation(rt, x, kid) => { + if let Some(input) = kid.compile(tctx, bucket, chunk_ids, ds)? { + compile_relation(rt, src, &x, tctx, input) + } else { + Ok(None) + } + } + PolarsTarget::Source(src) => { + if !src.sqd { + Ok(None) + } else { + let lf = ds.get_data_source(&bucket, &src.table_name, chunk_ids)?; + Ok(Some(lf)) + } + } + } + } +} + +fn compile_relation( + rt: &RelationType, + src: &Source, + x: &[Expression], + tctx: &TraversalContext, + lf: LazyFrame, +) -> PolarsTargetResult> { + let lf = match rt { + RelationType::Projection => compile_projection(src, x, tctx, lf)?, + RelationType::Filter if x.len() > 0 => { + lf.filter(PolarsExprTransformer::from_filter().transform_expr(&x[0], src, tctx)?) + } + RelationType::Filter => lf, + RelationType::Fetch(_, c) => lf.limit(*c as u32), //TODO: do the conversion in Fetch! + RelationType::Other(s) if s == &"reduced join" => lf, + RelationType::Other(_) => lf, + }; + Ok(Some(lf)) +} + +fn compile_projection( + src: &Source, + x: &[Expression], + tctx: &TraversalContext, + lf: LazyFrame, +) -> PolarsTargetResult { + Ok(if !src.projection.is_empty() { + lf.select(project_from_source(src, tctx)?) + } else { + lf.select(convert_expressions(x, src, tctx)?) + }) +} + +fn project_from_source(src: &Source, _tctx: &TraversalContext) -> PolarsTargetResult> { + Ok(map_through_projection(src)) +} + +fn map_through_projection(src: &Source) -> Vec { + let mut v = Vec::new(); + for idx in &src.projection { + if *idx < src.fields.len() { + // This is a very cheap workaround for a non-supported serialisation in polars-json + // It does not support BLOB. + if src.fields[*idx] == "accounts_bloom" { + v.push(lit("ignore").alias("accounts_bloom")); + } else { + v.push(col(src.fields[*idx].clone())); + } + } + } + v +} + +impl TargetPlan for PolarsTarget { + /// Leaf. + fn empty() -> Self { + PolarsTarget::Empty + } + + /// Store a generic relation in the Target Plan. + fn from_relation( + relt: RelationType, + exps: &[Expression], + _from: &Rel, + rel: Self, + ) -> PlanResult { + Ok(PolarsTarget::Relation(relt, exps.to_vec(), Box::new(rel))) + } + + /// Queries containing joins generate an error, because we don't handle joins locally! + fn from_join(_exps: &[Expression], _from: &Rel, _left: Self, _right: Self) -> PlanResult { + plan_err("joins are not supported on worker side".to_string()) + } + + /// Store the source in the Target Plan. + fn from_source(source: Source) -> Self { + PolarsTarget::Source(source) + } + + /// Get the source (since we have no joins, there is only one source). + fn get_source(&self) -> Option<&Source> { + match self { + PolarsTarget::Empty => None, + PolarsTarget::Relation(_, _, kid) => kid.get_source(), + PolarsTarget::Source(src) => Some(src), + } + } + + /// Since we don't support joins on worker side, + /// we don't need to bother about getting more than one source. + fn get_sources(&self) -> Vec { + Vec::with_capacity(0) + } +} + +// Get the expressions for the Select context +// Careful! We need to check if the expression actually relates to the source! +// We usually use the source.projection, only if that is None we use the projection rel. +// The only case when that *should* happen is when there is only one source in the query anyway. +fn convert_expressions( + xs: &[Expression], + source: &Source, + tctx: &TraversalContext, +) -> PolarsTargetResult> { + let mut v = Vec::new(); + for x in xs { + v.push(PolarsExprTransformer::from_projection().transform_expr(x, source, tctx)?); + } + Ok(v) +} + +enum LitType { + LiteralInt, + LiteralStr, +} + +struct PolarsExprTransformer { + convert_tp_to_str: bool, + convert_str_to_tp: bool, +} + +impl PolarsExprTransformer { + fn from_filter() -> PolarsExprTransformer { + PolarsExprTransformer { + convert_tp_to_str: false, + convert_str_to_tp: true, + } + } + fn from_projection() -> PolarsExprTransformer { + PolarsExprTransformer { + convert_tp_to_str: true, + convert_str_to_tp: false, + } + } + fn literal_list_to_series( + &self, + es: &[Expression], + s: &Source, + tctx: &TraversalContext, + ) -> PolarsTargetResult { + let t = match self.transform_expr(&es[0], s, tctx)? { + Expr::Literal(LiteralValue::Int(_)) => Ok(LitType::LiteralInt), + Expr::Literal(LiteralValue::String(_)) => Ok(LitType::LiteralStr), + _ => polars_err("unexpected literal".to_string()), + }?; + + match t { + LitType::LiteralInt => { + let mut os = Vec::new(); + for e in es.iter() { + os.push(Self::expr_to_int(&self.transform_expr(e, s, tctx)?)?); + } + Ok(polars::series::Series::new("".into(), os)) + } + LitType::LiteralStr => { + let mut os = Vec::new(); + for e in es.iter() { + os.push(Self::expr_to_str(&self.transform_expr(e, s, tctx)?)?); + } + Ok(polars::series::Series::new("".into(), os)) + } + } + } + + fn expr_to_int(e: &Expr) -> PolarsTargetResult { + match e { + Expr::Literal(LiteralValue::Int(i)) => Ok(*i), + _ => polars_err("unexpected literal".to_string()), + } + } + + // TODO: string or timestamp + fn expr_to_str(e: &Expr) -> PolarsTargetResult { + match e { + Expr::Literal(LiteralValue::String(s)) => Ok(s.to_string()), + _ => polars_err("unexpected literal".to_string()), + } + } + + // TODO: we won't need that in the future, the conversion is done by duckdb anyway. + fn get_col_from_source(&self, source: &Source, r: i32) -> PolarsTargetResult { + let nm = Self::get_field_name_from_source(source, r)?; + if self.convert_tp_to_str && TIMESTAMP_FIELD_NAMES.contains(&nm.as_str()) { + Ok(col(nm).dt().strftime("%Y-%m-%d %H:%M:%S")) + } else { + Ok(col(nm)) + } + } + + fn string_or_timestamp(&self, s: &str) -> Result { + if self.convert_str_to_tp { + match chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") { + Ok(t) => Ok(lit(t.and_utc().timestamp())), + _ => Ok(lit(s.to_string())), + } + } else { + Ok(lit(s.to_string())) + } + } +} + +impl ExprTransformer for PolarsExprTransformer { + fn err_producer(msg: String) -> Result { + polars_err(msg) + } + + fn transform_literal(&self, l: &expression::Literal) -> Result { + match l.literal_type { + Some(LiteralType::Boolean(b)) => Ok(lit(b)), + Some(LiteralType::I32(i)) => Ok(lit(i)), + Some(LiteralType::I64(i)) => Ok(lit(i)), + Some(LiteralType::String(ref s)) => self.string_or_timestamp(s), + _ => Self::err_producer("unsupported literal type".to_string()), + } + } + + fn transform_selection( + &self, + _tctx: &TraversalContext, + source: &Source, + f: &expression::FieldReference, + ) -> Result { + match f.reference_type { + Some(field_reference::ReferenceType::DirectReference(ref s)) => map_on_dirref( + source, + s, + |s, r| self.get_col_from_source(s, r), + |m| Self::err_producer(m), + ), + _ => Self::err_producer("unsupported reference type".to_string()), + } + } + + fn transform_fun( + &self, + tctx: &TraversalContext, + source: &Source, + f: &expression::ScalarFunction, + ) -> Result { + let fun = tctx.get_fun_from_ext(f)?; + let args = self.get_fun_args(tctx, source, &f.arguments)?; + + connect_expr(fun, args) + } + + fn transform_list( + &self, + tctx: &TraversalContext, + source: &Source, + l: &expression::SingularOrList, + ) -> Result { + let field = match l.value { + Some(ref x) => self.transform_expr(&*x, source, tctx), + None => polars_err("no field in list expression".to_string()), + }?; + + let values = self.literal_list_to_series(&l.options, source, tctx)?; + + Ok(field.is_in(lit(values).implode())) + } +} + +// TODO: add all relevant functions +fn connect_expr(fun: Ext, args: Vec) -> PolarsTargetResult { + tracing::debug!("{:?} {:?} -> ", fun, args); + match fun { + Ext::GT if args.len() >= 2 => Ok(args[0].clone().gt(args[1].clone())), + Ext::LT if args.len() >= 2 => Ok(args[0].clone().lt(args[1].clone())), + Ext::EQ if args.len() >= 2 => Ok(args[0].clone().eq(args[1].clone())), + Ext::NE if args.len() >= 2 => Ok(args[0].clone().neq(args[1].clone())), + Ext::GE if args.len() >= 2 => Ok(args[0].clone().gt_eq(args[1].clone())), + Ext::LE if args.len() >= 2 => Ok(args[0].clone().lt_eq(args[1].clone())), + Ext::And if args.len() >= 2 => Ok(args[0].clone().logical_and(args[1].clone())), + Ext::Or if args.len() >= 2 => Ok(args[0].clone().logical_or(args[1].clone())), + Ext::Add if args.len() >= 2 => Ok(args[0].clone().add(args[1].clone())), + Ext::Sub if args.len() >= 2 => Ok(args[0].clone().sub(args[1].clone())), + Ext::Mul if args.len() >= 2 => Ok(args[0].clone().mul(args[1].clone())), + Ext::Div if args.len() >= 2 => Ok(args[0].clone().div(args[1].clone())), + Ext::Not if args.len() >= 1 => Ok(args[0].clone().not()), + unknown => polars_err(format!( + "invalid or unknown function {:?} with {} args", + unknown, + args.len() + )), + } +} + +// #[cfg(test)] +// mod test { +// use super::*; +// use polars::prelude::{DataFrame, df}; +// use substrait::proto::Plan; + +// struct TestDataStore; + +// impl DataStore for TestDataStore { +// fn get_data_source( +// &self, +// _dataset: &str, +// table_name: &str, +// _chunks: &[String], +// ) -> PolarsTargetResult { +// if table_name == "block" { +// Ok(produce_block().lazy()) +// } else { +// polars_err(format!("unknown table: '{}'", table_name)) +// } +// } +// } + +// static JSON_PLAN_BLOCK_COLS: &str = +// include_str!("../../../resources/block_plain_with_cols.json"); +// static JSON_PLAN_BLOCK_SORT: &str = +// include_str!("../../../resources/block_simple_with_sort.json"); +// static JSON_PLAN_BLOCK_IN: &str = include_str!("../../../resources/block_plain_with_in.json"); +// static JSON_PLAN_BLOCK_COOL_JOIN: &str = +// include_str!("../../../resources/block_simple_join.json"); +// static JSON_PLAN_BLOCK_SQD_JOIN: &str = +// include_str!("../../../resources/block_remote_join2.json"); + +// fn make_block_example_with_cols() -> Plan { +// serde_json::from_str(JSON_PLAN_BLOCK_COLS).unwrap() +// } + +// fn make_block_example_with_sort() -> Plan { +// serde_json::from_str(JSON_PLAN_BLOCK_SORT).unwrap() +// } + +// fn make_block_example_with_in() -> Plan { +// serde_json::from_str(JSON_PLAN_BLOCK_IN).unwrap() +// } + +// fn make_block_join_local_example() -> Plan { +// serde_json::from_str(JSON_PLAN_BLOCK_COOL_JOIN).unwrap() +// } + +// fn make_block_join_remote_example() -> Plan { +// serde_json::from_str(JSON_PLAN_BLOCK_SQD_JOIN).unwrap() +// } + +// #[allow(dead_code)] // for future reference +// fn strftime_on_timestamp(lf: LazyFrame) -> DataFrame { +// lf.with_columns([col("timestamp").dt().strftime("%Y-%m-%d %H:%M:%S")]) +// .collect() +// .unwrap() +// } + +// fn expected_simple() -> DataFrame { +// df![ +// "number" => [217710084u32, 217710085, 217710086], +// "timestamp" => [ +// "2023-09-15 12:46:38", +// "2023-09-15 12:46:38", +// "2023-09-15 12:46:38", +// ], +// ] +// .unwrap() +// } + +// fn expected_sorted() -> DataFrame { +// df![ +// "number" => [217710084u32, 217710085, 217710086], +// "hash" => [ +// "ACabnvdRdFYLutek6qjpjwkx2y6TBSnCMiFKhQF3SdSx", +// "5rcMrqfqra4eLe5Tcxf5beV8GFFQhWnX4kSmrjkeYQKY", +// "53aH29s69KerKrbuwRHBMCF2f5e37tD6v2SKMViYksQz", +// ], +// "timestamp" => [ +// "2023-09-15 12:46:38", +// "2023-09-15 12:46:38", +// "2023-09-15 12:46:38", +// ], +// ] +// .unwrap() +// } + +// #[test] +// fn test_plan_with_simple_block() { +// let ts = TestDataStore; +// let p = make_block_example_with_cols(); +// let mut tctx = TraversalContext::new(Default::default()); +// let target = +// traverse_plan::(&p, &mut tctx).expect("plan resulted in an error"); +// let df = target +// .compile(&tctx, "", &vec![], &ts) +// .expect("cannot compile target"); +// let have = df.unwrap().collect().expect("cannot run dataframe"); // strftime_on_timestamp(df.unwrap()); + +// println!("{:?}", have); +// println!("{:?}", expected_simple()); + +// assert_eq!(have, expected_simple()); +// } + +// #[test] +// // We don't support sorting on workers, +// // but a plan with a sort should reduce to a working target +// fn test_plan_with_sorted_block() { +// let ts = TestDataStore; +// let p = make_block_example_with_sort(); +// let mut tctx = TraversalContext::new(Default::default()); +// let target = +// traverse_plan::(&p, &mut tctx).expect("plan resulted in an error"); +// let df = target +// .compile(&tctx, "", &vec![], &ts) +// .expect("cannot compile target"); +// let have = df.unwrap().collect().expect("cannot run dataframe"); // strftime_on_timestamp(df.unwrap()); + +// println!("{:?}", have); +// println!("{:?}", expected_sorted()); + +// assert_eq!(have, expected_sorted()); +// } + +// #[test] +// fn test_plan_with_block_in() { +// let ts = TestDataStore; +// let p = make_block_example_with_in(); +// let mut tctx = TraversalContext::new(Default::default()); +// let target = +// traverse_plan::(&p, &mut tctx).expect("plan resulted in an error"); +// let df = target +// .compile(&tctx, "", &vec![], &ts) +// .expect("cannot compile target"); +// let have = df.unwrap().collect().expect("cannot run dataframe"); // strftime_on_timestamp(df.unwrap()); + +// println!("{:?}", have); +// println!("{:?}", expected_simple()); + +// assert_eq!(have, expected_simple()); +// } + +// #[test] +// fn test_plan_with_simple_join() { +// let p = make_block_join_local_example(); +// let mut tctx = TraversalContext::new(Default::default()); +// let target = traverse_plan::(&p, &mut tctx); + +// let msg = match target { +// Err(PlanErr::Plan(ref s)) => s, +// _ => "", +// }; + +// assert_eq!(msg, "joins are not supported on worker side"); +// } + +// #[test] +// // We don't support worker-side joins! +// fn test_plan_with_remote_join() { +// let p = make_block_join_remote_example(); +// let mut tctx = TraversalContext::new(Default::default()); +// let target = traverse_plan::(&p, &mut tctx); + +// let msg = match target { +// Err(PlanErr::Plan(ref s)) => s, +// _ => "", +// }; + +// assert_eq!(msg, "joins are not supported on worker side"); +// } + +// fn produce_block() -> DataFrame { +// df![ +// "number" => [ +// 217710051, +// 217710084, +// 217710085, +// 217710086, +// 217710364, +// 217710937, +// 217710953, +// ], +// "hash" => [ +// "3pxVPNMbfcdTEDuq6ZY7KroLX53y3hoJX4XkXFSo7LTY", +// "ACabnvdRdFYLutek6qjpjwkx2y6TBSnCMiFKhQF3SdSx", +// "5rcMrqfqra4eLe5Tcxf5beV8GFFQhWnX4kSmrjkeYQKY", +// "53aH29s69KerKrbuwRHBMCF2f5e37tD6v2SKMViYksQz", +// "7kuVRVvBCHnNkEBNcmwEAG2akARhiUDCgKWJRDmwwLXp", +// "5M2NziC9dYPKgGPqDhkrSvJCLrX43QJvDxJCQE2nUXE1", +// "6GHM1c9pLTSDbdutNGvuvub44pFP6s71R6CsAqvuhhMr", +// ], +// "parent_number" => [ +// 217710050, +// 217710051, +// 217710084, +// 217710085, +// 217710363, +// 217710936, +// 217710952, +// ], +// "parent_hash" => [ +// "3pxVPNMbfcdTEDuq6ZY7KroLX53y3hoJX4XkXFSo7LTY", +// "ACabnvdRdFYLutek6qjpjwkx2y6TBSnCMiFKhQF3SdSx", +// "5rcMrqfqra4eLe5Tcxf5beV8GFFQhWnX4kSmrjkeYQKY", +// "53aH29s69KerKrbuwRHBMCF2f5e37tD6v2SKMViYksQz", +// "7kuVRVvBCHnNkEBNcmwEAG2akARhiUDCgKWJRDmwwLXp", +// "5M2NziC9dYPKgGPqDhkrSvJCLrX43QJvDxJCQE2nUXE1", +// "6GHM1c9pLTSDbdutNGvuvub44pFP6s71R6CsAqvuhhMr", +// ], +// "height" => [ +// 200000002, +// 200000035, +// 200000036, +// 200000037, +// 200000315, +// 200000880, +// 200000896, +// ], +// "timestamp" => [ +// "2023-09-15 12:46:22", +// "2023-09-15 12:46:38", +// "2023-09-15 12:46:38", +// "2023-09-15 12:46:38", +// "2023-09-15 12:46:38", +// "2023-09-15 12:52:57", +// "2023-09-15 12:53:04", +// ], +// ] +// .unwrap() +// } +// } diff --git a/src/controller/sql_request.rs b/src/controller/sql_request.rs new file mode 100644 index 0000000..eab3764 --- /dev/null +++ b/src/controller/sql_request.rs @@ -0,0 +1,24 @@ +use std::{path::PathBuf, str::FromStr, sync::Arc}; + +use polars::prelude::{LazyFrame, ScanArgsParquet}; + +use crate::controller::polars_target::{self, DataStore}; +pub struct WorkerChunkStore { + pub path: String, +} + +impl DataStore for WorkerChunkStore { + fn get_data_source( + &self, + _dataset: &str, + table_name: &str, + _chunks: &[String], + ) -> polars_target::PolarsTargetResult { + let path = PathBuf::from_str(&format!("{}/{}.parquet", self.path, table_name)).unwrap(); + + Ok(LazyFrame::scan_parquet_files( + Arc::from([path]), + ScanArgsParquet::default(), + )?) + } +} diff --git a/src/controller/worker.rs b/src/controller/worker.rs index dec72a3..4b8f20c 100644 --- a/src/controller/worker.rs +++ b/src/controller/worker.rs @@ -3,14 +3,23 @@ use std::sync::{ Arc, }; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD as base64, Engine}; +use polars::{ + io::SerWriter, + prelude::{JsonFormat, JsonWriter}, +}; +use prost::Message; +use sql_query_plan::plan; use sqd_assignments::Assignment; use sqd_query::ParquetChunk; +use substrait::proto::Plan; use tokio_util::sync::CancellationToken; use sqd_network_transport::{Keypair, PeerId}; use tracing::instrument; use crate::{ + controller::{polars_target, sql_request::WorkerChunkStore}, metrics, query::result::{QueryError, QueryOk, QueryResult}, storage::{ @@ -23,6 +32,11 @@ use crate::{ // Use the maximum value for the uncompressed result. After compression, the result will be smaller. const RESPONSE_LIMIT: usize = sqd_network_transport::protocol::MAX_QUERY_RESULT_SIZE as usize; +pub enum QueryType { + PlainQuery, + SqlQuery, +} + pub struct Worker { state_manager: Arc, queries_running: AtomicUsize, @@ -62,6 +76,7 @@ impl Worker { block_range: Option<(u64, u64)>, chunk_id: &str, client_id: Option, + query_type: QueryType, ) -> QueryResult { let before = self.queries_running.fetch_add(1, Ordering::SeqCst); metrics::RUNNING_QUERIES.inc(); @@ -79,8 +94,15 @@ impl Worker { .map(|id| id.to_string()) .unwrap_or("{unknown}".to_string()) ); - self.execute_query(query_str, dataset, block_range, chunk_id) - .await + // self.execute_query(query_str, dataset, block_range, chunk_id) + // .await + match query_type { + QueryType::PlainQuery => { + self.execute_query(query_str, dataset, block_range, chunk_id) + .await + } + QueryType::SqlQuery => self.execute_sql_query(query_str, dataset, chunk_id).await, + } } pub async fn run(&self, cancellation_token: CancellationToken) { @@ -149,4 +171,84 @@ impl Worker { ))) }) } + + #[instrument(skip_all)] + async fn execute_sql_query( + &self, + query_str: &str, + dataset: Dataset, + chunk_id: &str, + ) -> QueryResult { + let Ok(chunk) = chunk_id.parse::() else { + return Err(QueryError::BadRequest(format!( + "Can't parse chunk id '{chunk_id}'" + ))); + }; + let Ok(query_bytes) = base64.decode(query_str) else { + return Err(QueryError::BadRequest(format!( + "Can't decode plan '{query_str}'" + ))); + }; + let Ok(plan) = Plan::decode(&query_bytes[..]) else { + return Err(QueryError::BadRequest(format!( + "Can't query plan '{query_str}'" + ))); + }; + + let Some(chunk_guard) = self + .state_manager + .clone() + .get_chunk(dataset.clone(), chunk.clone()) + else { + return Err(QueryError::NotFound); + }; + + let (tx, rx) = tokio::sync::oneshot::channel(); + let local_chunk_id = chunk_id.to_owned().clone(); + sqd_polars::POOL.spawn(move || { + let result = (move || { + let start_time = std::time::Instant::now(); + let data_source = WorkerChunkStore { + path: chunk_guard.as_str().to_owned(), + }; + let (context, target) = plan::transform_plan::(&plan) + .map_err(|err| anyhow::anyhow!("Transform error: {:?}", err))?; + let lf = target + .compile(&context, &dataset, &[local_chunk_id], &data_source) + .map_err(|err| anyhow::anyhow!("Compile error: {:?}", err))?; + let mut df = match lf { + Some(lf) => { + tracing::debug!("LF Plan: {:?}", lf.describe_plan()); + lf.collect() + .map_err(|err| anyhow::anyhow!("Planning error: {:?}", err))? + } + None => { + return Err(QueryError::from(anyhow::anyhow!("Planning error: No data"))) + } + }; + let mut buf = std::io::BufWriter::new(Vec::new()); + JsonWriter::new(&mut buf) + .with_json_format(JsonFormat::JsonLines) + .finish(&mut df) + .map_err(|err| anyhow::anyhow!("Serialization error: {:?}", err))?; + let bytes = buf + .into_inner() + .map_err(|err| anyhow::anyhow!("Serialization error: {:?}", err))?; + + if bytes.len() > RESPONSE_LIMIT { + return Err(QueryError::from(anyhow::anyhow!("Response too large"))); + } + + Ok(QueryOk::new(bytes, 1, 0, start_time.elapsed())) + })(); + tx.send(result).unwrap_or_else(|_| { + tracing::warn!("Query runner didn't wait for the result"); + }) + }); + rx.await.unwrap_or_else(|_| { + Err(QueryError::from(anyhow::anyhow!( + "Query processor didn't produce a result" + ))) + }) + } } diff --git a/src/metrics.rs b/src/metrics.rs index c43ed1a..b5cdbd7 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -87,9 +87,11 @@ pub fn query_executed(result: &QueryResult) { } pub fn register_metrics(registry: &mut Registry, version: String) { - WORKER_INFO.get_or_create(&WorkerInfoLabels { version }).set(1); + WORKER_INFO + .get_or_create(&WorkerInfoLabels { version }) + .set(1); registry.register( - "worker_info_info", // Keep the _info suffix for backward compatibility + "worker_info_info", // Keep the _info suffix for backward compatibility "Worker information with version", WORKER_INFO.clone(), );